From 0028bfdfdcf03e3f4156bdee2d599149ae1018d9 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 30 Oct 2026 19:11:35 -0500 Subject: [PATCH] ci: reject missing security scan evidence Co-authored-by: Codex (GPT-6) Signed-off-by: James Maes --- .github/scripts/check-codeql-sarif.py | 53 +++++++++++ .github/scripts/test-scan-gates.py | 89 ++++++++++++++++++ .github/workflows/codeql-analysis.yml | 125 ++++++++++++++++++++++++++ .github/workflows/codeql.yml | 2 +- 4 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/check-codeql-sarif.py create mode 100644 .github/scripts/test-scan-gates.py create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/scripts/check-codeql-sarif.py b/.github/scripts/check-codeql-sarif.py new file mode 100644 index 0000000..cf44763 --- /dev/null +++ b/.github/scripts/check-codeql-sarif.py @@ -0,0 +1,53 @@ +"""Reject missing/malformed CodeQL evidence before enforcing the severity gate.""" +import json +import math +import os +from pathlib import Path + + +def check(directory, language, threshold): + if not math.isfinite(threshold) or not 0 <= threshold <= 10: + raise ValueError("Invalid security severity threshold") + canonical = {"java-kotlin": "java", "javascript-typescript": "javascript", "c-cpp": "cpp"}.get(language, language) + files = sorted(Path(directory).glob("*.sarif")) + if not canonical or not any(path.name == canonical + ".sarif" for path in files): + raise ValueError("Missing expected CodeQL SARIF report for " + language) + blocking = [] + for path in files: + report = json.loads(path.read_text(encoding="utf-8")) + runs = report.get("runs") + if report.get("version") != "2.1.0" or not isinstance(runs, list) or not runs: + raise ValueError("Invalid or empty SARIF runs: " + path.name) + for run in runs: + driver = run.get("tool", {}).get("driver", {}) + results = run.get("results") + if driver.get("name") != "CodeQL" or not isinstance(driver.get("rules"), list) or not isinstance(results, list): + raise ValueError("Missing CodeQL tool/rules/results evidence: " + path.name) + rules = {} + for component in [driver, *run.get("tool", {}).get("extensions", [])]: + for rule in component.get("rules", []): + rules[rule["id"]] = rule + if not rules: + raise ValueError("CodeQL report has no analyzed rule metadata: " + path.name) + for invocation in run.get("invocations", []): + if invocation.get("executionSuccessful") is False: + raise ValueError("CodeQL invocation failed: " + path.name) + for result in results: + rule_id = result.get("ruleId") or result.get("rule", {}).get("id") + if rule_id not in rules: + raise ValueError("Result references missing rule metadata: " + path.name) + if result.get("suppressions"): + continue + severity = float(rules[rule_id].get("properties", {}).get("security-severity", 0) or 0) + if not math.isfinite(severity) or not 0 <= severity <= 10: + raise ValueError("Invalid rule severity: " + rule_id) + if severity >= threshold: + blocking.append((severity, rule_id)) + for severity, rule_id in sorted(blocking, reverse=True): + print(f"::error title=CodeQL security finding::{rule_id} ({severity})") + print(f"Validated {len(files)} SARIF report(s); {len(blocking)} blocking result(s)") + return bool(blocking) + + +if __name__ == "__main__": + raise SystemExit(1 if check(os.environ["SARIF_DIR"], os.environ["EXPECTED_LANGUAGE"], float(os.environ["THRESHOLD"])) else 0) diff --git a/.github/scripts/test-scan-gates.py b/.github/scripts/test-scan-gates.py new file mode 100644 index 0000000..d9356b2 --- /dev/null +++ b/.github/scripts/test-scan-gates.py @@ -0,0 +1,89 @@ +"""Exercise the exact CI guards using synthetic evidence only.""" +import contextlib +import copy +import io +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest + + +sys.dont_write_bytecode = True + + +def load(name): + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + ".py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +codeql = load("check-codeql-sarif") + + +class SarifTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + self.report = {"version": "2.1.0", "runs": [{"tool": {"driver": {"name": "CodeQL", "rules": [{"id": "synthetic", "properties": {"security-severity": "8.0"}}]}}, "results": []}]} + + def save(self, report): + (self.directory / "java.sarif").write_text(json.dumps(report), encoding="utf-8") + + def test_clean_report_with_zero_findings_passes(self): + self.save(self.report) + self.assertFalse(codeql.check(self.directory, "java-kotlin", 7)) + + def test_missing_report_or_wrong_language_fails(self): + with self.assertRaises(ValueError): + codeql.check(self.directory, "java-kotlin", 7) + self.save(self.report) + with self.assertRaises(ValueError): + codeql.check(self.directory, "csharp", 7) + + def test_empty_or_malformed_report_fails(self): + for text in ["", "{", "null", "[]"]: + with self.subTest(text=text): + (self.directory / "java.sarif").write_text(text) + with self.assertRaises((ValueError, AttributeError)): + codeql.check(self.directory, "java-kotlin", 7) + + def test_missing_runs_tool_rules_or_results_fails(self): + variants = [{"version": "2.1.0"}, {"version": "2.1.0", "runs": []}] + for field in ["tool", "results"]: + bad = copy.deepcopy(self.report) + del bad["runs"][0][field] + variants.append(bad) + bad = copy.deepcopy(self.report) + bad["runs"][0]["tool"]["driver"]["rules"] = [] + variants.append(bad) + for bad in variants: + with self.subTest(report=bad): + self.save(bad) + with self.assertRaises(ValueError): + codeql.check(self.directory, "java-kotlin", 7) + + def test_high_finding_blocks_and_unknown_rule_fails(self): + self.report["runs"][0]["results"] = [{"ruleId": "synthetic"}] + self.save(self.report) + with contextlib.redirect_stdout(io.StringIO()): + self.assertTrue(codeql.check(self.directory, "java-kotlin", 7)) + self.report["runs"][0]["results"] = [{"ruleId": "missing"}] + self.save(self.report) + with self.assertRaises(ValueError): + codeql.check(self.directory, "java-kotlin", 7) + + def test_failed_invocation_and_invalid_threshold_fail(self): + self.report["runs"][0]["invocations"] = [{"executionSuccessful": False}] + self.save(self.report) + with self.assertRaises(ValueError): + codeql.check(self.directory, "java-kotlin", 7) + with self.assertRaises(ValueError): + codeql.check(self.directory, "java-kotlin", float("nan")) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..e22af6e --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,125 @@ +# Local copy of KofTwentyTwo/standards at fdf5ed029603c938615a32b1beca3628b8787945 (v0.1.2). +# Pins, inputs and analysis checks preserved; report validation now fails closed. +# Static analysis (K22-CI-22): CodeQL with the security-extended suite, failing on +# alerts of high severity or above. Called as: +# +# jobs: +# codeql: +# uses: KofTwentyTwo/standards/.github/workflows/codeql.yml@ # vX.Y.Z +# permissions: +# contents: read +# security-events: write +# with: +# languages: '["csharp", "actions"]' +# build-mode: manual +# build-command: dotnet build MySolution.slnx -p:Platform=x64 +# runs-on: windows-latest +# +# producing `codeql / analyze ()` for each language. The CodeQL action +# uploads results but never fails a build on them, so the last step reads the SARIF +# and fails on any result whose rule has security-severity >= the threshold. +# Suppress a false positive with a query filter in a CodeQL config file passed as +# `config-file`, citing the exception record in a comment. +name: codeql + +on: + workflow_call: + inputs: + languages: + description: JSON array of CodeQL languages, e.g. '["csharp","actions"]'. + type: string + required: true + build-mode: + description: CodeQL build mode for compiled languages (none, autobuild, manual). Interpreted languages always use none. + type: string + default: autobuild + build-command: + description: Command run for build-mode manual. + type: string + default: "" + runs-on: + description: Runner label; Windows-only builds (WinUI, WPF) need windows-latest. + type: string + default: ubuntu-24.04 + setup-dotnet: + description: Install the .NET SDK from global.json before building. + type: boolean + default: false + config-file: + description: Optional path to a CodeQL configuration file in the calling repository. + type: string + default: "" + fail-on-security-severity: + description: Minimum CVSS-style security-severity that fails the job (7.0 = high). + type: string + default: "7.0" + +permissions: {} + +jobs: + analyze: + name: analyze (${{ matrix.language }}) + runs-on: ${{ inputs.runs-on }} + timeout-minutes: 60 + permissions: + contents: read + security-events: write # upload SARIF results to code scanning + strategy: + fail-fast: false + matrix: + language: ${{ fromJSON(inputs.languages) }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Test scan guard rejection paths + shell: python + run: | + import runpy + runpy.run_path(".github/scripts/test-scan-gates.py", run_name="__main__") + + - name: Set up .NET SDK + if: inputs.setup-dotnet + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + global-json-file: global.json + + - name: Initialize CodeQL + uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ contains(fromJSON('["actions","javascript-typescript","python","ruby"]'), matrix.language) && 'none' || inputs.build-mode }} + queries: security-extended + config-file: ${{ inputs.config-file }} + + - name: Build (manual mode) + if: inputs.build-mode == 'manual' && !contains(fromJSON('["actions","javascript-typescript","python","ruby"]'), matrix.language) + shell: bash + env: + BUILD_COMMAND: ${{ inputs.build-command }} + run: | + if [[ -z "$BUILD_COMMAND" ]]; then + echo "::error::build-mode is manual but build-command is empty" + exit 1 + fi + eval "$BUILD_COMMAND" + + - name: Analyze + uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + with: + category: /language:${{ matrix.language }} + output: codeql-sarif + # Fork pull requests have a read-only token; analyze still runs and gates. + upload: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'never' || 'always' }} + + - name: Gate on security severity + shell: python + env: + SARIF_DIR: codeql-sarif + THRESHOLD: ${{ inputs.fail-on-security-severity }} + EXPECTED_LANGUAGE: ${{ matrix.language }} + run: | + import runpy + runpy.run_path(".github/scripts/check-codeql-sarif.py", run_name="__main__") diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 07515fd..bc38661 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -9,7 +9,7 @@ on: permissions: {} jobs: codeql: - uses: KofTwentyTwo/standards/.github/workflows/codeql.yml@fdf5ed029603c938615a32b1beca3628b8787945 # v0.1.2 + uses: ./.github/workflows/codeql-analysis.yml permissions: contents: read security-events: write