ci: reject missing security scan evidence

Co-authored-by: Codex (GPT-6) <noreply@openai.com>
Signed-off-by: James Maes <james@kof22.com>

.github/scripts/check-codeql-sarif.py

@@ -0,0 +1,53 @@
+"""Reject missing/malformed CodeQL evidence before enforcing the severity gate."""
+import json
+import math
+import os
+from pathlib import Path
+
+
+def check(directory, language, threshold):
+    if not math.isfinite(threshold) or not 0 <= threshold <= 10:
+        raise ValueError("Invalid security severity threshold")
+    canonical = {"java-kotlin": "java", "javascript-typescript": "javascript", "c-cpp": "cpp"}.get(language, language)
+    files = sorted(Path(directory).glob("*.sarif"))
+    if not canonical or not any(path.name == canonical + ".sarif" for path in files):
+        raise ValueError("Missing expected CodeQL SARIF report for " + language)
+    blocking = []
+    for path in files:
+        report = json.loads(path.read_text(encoding="utf-8"))
+        runs = report.get("runs")
+        if report.get("version") != "2.1.0" or not isinstance(runs, list) or not runs:
+            raise ValueError("Invalid or empty SARIF runs: " + path.name)
+        for run in runs:
+            driver = run.get("tool", {}).get("driver", {})
+            results = run.get("results")
+            if driver.get("name") != "CodeQL" or not isinstance(driver.get("rules"), list) or not isinstance(results, list):
+                raise ValueError("Missing CodeQL tool/rules/results evidence: " + path.name)
+            rules = {}
+            for component in [driver, *run.get("tool", {}).get("extensions", [])]:
+                for rule in component.get("rules", []):
+                    rules[rule["id"]] = rule
+            if not rules:
+                raise ValueError("CodeQL report has no analyzed rule metadata: " + path.name)
+            for invocation in run.get("invocations", []):
+                if invocation.get("executionSuccessful") is False:
+                    raise ValueError("CodeQL invocation failed: " + path.name)
+            for result in results:
+                rule_id = result.get("ruleId") or result.get("rule", {}).get("id")
+                if rule_id not in rules:
+                    raise ValueError("Result references missing rule metadata: " + path.name)
+                if result.get("suppressions"):
+                    continue
+                severity = float(rules[rule_id].get("properties", {}).get("security-severity", 0) or 0)
+                if not math.isfinite(severity) or not 0 <= severity <= 10:
+                    raise ValueError("Invalid rule severity: " + rule_id)
+                if severity >= threshold:
+                    blocking.append((severity, rule_id))
+    for severity, rule_id in sorted(blocking, reverse=True):
+        print(f"::error title=CodeQL security finding::{rule_id} ({severity})")
+    print(f"Validated {len(files)} SARIF report(s); {len(blocking)} blocking result(s)")
+    return bool(blocking)
+
+
+if __name__ == "__main__":
+    raise SystemExit(1 if check(os.environ["SARIF_DIR"], os.environ["EXPECTED_LANGUAGE"], float(os.environ["THRESHOLD"])) else 0)

.github/scripts/test-scan-gates.py

@@ -0,0 +1,89 @@
+"""Exercise the exact CI guards using synthetic evidence only."""
+import contextlib
+import copy
+import io
+import importlib.util
+import json
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+
+
+sys.dont_write_bytecode = True
+
+
+def load(name):
+    spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + ".py"))
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+codeql = load("check-codeql-sarif")
+
+
+class SarifTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temporary.cleanup)
+        self.directory = Path(self.temporary.name)
+        self.report = {"version": "2.1.0", "runs": [{"tool": {"driver": {"name": "CodeQL", "rules": [{"id": "synthetic", "properties": {"security-severity": "8.0"}}]}}, "results": []}]}
+
+    def save(self, report):
+        (self.directory / "java.sarif").write_text(json.dumps(report), encoding="utf-8")
+
+    def test_clean_report_with_zero_findings_passes(self):
+        self.save(self.report)
+        self.assertFalse(codeql.check(self.directory, "java-kotlin", 7))
+
+    def test_missing_report_or_wrong_language_fails(self):
+        with self.assertRaises(ValueError):
+            codeql.check(self.directory, "java-kotlin", 7)
+        self.save(self.report)
+        with self.assertRaises(ValueError):
+            codeql.check(self.directory, "csharp", 7)
+
+    def test_empty_or_malformed_report_fails(self):
+        for text in ["", "{", "null", "[]"]:
+            with self.subTest(text=text):
+                (self.directory / "java.sarif").write_text(text)
+                with self.assertRaises((ValueError, AttributeError)):
+                    codeql.check(self.directory, "java-kotlin", 7)
+
+    def test_missing_runs_tool_rules_or_results_fails(self):
+        variants = [{"version": "2.1.0"}, {"version": "2.1.0", "runs": []}]
+        for field in ["tool", "results"]:
+            bad = copy.deepcopy(self.report)
+            del bad["runs"][0][field]
+            variants.append(bad)
+        bad = copy.deepcopy(self.report)
+        bad["runs"][0]["tool"]["driver"]["rules"] = []
+        variants.append(bad)
+        for bad in variants:
+            with self.subTest(report=bad):
+                self.save(bad)
+                with self.assertRaises(ValueError):
+                    codeql.check(self.directory, "java-kotlin", 7)
+
+    def test_high_finding_blocks_and_unknown_rule_fails(self):
+        self.report["runs"][0]["results"] = [{"ruleId": "synthetic"}]
+        self.save(self.report)
+        with contextlib.redirect_stdout(io.StringIO()):
+            self.assertTrue(codeql.check(self.directory, "java-kotlin", 7))
+        self.report["runs"][0]["results"] = [{"ruleId": "missing"}]
+        self.save(self.report)
+        with self.assertRaises(ValueError):
+            codeql.check(self.directory, "java-kotlin", 7)
+
+    def test_failed_invocation_and_invalid_threshold_fail(self):
+        self.report["runs"][0]["invocations"] = [{"executionSuccessful": False}]
+        self.save(self.report)
+        with self.assertRaises(ValueError):
+            codeql.check(self.directory, "java-kotlin", 7)
+        with self.assertRaises(ValueError):
+            codeql.check(self.directory, "java-kotlin", float("nan"))
+
+
+if __name__ == "__main__":
+    unittest.main()