ci: reject missing security scan evidence

Co-authored-by: Codex (GPT-6) <noreply@openai.com>
Signed-off-by: James Maes <james@kof22.com>
This commit is contained in:
james.maesandCodex committed 2026-10-30 19:11:35 -05:00
1 parent 1655d4e811
commit 0028bfdfdc
4 files changed
+268 -1

No files matched your search

+53
View File
@@ -0,0 +1,53 @@
"""Reject missing/malformed CodeQL evidence before enforcing the severity gate."""
import json
import math
import os
from pathlib import Path
def check(directory, language, threshold):
if not math.isfinite(threshold) or not 0 <= threshold <= 10:
raise ValueError("Invalid security severity threshold")
canonical = {"java-kotlin": "java", "javascript-typescript": "javascript", "c-cpp": "cpp"}.get(language, language)
files = sorted(Path(directory).glob("*.sarif"))
if not canonical or not any(path.name == canonical + ".sarif" for path in files):
raise ValueError("Missing expected CodeQL SARIF report for " + language)
blocking = []
for path in files:
report = json.loads(path.read_text(encoding="utf-8"))
runs = report.get("runs")
if report.get("version") != "2.1.0" or not isinstance(runs, list) or not runs:
raise ValueError("Invalid or empty SARIF runs: " + path.name)
for run in runs:
driver = run.get("tool", {}).get("driver", {})
results = run.get("results")
if driver.get("name") != "CodeQL" or not isinstance(driver.get("rules"), list) or not isinstance(results, list):
raise ValueError("Missing CodeQL tool/rules/results evidence: " + path.name)
rules = {}
for component in [driver, *run.get("tool", {}).get("extensions", [])]:
for rule in component.get("rules", []):
rules[rule["id"]] = rule
if not rules:
raise ValueError("CodeQL report has no analyzed rule metadata: " + path.name)
for invocation in run.get("invocations", []):
if invocation.get("executionSuccessful") is False:
raise ValueError("CodeQL invocation failed: " + path.name)
for result in results:
rule_id = result.get("ruleId") or result.get("rule", {}).get("id")
if rule_id not in rules:
raise ValueError("Result references missing rule metadata: " + path.name)
if result.get("suppressions"):
continue
severity = float(rules[rule_id].get("properties", {}).get("security-severity", 0) or 0)
if not math.isfinite(severity) or not 0 <= severity <= 10:
raise ValueError("Invalid rule severity: " + rule_id)
if severity >= threshold:
blocking.append((severity, rule_id))
for severity, rule_id in sorted(blocking, reverse=True):
print(f"::error title=CodeQL security finding::{rule_id} ({severity})")
print(f"Validated {len(files)} SARIF report(s); {len(blocking)} blocking result(s)")
return bool(blocking)
if __name__ == "__main__":
raise SystemExit(1 if check(os.environ["SARIF_DIR"], os.environ["EXPECTED_LANGUAGE"], float(os.environ["THRESHOLD"])) else 0)
+89
View File
@@ -0,0 +1,89 @@
"""Exercise the exact CI guards using synthetic evidence only."""
import contextlib
import copy
import io
import importlib.util
import json
from pathlib import Path
import sys
import tempfile
import unittest
sys.dont_write_bytecode = True
def load(name):
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + ".py"))
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
codeql = load("check-codeql-sarif")
class SarifTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.directory = Path(self.temporary.name)
self.report = {"version": "2.1.0", "runs": [{"tool": {"driver": {"name": "CodeQL", "rules": [{"id": "synthetic", "properties": {"security-severity": "8.0"}}]}}, "results": []}]}
def save(self, report):
(self.directory / "java.sarif").write_text(json.dumps(report), encoding="utf-8")
def test_clean_report_with_zero_findings_passes(self):
self.save(self.report)
self.assertFalse(codeql.check(self.directory, "java-kotlin", 7))
def test_missing_report_or_wrong_language_fails(self):
with self.assertRaises(ValueError):
codeql.check(self.directory, "java-kotlin", 7)
self.save(self.report)
with self.assertRaises(ValueError):
codeql.check(self.directory, "csharp", 7)
def test_empty_or_malformed_report_fails(self):
for text in ["", "{", "null", "[]"]:
with self.subTest(text=text):
(self.directory / "java.sarif").write_text(text)
with self.assertRaises((ValueError, AttributeError)):
codeql.check(self.directory, "java-kotlin", 7)
def test_missing_runs_tool_rules_or_results_fails(self):
variants = [{"version": "2.1.0"}, {"version": "2.1.0", "runs": []}]
for field in ["tool", "results"]:
bad = copy.deepcopy(self.report)
del bad["runs"][0][field]
variants.append(bad)
bad = copy.deepcopy(self.report)
bad["runs"][0]["tool"]["driver"]["rules"] = []
variants.append(bad)
for bad in variants:
with self.subTest(report=bad):
self.save(bad)
with self.assertRaises(ValueError):
codeql.check(self.directory, "java-kotlin", 7)
def test_high_finding_blocks_and_unknown_rule_fails(self):
self.report["runs"][0]["results"] = [{"ruleId": "synthetic"}]
self.save(self.report)
with contextlib.redirect_stdout(io.StringIO()):
self.assertTrue(codeql.check(self.directory, "java-kotlin", 7))
self.report["runs"][0]["results"] = [{"ruleId": "missing"}]
self.save(self.report)
with self.assertRaises(ValueError):
codeql.check(self.directory, "java-kotlin", 7)
def test_failed_invocation_and_invalid_threshold_fail(self):
self.report["runs"][0]["invocations"] = [{"executionSuccessful": False}]
self.save(self.report)
with self.assertRaises(ValueError):
codeql.check(self.directory, "java-kotlin", 7)
with self.assertRaises(ValueError):
codeql.check(self.directory, "java-kotlin", float("nan"))
if __name__ == "__main__":
unittest.main()
+125
View File
@@ -0,0 +1,125 @@
# Local copy of KofTwentyTwo/standards at fdf5ed029603c938615a32b1beca3628b8787945 (v0.1.2).
# Pins, inputs and analysis checks preserved; report validation now fails closed.
# Static analysis (K22-CI-22): CodeQL with the security-extended suite, failing on
# alerts of high severity or above. Called as:
#
# jobs:
# codeql:
# uses: KofTwentyTwo/standards/.github/workflows/codeql.yml@<sha> # vX.Y.Z
# permissions:
# contents: read
# security-events: write
# with:
# languages: '["csharp", "actions"]'
# build-mode: manual
# build-command: dotnet build MySolution.slnx -p:Platform=x64
# runs-on: windows-latest
#
# producing `codeql / analyze (<language>)` for each language. The CodeQL action
# uploads results but never fails a build on them, so the last step reads the SARIF
# and fails on any result whose rule has security-severity >= the threshold.
# Suppress a false positive with a query filter in a CodeQL config file passed as
# `config-file`, citing the exception record in a comment.
name: codeql
on:
workflow_call:
inputs:
languages:
description: JSON array of CodeQL languages, e.g. '["csharp","actions"]'.
type: string
required: true
build-mode:
description: CodeQL build mode for compiled languages (none, autobuild, manual). Interpreted languages always use none.
type: string
default: autobuild
build-command:
description: Command run for build-mode manual.
type: string
default: ""
runs-on:
description: Runner label; Windows-only builds (WinUI, WPF) need windows-latest.
type: string
default: ubuntu-24.04
setup-dotnet:
description: Install the .NET SDK from global.json before building.
type: boolean
default: false
config-file:
description: Optional path to a CodeQL configuration file in the calling repository.
type: string
default: ""
fail-on-security-severity:
description: Minimum CVSS-style security-severity that fails the job (7.0 = high).
type: string
default: "7.0"
permissions: {}
jobs:
analyze:
name: analyze (${{ matrix.language }})
runs-on: ${{ inputs.runs-on }}
timeout-minutes: 60
permissions:
contents: read
security-events: write # upload SARIF results to code scanning
strategy:
fail-fast: false
matrix:
language: ${{ fromJSON(inputs.languages) }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Test scan guard rejection paths
shell: python
run: |
import runpy
runpy.run_path(".github/scripts/test-scan-gates.py", run_name="__main__")
- name: Set up .NET SDK
if: inputs.setup-dotnet
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
with:
global-json-file: global.json
- name: Initialize CodeQL
uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
with:
languages: ${{ matrix.language }}
build-mode: ${{ contains(fromJSON('["actions","javascript-typescript","python","ruby"]'), matrix.language) && 'none' || inputs.build-mode }}
queries: security-extended
config-file: ${{ inputs.config-file }}
- name: Build (manual mode)
if: inputs.build-mode == 'manual' && !contains(fromJSON('["actions","javascript-typescript","python","ruby"]'), matrix.language)
shell: bash
env:
BUILD_COMMAND: ${{ inputs.build-command }}
run: |
if [[ -z "$BUILD_COMMAND" ]]; then
echo "::error::build-mode is manual but build-command is empty"
exit 1
fi
eval "$BUILD_COMMAND"
- name: Analyze
uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
with:
category: /language:${{ matrix.language }}
output: codeql-sarif
# Fork pull requests have a read-only token; analyze still runs and gates.
upload: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'never' || 'always' }}
- name: Gate on security severity
shell: python
env:
SARIF_DIR: codeql-sarif
THRESHOLD: ${{ inputs.fail-on-security-severity }}
EXPECTED_LANGUAGE: ${{ matrix.language }}
run: |
import runpy
runpy.run_path(".github/scripts/check-codeql-sarif.py", run_name="__main__")
+1 -1
View File
@@ -9,7 +9,7 @@ on:
permissions: {}
jobs:
codeql:
uses: KofTwentyTwo/standards/.github/workflows/codeql.yml@fdf5ed029603c938615a32b1beca3628b8787945 # v0.1.2
uses: ./.github/workflows/codeql-analysis.yml
permissions:
contents: read
security-events: write