ci: reject missing security scan evidence
Co-authored-by: Codex (GPT-6) <noreply@openai.com> Signed-off-by: James Maes <james@kof22.com>
This commit is contained in:
4 files changed
+268
-1
No files matched your search
@@ -0,0 +1,53 @@
|
||||
"""Reject missing/malformed CodeQL evidence before enforcing the severity gate."""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def check(directory, language, threshold):
|
||||
if not math.isfinite(threshold) or not 0 <= threshold <= 10:
|
||||
raise ValueError("Invalid security severity threshold")
|
||||
canonical = {"java-kotlin": "java", "javascript-typescript": "javascript", "c-cpp": "cpp"}.get(language, language)
|
||||
files = sorted(Path(directory).glob("*.sarif"))
|
||||
if not canonical or not any(path.name == canonical + ".sarif" for path in files):
|
||||
raise ValueError("Missing expected CodeQL SARIF report for " + language)
|
||||
blocking = []
|
||||
for path in files:
|
||||
report = json.loads(path.read_text(encoding="utf-8"))
|
||||
runs = report.get("runs")
|
||||
if report.get("version") != "2.1.0" or not isinstance(runs, list) or not runs:
|
||||
raise ValueError("Invalid or empty SARIF runs: " + path.name)
|
||||
for run in runs:
|
||||
driver = run.get("tool", {}).get("driver", {})
|
||||
results = run.get("results")
|
||||
if driver.get("name") != "CodeQL" or not isinstance(driver.get("rules"), list) or not isinstance(results, list):
|
||||
raise ValueError("Missing CodeQL tool/rules/results evidence: " + path.name)
|
||||
rules = {}
|
||||
for component in [driver, *run.get("tool", {}).get("extensions", [])]:
|
||||
for rule in component.get("rules", []):
|
||||
rules[rule["id"]] = rule
|
||||
if not rules:
|
||||
raise ValueError("CodeQL report has no analyzed rule metadata: " + path.name)
|
||||
for invocation in run.get("invocations", []):
|
||||
if invocation.get("executionSuccessful") is False:
|
||||
raise ValueError("CodeQL invocation failed: " + path.name)
|
||||
for result in results:
|
||||
rule_id = result.get("ruleId") or result.get("rule", {}).get("id")
|
||||
if rule_id not in rules:
|
||||
raise ValueError("Result references missing rule metadata: " + path.name)
|
||||
if result.get("suppressions"):
|
||||
continue
|
||||
severity = float(rules[rule_id].get("properties", {}).get("security-severity", 0) or 0)
|
||||
if not math.isfinite(severity) or not 0 <= severity <= 10:
|
||||
raise ValueError("Invalid rule severity: " + rule_id)
|
||||
if severity >= threshold:
|
||||
blocking.append((severity, rule_id))
|
||||
for severity, rule_id in sorted(blocking, reverse=True):
|
||||
print(f"::error title=CodeQL security finding::{rule_id} ({severity})")
|
||||
print(f"Validated {len(files)} SARIF report(s); {len(blocking)} blocking result(s)")
|
||||
return bool(blocking)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(1 if check(os.environ["SARIF_DIR"], os.environ["EXPECTED_LANGUAGE"], float(os.environ["THRESHOLD"])) else 0)
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Exercise the exact CI guards using synthetic evidence only."""
|
||||
import contextlib
|
||||
import copy
|
||||
import io
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
sys.dont_write_bytecode = True
|
||||
|
||||
|
||||
def load(name):
|
||||
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + ".py"))
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
codeql = load("check-codeql-sarif")
|
||||
|
||||
|
||||
class SarifTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.directory = Path(self.temporary.name)
|
||||
self.report = {"version": "2.1.0", "runs": [{"tool": {"driver": {"name": "CodeQL", "rules": [{"id": "synthetic", "properties": {"security-severity": "8.0"}}]}}, "results": []}]}
|
||||
|
||||
def save(self, report):
|
||||
(self.directory / "java.sarif").write_text(json.dumps(report), encoding="utf-8")
|
||||
|
||||
def test_clean_report_with_zero_findings_passes(self):
|
||||
self.save(self.report)
|
||||
self.assertFalse(codeql.check(self.directory, "java-kotlin", 7))
|
||||
|
||||
def test_missing_report_or_wrong_language_fails(self):
|
||||
with self.assertRaises(ValueError):
|
||||
codeql.check(self.directory, "java-kotlin", 7)
|
||||
self.save(self.report)
|
||||
with self.assertRaises(ValueError):
|
||||
codeql.check(self.directory, "csharp", 7)
|
||||
|
||||
def test_empty_or_malformed_report_fails(self):
|
||||
for text in ["", "{", "null", "[]"]:
|
||||
with self.subTest(text=text):
|
||||
(self.directory / "java.sarif").write_text(text)
|
||||
with self.assertRaises((ValueError, AttributeError)):
|
||||
codeql.check(self.directory, "java-kotlin", 7)
|
||||
|
||||
def test_missing_runs_tool_rules_or_results_fails(self):
|
||||
variants = [{"version": "2.1.0"}, {"version": "2.1.0", "runs": []}]
|
||||
for field in ["tool", "results"]:
|
||||
bad = copy.deepcopy(self.report)
|
||||
del bad["runs"][0][field]
|
||||
variants.append(bad)
|
||||
bad = copy.deepcopy(self.report)
|
||||
bad["runs"][0]["tool"]["driver"]["rules"] = []
|
||||
variants.append(bad)
|
||||
for bad in variants:
|
||||
with self.subTest(report=bad):
|
||||
self.save(bad)
|
||||
with self.assertRaises(ValueError):
|
||||
codeql.check(self.directory, "java-kotlin", 7)
|
||||
|
||||
def test_high_finding_blocks_and_unknown_rule_fails(self):
|
||||
self.report["runs"][0]["results"] = [{"ruleId": "synthetic"}]
|
||||
self.save(self.report)
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
self.assertTrue(codeql.check(self.directory, "java-kotlin", 7))
|
||||
self.report["runs"][0]["results"] = [{"ruleId": "missing"}]
|
||||
self.save(self.report)
|
||||
with self.assertRaises(ValueError):
|
||||
codeql.check(self.directory, "java-kotlin", 7)
|
||||
|
||||
def test_failed_invocation_and_invalid_threshold_fail(self):
|
||||
self.report["runs"][0]["invocations"] = [{"executionSuccessful": False}]
|
||||
self.save(self.report)
|
||||
with self.assertRaises(ValueError):
|
||||
codeql.check(self.directory, "java-kotlin", 7)
|
||||
with self.assertRaises(ValueError):
|
||||
codeql.check(self.directory, "java-kotlin", float("nan"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,125 @@
|
||||
# Local copy of KofTwentyTwo/standards at fdf5ed029603c938615a32b1beca3628b8787945 (v0.1.2).
|
||||
# Pins, inputs and analysis checks preserved; report validation now fails closed.
|
||||
# Static analysis (K22-CI-22): CodeQL with the security-extended suite, failing on
|
||||
# alerts of high severity or above. Called as:
|
||||
#
|
||||
# jobs:
|
||||
# codeql:
|
||||
# uses: KofTwentyTwo/standards/.github/workflows/codeql.yml@<sha> # vX.Y.Z
|
||||
# permissions:
|
||||
# contents: read
|
||||
# security-events: write
|
||||
# with:
|
||||
# languages: '["csharp", "actions"]'
|
||||
# build-mode: manual
|
||||
# build-command: dotnet build MySolution.slnx -p:Platform=x64
|
||||
# runs-on: windows-latest
|
||||
#
|
||||
# producing `codeql / analyze (<language>)` for each language. The CodeQL action
|
||||
# uploads results but never fails a build on them, so the last step reads the SARIF
|
||||
# and fails on any result whose rule has security-severity >= the threshold.
|
||||
# Suppress a false positive with a query filter in a CodeQL config file passed as
|
||||
# `config-file`, citing the exception record in a comment.
|
||||
name: codeql
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
languages:
|
||||
description: JSON array of CodeQL languages, e.g. '["csharp","actions"]'.
|
||||
type: string
|
||||
required: true
|
||||
build-mode:
|
||||
description: CodeQL build mode for compiled languages (none, autobuild, manual). Interpreted languages always use none.
|
||||
type: string
|
||||
default: autobuild
|
||||
build-command:
|
||||
description: Command run for build-mode manual.
|
||||
type: string
|
||||
default: ""
|
||||
runs-on:
|
||||
description: Runner label; Windows-only builds (WinUI, WPF) need windows-latest.
|
||||
type: string
|
||||
default: ubuntu-24.04
|
||||
setup-dotnet:
|
||||
description: Install the .NET SDK from global.json before building.
|
||||
type: boolean
|
||||
default: false
|
||||
config-file:
|
||||
description: Optional path to a CodeQL configuration file in the calling repository.
|
||||
type: string
|
||||
default: ""
|
||||
fail-on-security-severity:
|
||||
description: Minimum CVSS-style security-severity that fails the job (7.0 = high).
|
||||
type: string
|
||||
default: "7.0"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: analyze (${{ matrix.language }})
|
||||
runs-on: ${{ inputs.runs-on }}
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write # upload SARIF results to code scanning
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: ${{ fromJSON(inputs.languages) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Test scan guard rejection paths
|
||||
shell: python
|
||||
run: |
|
||||
import runpy
|
||||
runpy.run_path(".github/scripts/test-scan-gates.py", run_name="__main__")
|
||||
|
||||
- name: Set up .NET SDK
|
||||
if: inputs.setup-dotnet
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
||||
with:
|
||||
global-json-file: global.json
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ contains(fromJSON('["actions","javascript-typescript","python","ruby"]'), matrix.language) && 'none' || inputs.build-mode }}
|
||||
queries: security-extended
|
||||
config-file: ${{ inputs.config-file }}
|
||||
|
||||
- name: Build (manual mode)
|
||||
if: inputs.build-mode == 'manual' && !contains(fromJSON('["actions","javascript-typescript","python","ruby"]'), matrix.language)
|
||||
shell: bash
|
||||
env:
|
||||
BUILD_COMMAND: ${{ inputs.build-command }}
|
||||
run: |
|
||||
if [[ -z "$BUILD_COMMAND" ]]; then
|
||||
echo "::error::build-mode is manual but build-command is empty"
|
||||
exit 1
|
||||
fi
|
||||
eval "$BUILD_COMMAND"
|
||||
|
||||
- name: Analyze
|
||||
uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
|
||||
with:
|
||||
category: /language:${{ matrix.language }}
|
||||
output: codeql-sarif
|
||||
# Fork pull requests have a read-only token; analyze still runs and gates.
|
||||
upload: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'never' || 'always' }}
|
||||
|
||||
- name: Gate on security severity
|
||||
shell: python
|
||||
env:
|
||||
SARIF_DIR: codeql-sarif
|
||||
THRESHOLD: ${{ inputs.fail-on-security-severity }}
|
||||
EXPECTED_LANGUAGE: ${{ matrix.language }}
|
||||
run: |
|
||||
import runpy
|
||||
runpy.run_path(".github/scripts/check-codeql-sarif.py", run_name="__main__")
|
||||
@@ -9,7 +9,7 @@ on:
|
||||
permissions: {}
|
||||
jobs:
|
||||
codeql:
|
||||
uses: KofTwentyTwo/standards/.github/workflows/codeql.yml@fdf5ed029603c938615a32b1beca3628b8787945 # v0.1.2
|
||||
uses: ./.github/workflows/codeql-analysis.yml
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
Reference in new issue
Block a user