This repository has been archived on 2026-11-08. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
platform-ci/platform_ci/__main__.py

440 lines
26 KiB
Python

"""Command line entry used by the shared workflow: prepare, verify, cache-metadata, package, scan, report.
All run facts arrive through environment variables set by the workflow, never through
command line interpolation, so branch names and other event text are never parsed by a shell.
"""
import argparse
import hashlib
import json
import logging
import os
import re
import sys
import tempfile
from collections.abc import Callable, Mapping, Sequence
from pathlib import Path
from typing import cast
from platform_ci.artifact_service import require_private_artifact_service
from platform_ci.build_tools import CONTAINER_EXECUTION, DOCKER_POOL_EXECUTION, DOCKER_PROFILE_ENVIRONMENT, HOST_EXECUTION, POOL_EXECUTION
from platform_ci.caching import build_cache_metadata, cache_save_unneeded_reason, cache_unavailable_reason, cache_unsupported_reason
from platform_ci.configuration import load_project_configuration
from platform_ci.delivery.finalize_version import finalize_version
from platform_ci.delivery.forge import CurrentRun, ForgeReader, RepositoryScope, current_workflow_trust
from platform_ci.delivery.promotion import REUSE_MAX_ARTIFACT_BYTES, REUSE_RUNS_SCANNED, ReuseOutcome, decide_reuse
from platform_ci.delivery.release_tag import release_tag, tag_creating_channels
from platform_ci.delivery.reuse_forge import live_reuse_forge
from platform_ci.delivery.selection import release_activity_for_run
from platform_ci.dependency_checkout import checkout_dependencies
from platform_ci.execution import run_process, run_target_commands, sandbox_environment
from platform_ci.image_packaging import ImageRun, package_images, verify_image_binding
from platform_ci.packaging import MANIFEST_FILE_NAME, MANIFEST_SCHEMA_VERSION, PackagingError, RunIdentity, collect_outputs, manifest_digest
from platform_ci.plan import EventContext, Plan, Target, build_matrix, has_execution, prepare_plan, to_compact_json
from platform_ci.reporting import ReuseReport, render_report
from platform_ci.runtimes import RuntimeCatalog, load_runtime_catalog
from platform_ci.secret_scan import SecretScanResult, scan_history
from platform_ci.test_reports import check_junit_reports
from platform_ci.versions import read_declared_version
from platform_ci.workspace import (
GIT_METADATA_DIRECTORY_NAME,
PLATFORM_CHECKOUT_DIRECTORY_NAME,
PipelineError,
require_commit_hash,
resolve_component_directory,
)
LOG = logging.getLogger(__name__)
def main(arguments: Sequence[str] | None = None, environment: Mapping[str, str] = os.environ, runtime_catalog: RuntimeCatalog | None = None, reuse_forge: ForgeReader | None = None) -> int:
"""Run one pipeline command and return its exit code; pipeline errors become messages, not tracebacks.
`runtime_catalog` replaces the catalog shipped in the platform revision, and `reuse_forge` the read-only forge client of the
reuse step; only tests pass them.
"""
parser = argparse.ArgumentParser(prog="platform_ci")
subcommands = parser.add_subparsers(dest="command", required=True)
subcommands.add_parser("prepare").add_argument("--source", type=Path, required=True)
subcommands.add_parser("reuse").add_argument("--spool", type=Path, default=Path(tempfile.gettempdir()))
subcommands.add_parser("verify").add_argument("--source", type=Path, required=True)
subcommands.add_parser("cache-metadata").add_argument("--source", type=Path, required=True)
package_parser = subcommands.add_parser("package")
package_parser.add_argument("--source", type=Path, required=True)
package_parser.add_argument("--output", type=Path, required=True)
subcommands.add_parser("package-images").add_argument("--source", type=Path, required=True)
scan_parser = subcommands.add_parser("scan")
scan_parser.add_argument("--source", type=Path, required=True)
scan_parser.add_argument("--report", type=Path, required=True)
scan_parser.add_argument("--output", type=Path, required=True)
subcommands.add_parser("cache-save-check")
subcommands.add_parser("release-tag")
subcommands.add_parser("finalize-version").add_argument("--original-run", type=int)
subcommands.add_parser("checkout-dependencies").add_argument("--source", type=Path, required=True)
subcommands.add_parser("report")
subcommands.add_parser("require-artifact-service")
parsed = parser.parse_args(arguments)
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
commands: dict[str, Callable[[], int]] = {
# the catalog is read only by `prepare`, and only when no test supplied one
"prepare": lambda: _prepare(parsed.source, environment, runtime_catalog or load_runtime_catalog(), reuse_forge),
"reuse": lambda: _reuse(parsed.spool, environment, reuse_forge),
"verify": lambda: _verify(parsed.source, environment),
"cache-metadata": lambda: _cache_metadata(parsed.source, environment),
"package": lambda: _package(parsed.source, parsed.output, environment),
"package-images": lambda: _package_images(parsed.source, environment),
"scan": lambda: _scan(parsed.source, parsed.report, parsed.output, environment),
"cache-save-check": lambda: _cache_save_check(environment),
"release-tag": lambda: _release_tag(environment),
"finalize-version": lambda: _finalize_version(environment, parsed.original_run),
"checkout-dependencies": lambda: _checkout_dependencies(parsed.source, environment),
"report": lambda: _report(environment),
"require-artifact-service": lambda: _require_artifact_service(environment),
}
try:
return commands[parsed.command]()
except PipelineError as error:
LOG.error("%s", error)
return 1
def _prepare(source: Path, environment: Mapping[str, str], runtimes: RuntimeCatalog, forge: ForgeReader | None = None) -> int:
"""Validate `.ci.yaml`, resolve each toolchain to its qualified image, build the plan, and publish it as workflow outputs."""
project = load_project_configuration(source)
event = EventContext(
name=_require(environment, "CI_EVENT_NAME"),
ref=_require(environment, "CI_EVENT_REF"),
source_revision=_require(environment, "CI_SOURCE_REVISION"),
platform_revision=_require(environment, "CI_PLATFORM_REVISION"),
default_branch=environment.get("CI_DEFAULT_BRANCH", ""),
source_tree=_optional_tree(environment.get("CI_SOURCE_TREE", "")),
head_branch=environment.get("CI_HEAD_REF", "") if environment.get("CI_EVENT_NAME") == "pull_request" else "",
)
# the declared version is read from the checked-out source with the same reader the build's input record uses; an effective version is applied before the one build
plan = prepare_plan(
project,
event,
runtimes,
lambda target: read_declared_version(target["build_tool"], source / target["directory"], target["commands"], target.get("version_source")),
_release_activity_reader(environment, event.source_revision, forge),
)
sys.stdout.write(json.dumps(plan, indent=2, sort_keys=True) + "\n")
# the trusted tag phase runs only when the repository's own configuration opted a channel of this pushed branch in
branch = event.ref.removeprefix("refs/heads/") if event.name == "push" and event.ref.startswith("refs/heads/") else ""
tagging = bool(branch) and bool(tag_creating_channels(project, branch))
outputs = {"verify": "true" if plan["verification"]["required"] else "false", **_verification_outputs(plan, reused=False), "release_tag": "true" if tagging else "false", "plan": to_compact_json(plan)}
_write_outputs(environment, outputs)
return 0
def _release_activity_reader(environment: Mapping[str, str], source_revision: str, forge: ForgeReader | None) -> Callable[[], bool] | None:
"""Return the lazy release-activity answer of THIS run for a develop SNAPSHOT's effective version, or None when the run's identity or read-only token was not provided.
It reads, only when a plan needs it, the server's `started_at` of this run's own prepare job and the forge's commit history of the source revision (`selection.release_activity_for_run`),
the same function the trusted publisher applies to the run it selects. No clock, input file or source text takes part. With None such a plan fails closed (`ReleaseActivityError`).
"""
repository, repository_id, run_id = environment.get("CI_REPOSITORY", ""), environment.get("CI_REPOSITORY_ID", ""), environment.get("CI_RUN_ID", "")
token = environment.get("CI_FORGE_TOKEN", "")
if not repository or not repository_id.isdigit() or not run_id.isdigit() or not (token or forge is not None):
return None
scope = RepositoryScope(repository, int(repository_id), REUSE_MAX_ARTIFACT_BYTES, REUSE_RUNS_SCANNED, frozenset())
client = forge if forge is not None else live_reuse_forge(token, scope)
return lambda: release_activity_for_run(client, scope, source_revision, int(run_id))
def _optional_tree(value: str) -> str:
"""Return an absent tree id as empty (reuse is then impossible) and refuse a malformed one."""
if value and re.fullmatch(r"[0-9a-f]{40}", value) is None:
raise PipelineError(f"CI_SOURCE_TREE must be a full 40 character tree id, got {value!r}")
return value
def _verification_outputs(plan: Plan, *, reused: bool) -> dict[str, str]:
"""Return the verification matrices and flags; when tested bytes are reused there is nothing to build, so every kind is empty and off."""
required = plan["verification"]["required"] and not reused
return {
"matrix": to_compact_json({"include": []} if reused else build_matrix(plan)),
"native_matrix": to_compact_json({"include": []} if reused else build_matrix(plan, HOST_EXECUTION)),
# an empty matrix is not a valid job matrix, so each verification job also needs its own flag
"pool_matrix": to_compact_json({"include": []} if reused else build_matrix(plan, POOL_EXECUTION)),
"container_targets": "true" if required and has_execution(plan, CONTAINER_EXECUTION) else "false",
"pool_targets": "true" if required and has_execution(plan, POOL_EXECUTION) else "false",
"docker_pool_matrix": to_compact_json({"include": []} if reused else build_matrix(plan, DOCKER_POOL_EXECUTION)),
"docker_pool_targets": "true" if required and has_execution(plan, DOCKER_POOL_EXECUTION) else "false",
"native_targets": "true" if required and has_execution(plan, HOST_EXECUTION) else "false",
}
def _write_outputs(environment: Mapping[str, str], outputs: Mapping[str, str]) -> None:
"""Append single-line workflow outputs."""
with Path(_require(environment, "GITHUB_OUTPUT")).open("a", encoding="utf-8", newline="\n") as output_file:
output_file.writelines(f"{name}={value}\n" for name, value in outputs.items())
def _reuse(spool: Path, environment: Mapping[str, str], forge: ForgeReader | None) -> int:
"""Decide, before any build, whether proven tested bytes replace verification, and publish the final matrices and the evidence.
Any problem means no reuse and the normal build: a missing token, an unreachable forge, a changed input or an unproven lineage can
only add work, never remove it. The decision never touches the source checkout.
"""
plan = cast("Plan", json.loads(_require(environment, "CI_PLAN")))
outcome = _reuse_outcome(plan, spool, environment, forge)
sys.stdout.write(outcome.note + "\n")
evidence = to_compact_json(outcome.evidence) if outcome.evidence is not None else ""
_write_outputs(environment, {**_verification_outputs(plan, reused=outcome.reused), "evidence": evidence, "note": outcome.note.replace("\n", " ")})
return 0
def _reuse_outcome(plan: Plan, spool: Path, environment: Mapping[str, str], forge: ForgeReader | None) -> ReuseOutcome:
"""Run the reuse decision with the injected or live read-only client; an unavailable client is 'no reuse', not an error."""
if not plan["reuse"]["eligible"]:
return ReuseOutcome(None, f"reuse not used: {plan['reuse']['reason']}")
repository, repository_id = environment.get("CI_REPOSITORY", ""), environment.get("CI_REPOSITORY_ID", "")
if not repository or not repository_id.isdigit():
return ReuseOutcome(None, "reuse not used: the repository identity was not provided")
scope = RepositoryScope(repository, int(repository_id), REUSE_MAX_ARTIFACT_BYTES, REUSE_RUNS_SCANNED, frozenset({plan["platform_revision"]}))
client = forge
if client is None:
token = environment.get("CI_FORGE_TOKEN", "")
if not token:
return ReuseOutcome(None, "reuse not used: no read-only forge token was provided")
client = live_reuse_forge(token, scope)
trusted = _trusted_scope(client, scope, plan, environment)
if isinstance(trusted, ReuseOutcome):
return trusted
spool.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=spool) as scratch:
return decide_reuse(plan, trusted, client, Path(scratch))
def _trusted_scope(client: ForgeReader, scope: RepositoryScope, plan: Plan, environment: Mapping[str, str]) -> RepositoryScope | ReuseOutcome:
"""Return the scope that trusts exactly the workflow this run executes under, proven by the forge, or the no-reuse outcome saying why not.
Trust comes from the forge, not from any artifact. A scoped run is proven by the source-filtered listing for the canonical source id (an
identity constant of the shared workflow, not routing) plus the server-evaluated prepare job name; a repository-level run by the unscoped
listing plus its canonical caller. Any failure, including a 404 or a forge error, only means no reuse and a normal build.
"""
run_id, source_id = environment.get("CI_RUN_ID", ""), environment.get("CI_SCOPED_SOURCE_ID", "0")
if not run_id.isdigit() or not source_id.isdigit():
return ReuseOutcome(None, "reuse not used: the current run id or canonical source id was not provided")
try:
trusted = current_workflow_trust(client, scope, CurrentRun(int(run_id), plan["source_revision"], int(source_id), environment.get("CI_WORKFLOW_SHA", "")))
except (PipelineError, OSError, ValueError, LookupError) as error:
return ReuseOutcome(None, f"reuse not used: this run's workflow identity could not be read ({type(error).__name__})")
if trusted is None:
return ReuseOutcome(None, "reuse not used: the forge does not prove this run was executed by the canonical workflow, so no earlier run can be trusted")
return trusted
def _checkout_dependencies(source: Path, environment: Mapping[str, str]) -> int:
"""Fetch the target's pinned private dependencies (authorized by the operator's manifest) before the build; the credential exists only in this step."""
target = cast("Target", json.loads(_require(environment, "CI_TARGET")))
for line in checkout_dependencies(target, source, environment):
sys.stdout.write(line + "\n")
return 0
def _release_tag(environment: Mapping[str, str]) -> int:
"""Run the trusted tag-write phase (the only command that sees the write token and signing key); it never touches the source checkout."""
for line in release_tag(environment):
sys.stdout.write(line + "\n")
return 0
def _finalize_version(environment: Mapping[str, str], original_run: int | None = None) -> int:
"""Run pinned source write-back directly or through an authenticated original-run callback; never execute the source."""
for line in finalize_version(environment, original_run=original_run):
sys.stdout.write(line + "\n")
return 0
def _cache_save_check(environment: Mapping[str, str]) -> int:
"""Publish whether the dependency cache directory exists to be saved; a path that was never created is not a cache hit or a save."""
unneeded = cache_save_unneeded_reason(Path(_require(environment, "CI_CACHE_PATH")))
outputs = {"exists": "false", "note": unneeded} if unneeded else {"exists": "true", "note": "cache directory exists"}
sys.stdout.write(outputs["note"] + "\n")
_write_outputs(environment, outputs)
return 0
def _verify(source: Path, environment: Mapping[str, str]) -> int:
"""Run the target's native commands once, then judge the JUnit reports they already produced; the exit code is the verdict.
Nothing is rerun to read the reports. A declared JUnit report that matches no file is not judged here: whether a
required report exists is packaging's decision. A malformed report, a failure or an error fails the build result.
"""
# the target came from this pipeline's own prepare output, not from the source tree
target = cast("Target", json.loads(_require(environment, "CI_TARGET")))
if not run_target_commands(target, source, parent_environment=environment):
return 1
return 0 if _junit_reports_pass(target, source) else 1
def _junit_reports_pass(target: Target, source: Path) -> bool:
"""Validate every file matched by a declared `junit` report; log only the report name and generic problems."""
component_directory = resolve_component_directory(source, target["directory"])
passed = True
for name, report in sorted(target["reports"].items()):
if report["format"] != "junit":
continue
matched = _matched_report_files(component_directory, report["paths"])
if not matched:
continue
summary = check_junit_reports(source, matched)
for problem in summary.problems:
LOG.error("verify %s: junit report %r: %s", target["name"], name, problem)
passed = passed and summary.passed
return passed
def _matched_report_files(component_directory: Path, patterns: Sequence[str]) -> list[Path]:
"""Return the sorted unique files matched by the globs, skipping Git metadata and the platform checkout as packaging does."""
matches: dict[str, Path] = {}
for pattern in patterns:
for candidate in component_directory.glob(pattern):
relative = candidate.relative_to(component_directory)
if GIT_METADATA_DIRECTORY_NAME in relative.parts or PLATFORM_CHECKOUT_DIRECTORY_NAME in relative.parts[:1]:
continue
if candidate.resolve().is_file():
matches[relative.as_posix()] = candidate
return [matches[key] for key in sorted(matches)]
def _cache_metadata(source: Path, environment: Mapping[str, str]) -> int:
"""Publish the dependency-cache key, directory and save eligibility as workflow outputs. This command cannot build or touch the cache."""
target = cast("Target", json.loads(_require(environment, "CI_TARGET")))
_require(environment, "CI_EVENT_NAME")
_require(environment, "CI_EVENT_REF")
unavailable = cache_unsupported_reason(target) or cache_unavailable_reason(environment)
if unavailable is None:
metadata = build_cache_metadata(target, source, environment)
outputs = {"available": "true", "key": metadata.key, "path": metadata.path, "save": "true" if metadata.save else "false"}
if metadata.trivy_key:
outputs |= {"trivy_available": "true", "trivy_key": metadata.trivy_key, "trivy_path": metadata.trivy_path, "trivy_save": outputs["save"]}
else:
# nothing is hashed or published for the cache actions, so neither can be attempted
outputs = {"available": "false", "save": "false", "note": unavailable}
sys.stdout.write("".join(f"{name}={value}\n" for name, value in outputs.items()))
with Path(_require(environment, "GITHUB_OUTPUT")).open("a", encoding="utf-8", newline="\n") as output_file:
output_file.writelines(f"{name}={value}\n" for name, value in outputs.items())
return 0
def _package(source: Path, output: Path, environment: Mapping[str, str]) -> int:
"""Collect the outputs the verification build left behind. This command cannot build."""
target = cast("Target", json.loads(_require(environment, "CI_TARGET")))
build_outcome = _require(environment, "CI_BUILD_OUTCOME")
if build_outcome not in ("success", "failure"):
raise PipelineError(f"CI_BUILD_OUTCOME must be success or failure, got {build_outcome!r}")
identity = RunIdentity(
source_revision=require_commit_hash(_require(environment, "CI_SOURCE_REVISION"), "source revision"),
platform_revision=require_commit_hash(_require(environment, "CI_PLATFORM_REVISION"), "platform revision"),
run_id=_require(environment, "CI_RUN_ID"),
run_attempt=_require(environment, "CI_RUN_ATTEMPT"),
source_tree=_require(environment, "CI_SOURCE_TREE") if build_outcome == "success" else environment.get("CI_SOURCE_TREE", ""),
)
manifest = collect_outputs(target, source, output, identity, build_succeeded=build_outcome == "success")
if build_outcome == "success":
# the bundle must retain the image that was scanned, built from exactly the tested bytes it lists
verify_image_binding(target, source, manifest)
sys.stdout.write(f"collected {len(cast('list[object]', manifest['files']))} files, manifest digest {manifest['digest']}\n")
return 0
def _package_images(source: Path, environment: Mapping[str, str]) -> int:
"""Package the target's declared images from the tested outputs in the checkout. This command cannot build the source."""
target = cast("Target", json.loads(_require(environment, "CI_TARGET")))
if not target["images"]:
sys.stdout.write("no images declared for this target\n")
return 0
# The docker profile's test-daemon contract belongs to the native verification only: image packaging never talks to that daemon (it uses BuildKit),
# and its captured boundary check refuses a job whose environment points at a Docker daemon, so those fixed variables are not passed on.
docker_variables = {name for name, _value in DOCKER_PROFILE_ENVIRONMENT}
tool_environment = sandbox_environment(environment) | {name: value for name, value in target["environment"].items() if name not in docker_variables}
run = ImageRun(require_commit_hash(_require(environment, "CI_SOURCE_REVISION"), "source revision"), require_commit_hash(_require(environment, "CI_PLATFORM_REVISION"), "platform revision"), run_process, tool_environment)
sys.stdout.write("".join(f"packaged image {image.name}, archive sha256 {image.archive_sha256}\n" for image in package_images(target, source, run)))
return 0
def _scan(source: Path, report: Path, output: Path, environment: Mapping[str, str]) -> int:
"""Scan the exact source revision's full history once and write the redacted security manifest; a failed scan exits 1.
The manifest carries counts and the raw report's size and digest, never findings, log bodies or secret text.
"""
identity = RunIdentity(
source_revision=_require(environment, "CI_SOURCE_REVISION"),
platform_revision=require_commit_hash(_require(environment, "CI_PLATFORM_REVISION"), "platform revision"),
run_id=_require(environment, "CI_RUN_ID"),
run_attempt=_require(environment, "CI_RUN_ATTEMPT"),
)
if output.exists():
raise PackagingError(f"output directory {output} already exists")
result = scan_history(source, identity.source_revision, report)
manifest = _security_manifest(identity, result)
output.mkdir(parents=True)
(output / MANIFEST_FILE_NAME).write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n")
sys.stdout.write(f"secret scan {'passed' if result.passed else 'FAILED'}: findings={result.findings}, manifest digest {manifest['digest']}\n")
return 0 if result.passed else 1
def _security_manifest(identity: RunIdentity, result: SecretScanResult) -> dict[str, object]:
"""Bind the scan outcome to the source, platform and run; describe the raw report by digest and size only."""
report: dict[str, object] | None = None
if result.report_path is not None and result.report_path.is_file():
with result.report_path.open("rb") as stream:
report = {"sha256": hashlib.file_digest(stream, "sha256").hexdigest(), "size": result.report_path.stat().st_size}
manifest: dict[str, object] = {
"schema": MANIFEST_SCHEMA_VERSION,
"kind": "secret-scan",
"source_revision": identity.source_revision,
"platform_revision": identity.platform_revision,
"run_id": identity.run_id,
"run_attempt": identity.run_attempt,
"scope": "full history reachable from source_revision",
"passed": result.passed,
"findings": result.findings,
"exit_code": result.exit_code,
"gitleaks_version": result.gitleaks_version,
"failure": result.failure,
"report": report,
}
manifest["digest"] = manifest_digest(manifest)
return manifest
def _report(environment: Mapping[str, str]) -> int:
"""Print expected versus actual results; fail when preparation, verification or the secret scan did not end as required."""
if _require(environment, "CI_PREPARE_RESULT") != "success":
raise PipelineError("preparation did not succeed, so there is no plan and no work was verified")
plan = cast("Plan", json.loads(_require(environment, "CI_PLAN")))
text, passed = render_report(
plan,
_require(environment, "CI_VERIFY_RESULT"),
_require(environment, "CI_SECRETS_RESULT"),
environment.get("CI_NATIVE_VERIFY_RESULT", "skipped"),
ReuseReport(environment.get("CI_REUSE_EVIDENCE", ""), environment.get("CI_REUSE_NOTE", "")),
pool_verification_result=environment.get("CI_POOL_VERIFY_RESULT", "skipped"),
docker_pool_verification_result=environment.get("CI_DOCKER_POOL_VERIFY_RESULT", "skipped"),
)
sys.stdout.write(text)
return 0 if passed else 1
def _require_artifact_service(environment: Mapping[str, str]) -> int:
"""Fail unless the job's artifact endpoints are the private forge; one command that works in any runner shell."""
require_private_artifact_service(environment)
return 0
def _require(environment: Mapping[str, str], name: str) -> str:
"""Return a required environment variable or fail naming it."""
value = environment.get(name, "")
if not value:
raise PipelineError(f"required environment variable {name} is not set")
return value
if __name__ == "__main__":
sys.exit(main())