diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fbd44e0..99bfc96 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,8 +3,16 @@ name: Release 4.1.0 on: workflow_dispatch: inputs: + version: + description: Reviewed source version (GA or exact coordinated RC) + required: true + type: choice + default: '4.1.0' + options: + - '4.1.0' + - '4.1.0-RC.1' release_sha: - description: Full develop commit SHA already tagged 4.1.0 + description: Full develop commit SHA already signed and tagged with the selected version required: true type: string smoke_run_id: @@ -32,8 +40,8 @@ jobs: SMOKE_RUN_ID: ${{ inputs.smoke_run_id }} RELEASE_ENABLED: ${{ vars.QQQ_ALL_RELEASE_ENABLED }} LICENSE_CLEARED: ${{ vars.QQQ_ALL_RELEASE_LICENSE_APPROVED }} - VERSION: 4.1.0 - TAG: 4.1.0 + VERSION: ${{ inputs.version }} + TAG: ${{ inputs.version }} steps: - uses: actions/checkout@v4 with: @@ -46,19 +54,21 @@ jobs: test "$GITHUB_REF" = refs/heads/develop test "$RELEASE_ENABLED" = true test "$LICENSE_CLEARED" = true + case "$VERSION" in 4.1.0|4.1.0-RC.1) ;; *) exit 1 ;; esac [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$SMOKE_RUN_ID" =~ ^[0-9]+$ ]] test "$(git rev-parse HEAD)" = "$GITHUB_SHA" test "$RELEASE_SHA" = "$GITHUB_SHA" test "$(git cat-file -t "refs/tags/$TAG")" = tag test "$(git rev-parse "refs/tags/$TAG^{commit}")" = "$RELEASE_SHA" - - name: Verify upstream GA, exact-commit smoke, Central POMs, and public GHCR + gh api "repos/$GITHUB_REPOSITORY/git/tags/$(git rev-parse "refs/tags/$TAG")" > "$RUNNER_TEMP/tag-object.json" + - name: Verify upstream release, signed tag, exact-commit smoke, public pins, and public GHCR shell: bash run: | set -euo pipefail - ga_state="$(gh api repos/QRun-IO/qqq/releases/tags/v4.1.0 \ - --jq 'if (.tag_name == "v4.1.0" and .draft == false and .prerelease == false) then "ready" else "blocked" end')" - test "$ga_state" = ready + gh api "repos/QRun-IO/qqq/releases/tags/v$VERSION" > "$RUNNER_TEMP/upstream-release.json" + next_version="$(python3 -c 'import sys; sys.path.insert(0, "scripts"); from release_gate import properties, validate_poms; from pathlib import Path; import os; validate_poms(Path("."), os.environ["VERSION"]); print(properties(Path("qqq-all-bom/pom.xml"))["qqq-frontend-next.version"])')" + gh api "repos/QRun-IO/qqq-frontend-next/releases/tags/v$next_version" > "$RUNNER_TEMP/next-release.json" gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SMOKE_RUN_ID" > "$RUNNER_TEMP/smoke-run.json" gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SMOKE_RUN_ID/jobs?per_page=100" > "$RUNNER_TEMP/smoke-jobs.json" central="https://repo.maven.apache.org/maven2/com/kingsrook/qqq" @@ -71,7 +81,12 @@ jobs: --smoke-run "$RUNNER_TEMP/smoke-run.json" \ --smoke-jobs "$RUNNER_TEMP/smoke-jobs.json" \ --published-parent "$RUNNER_TEMP/qqq-all-parent.pom" \ - --published-bom "$RUNNER_TEMP/qqq-all-bom.pom" + --published-bom "$RUNNER_TEMP/qqq-all-bom.pom" \ + --upstream-release "$RUNNER_TEMP/upstream-release.json" \ + --next-release "$RUNNER_TEMP/next-release.json" \ + --tag-object "$RUNNER_TEMP/tag-object.json" \ + --tag-sha "$(git rev-parse "refs/tags/$TAG")" \ + --public-inputs "$RUNNER_TEMP/public-inputs" test "$(gh api /orgs/QRun-IO/packages/container/qqq-all --jq .visibility)" = public gh api --paginate 'orgs/QRun-IO/packages/container/qqq-all/versions?per_page=100' \ --jq '.[] | .metadata.container.tags[]?' > "$RUNNER_TEMP/ghcr-tags.txt" @@ -85,12 +100,20 @@ jobs: java-version: '21' cache: maven - name: Verify release reactor - run: mvn -B -ntp verify + shell: bash + run: | + set -euo pipefail + printf '\n' > "$RUNNER_TEMP/public-settings.xml" + python3 -m unittest discover -s scripts -p 'test_*.py' -v + mvn -B -ntp -s "$RUNNER_TEMP/public-settings.xml" -gs "$RUNNER_TEMP/public-settings.xml" \ + -Dmaven.repo.local="$RUNNER_TEMP/release-m2" verify \ + org.apache.maven.plugins:maven-enforcer-plugin:3.5.0:enforce -Drules=requireReleaseDeps - name: Recheck develop, annotated tag, and release absence before publication shell: bash run: | set -euo pipefail test "$(gh api "repos/$GITHUB_REPOSITORY/branches/develop" --jq .commit.sha)" = "$RELEASE_SHA" + test "$(git ls-remote origin "refs/tags/$TAG" | cut -f1)" = "$(git rev-parse "refs/tags/$TAG")" test "$(git ls-remote origin "refs/tags/$TAG^{}" | cut -f1)" = "$RELEASE_SHA" if release_check="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$TAG" 2>&1)"; then echo "GitHub release $TAG already exists" >&2 @@ -99,6 +122,12 @@ jobs: echo "$release_check" >&2 exit 1 fi + gh api --paginate 'orgs/QRun-IO/packages/container/qqq-all/versions?per_page=100' \ + --jq '.[] | .metadata.container.tags[]?' > "$RUNNER_TEMP/ghcr-tags-final.txt" + if grep -Fxq "$VERSION" "$RUNNER_TEMP/ghcr-tags-final.txt"; then + echo "GHCR tag $VERSION already exists" >&2 + exit 1 + fi - uses: docker/login-action@v3 with: registry: ghcr.io @@ -115,10 +144,36 @@ jobs: --label "org.opencontainers.image.revision=$RELEASE_SHA" \ --tag "$image" . docker push "$image" - printf 'Published image: `%s`\n' "$image" >> "$GITHUB_STEP_SUMMARY" - - name: Publish GitHub release with Compose asset + printf 'Published image: %s\n' "$image" >> "$GITHUB_STEP_SUMMARY" + - name: Build image Compose distribution bundle shell: bash run: | set -euo pipefail - gh release create "$TAG" compose.yaml --repo "$GITHUB_REPOSITORY" \ - --verify-tag --title "qqq-all $VERSION" --notes-from-tag + image="$(docker image inspect --format '{{index .RepoDigests 0}}' "ghcr.io/qrun-io/qqq-all:$VERSION")" + python3 scripts/release_bundle.py --root . --version "$VERSION" --sha "$RELEASE_SHA" \ + --image "$image" --output "$RUNNER_TEMP" + - name: Publish GitHub release with Compose bundle + shell: bash + run: | + set -euo pipefail + release_flags=() + case "$VERSION" in + 4.1.0) ;; + 4.1.0-RC.1) release_flags=(--prerelease --latest=false) ;; + *) exit 1 ;; + esac + gh release create "$TAG" "$RUNNER_TEMP/qqq-all-$VERSION.tar.gz" \ + "$RUNNER_TEMP/qqq-all-$VERSION/compose.yaml" --repo "$GITHUB_REPOSITORY" \ + --verify-tag --title "qqq-all $VERSION" --notes-from-tag "${release_flags[@]}" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: release-evidence-${{ inputs.version }} + path: | + ${{ runner.temp }}/public-inputs/receipts.json + ${{ runner.temp }}/tag-object.json + ${{ runner.temp }}/upstream-release.json + ${{ runner.temp }}/next-release.json + ${{ runner.temp }}/smoke-run.json + ${{ runner.temp }}/smoke-jobs.json + ${{ runner.temp }}/qqq-all-${{ inputs.version }}/release.json diff --git a/compose.yaml b/compose.yaml index 1dbf5d6..ccd5693 100644 --- a/compose.yaml +++ b/compose.yaml @@ -181,7 +181,9 @@ services: profiles: [full] command: "${SFTP_USER:-qqq}:${SFTP_PASSWORD:?set SFTP_PASSWORD}:1001:1001:upload" volumes: - - sftp-data:/home/${SFTP_USER:-qqq}/upload + - type: volume + source: sftp-data + target: /home/${SFTP_USER:-qqq}/upload healthcheck: test: ["CMD-SHELL", "pgrep sshd >/dev/null"] interval: 5s diff --git a/docs/release.md b/docs/release.md index c245d92..b3daae6 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,6 +1,18 @@ # 4.1.0 release preparation -This procedure prepares a release; this repository currently remains on `4.1.0-SNAPSHOT`. The [manual release workflow](../.github/workflows/release.yml) only publishes the versioned GHCR image and a GitHub release containing `compose.yaml`. It does **not** deploy Maven artifacts or create a tag. Publication is authorized once the release gates pass. Keep the workflow disabled until Apache-2.0 alignment, public artifact verification and GHCR visibility are complete. +This procedure prepares a release; this repository currently remains on `4.1.0-SNAPSHOT`. The [manual release workflow](../.github/workflows/release.yml) publishes the versioned GHCR image and a GitHub release containing an image-based Compose bundle plus `compose.yaml`. It does **not** deploy Maven artifacts or create a tag. Publication is authorized once the release gates pass. Keep the workflow disabled until Apache-2.0 alignment, public artifact verification and GHCR visibility are complete. + +## Exact coordinated RC under QQQ #921 + +The same workflow accepts only `4.1.0` (default GA path) or exact `4.1.0-RC.1`. For the [coordinated RC chain](https://github.com/QRun-IO/qqq/issues/921), first publish and verify the accepted Next candidate, QQQ `v4.1.0-RC.1`, and the corresponding build parent/eight QBits. Then review actual source pins: root `revision` and BOM `qqq.version` both `4.1.0-RC.1`, explicit accepted Next version, and immutable published QBit versions. No command-line version overrides substitute for source edits. The current RC8/snapshot pins are intentionally unchanged and cannot pass release validation. Acceptance limitations belong to the exact accepted pair; the RC8 notes below are historical evidence, not automatic approval for a newer pair. + +Use the existing `release-central` profile and the parent/BOM-only command below to stage the exact RC POMs for **manual** Central review (`autoPublish=false`). Its enforcers accept only GA or exact RC1, require the QQQ pin to equal the source revision, and reject snapshot/range/LATEST dependency pins. After both POMs are public and the reviewed source is merged, require successful push `Smoke` core/full on that exact develop SHA and an annotated signed tag named `4.1.0-RC.1` at it. The workflow requires GitHub's tag verification to report `verified=true`, `reason=valid`, and checks the exact local/remote tag object and commit. Public QQQ/Next release tags must match their pins and prerelease state; first-party BOM inputs (including recursive public parents) must resolve anonymously, with qbit-build-parent importing this QQQ version. The full reactor also rejects transitive snapshot dependencies. Public POM comparison includes the explicit Next pin. Existing GHCR visibility, protected environment, license/enable controls, destination-absence checks and live ref rechecks remain required. + +When those gates and owner authorization are satisfied, dispatch **Release 4.1.0** on develop with `version=4.1.0-RC.1`, the full `release_sha`, and its successful push `smoke_run_id`. RC publication uses `--prerelease --latest=false`; it never becomes the latest GA release. The default `version=4.1.0` still requires upstream non-prerelease GA and follows the separate GA requirements below. Do not dispatch while any pin or external prerequisite is pending. + +## Image Compose bundle + +The workflow attaches `qqq-all-.tar.gz` and its rendered `compose.yaml`. Download/extract the **whole bundle**, retain the `infra` directory beside Compose, and follow its README. Both app profiles use the exact pushed GHCR digest and have no app build context. The bundle includes the tracked PostgreSQL/MySQL/Mongo initialization, Keycloak realm, MinIO Dockerfile, `.env.example`, LICENSE/NOTICE and `release.json` source/image identity. It excludes local `.env`, application source and Maven outputs. Full-profile MinIO still builds locally and needs network access; other service images are unchanged. The standalone Compose asset alone is insufficient for full-profile operation. Configure a unique Compose project and demo credentials, build only MinIO for full, then start with `--no-build`. Verify anonymous image pull and real core/full startup from the extracted bundle after publication; config-rendering tests are not runtime acceptance. ## Next RC8 integration @@ -25,10 +37,10 @@ The owner selected Apache-2.0 for first-party code in [QQQ #923](https://github. ## Exact commit and manual dispatch -After the release POMs are public, merge the reviewed version-pin change and the C7 smoke workflow to `develop`. Require a successful **push** run of `Smoke` on the exact intended commit; both `core` and `full` jobs must pass. Review the run's full SHA and run ID in Actions. Create an **annotated** `4.1.0` tag at that exact `develop` commit through the normal signed-tag process. The workflow checks that the local tag is annotated and that the remote tag resolves to the same SHA; it does not verify a tag signature, so the operator must do that separately. Do not move the tag or advance `develop` before dispatch. +After the release POMs are public, merge the reviewed version-pin change and the C7 smoke workflow to `develop`. Require a successful **push** run of `Smoke` on the exact intended commit; both `core` and `full` jobs must pass. Review the run's full SHA and run ID in Actions. Create an **annotated**, signed `4.1.0` tag at that exact `develop` commit. The workflow checks GitHub's valid signature verification, the tag object and its exact commit, including live remote refs. Do not move the tag or advance `develop` before dispatch. Configure a protected `qqq-all-release` GitHub environment with required reviewers and branch restriction to `develop`. After Apache alignment and the publication preconditions pass, set environment variables `QQQ_ALL_RELEASE_LICENSE_APPROVED=true` and `QQQ_ALL_RELEASE_ENABLED=true`. Unset them to close the gate. These variables and the environment approval are operational controls, not substitutes for the release review. -Dispatch **Release 4.1.0** on `develop` with `release_sha` (the full 40-character commit SHA) and `smoke_run_id` (the successful push run ID). The job requires that checkout, `develop`, the annotated tag, and the smoke run all refer to that SHA. It checks upstream GA, both public Central POMs, public GHCR visibility, and the absence of a `4.1.0` GHCR tag; runs `mvn -B -ntp verify`; then rechecks the live `develop` and remote tag refs before publication. Only then does it build and push `ghcr.io/qrun-io/qqq-all:4.1.0` and create the GitHub `4.1.0` release with `compose.yaml` attached. +Dispatch **Release 4.1.0** on `develop` with `version=4.1.0` (default), `release_sha` (the full 40-character commit SHA) and `smoke_run_id` (the successful push run ID). The job requires that checkout, `develop`, the annotated signed tag, and the smoke run all refer to that SHA. It checks upstream GA, both public Central POMs and immutable upstream inputs, public GHCR visibility, and destination absence; verifies the reactor with a fresh public-only cache and rejects transitive snapshots; then rechecks the live `develop` and remote tag refs before publication. Only then does it build and push `ghcr.io/qrun-io/qqq-all:4.1.0` and create the GitHub `4.1.0` release with the Compose bundle and rendered `compose.yaml` attached. The image push and GitHub release creation are separate API operations, not an atomic transaction. If the release step fails after the image push, inspect the image digest and workflow logs before recovery; never reuse or overwrite the version tag blindly. Verify the public image pull, release asset, and both Maven POMs from an unauthenticated environment after publication. No workflow in this preparation branch publishes or tags anything by itself. diff --git a/pom.xml b/pom.xml index ca9119f..6522963 100644 --- a/pom.xml +++ b/pom.xml @@ -62,15 +62,15 @@ 3.5.0 - require-ga-revision + require-release-revision validate enforce revision - ^4[.]1[.]0$ - Central release requires a reviewed 4.1.0 source revision. + ^4[.]1[.]0(-RC[.]1)?$ + Central release requires reviewed 4.1.0 or exact 4.1.0-RC.1 source pins. diff --git a/qqq-all-bom/pom.xml b/qqq-all-bom/pom.xml index 5152769..76f7a07 100644 --- a/qqq-all-bom/pom.xml +++ b/qqq-all-bom/pom.xml @@ -56,16 +56,17 @@ qqq.version - ^4[.]1[.]0$ + ^\Q${revision}\E$ - qbit-quick-search.version^(?!.*SNAPSHOT).+$ - qbit-user-role-permissions.version^(?!.*SNAPSHOT).+$ - qbit-customizable-table-views.version^(?!.*SNAPSHOT).+$ - qbit-standard-process-trace.version^(?!.*SNAPSHOT).+$ - qbit-webhooks.version^(?!.*SNAPSHOT).+$ - qbit-workflows.version^(?!.*SNAPSHOT).+$ - qbit-geo-data.version^(?!.*SNAPSHOT).+$ - qbit-sftp-data-integration.version^(?!.*SNAPSHOT).+$ + qqq-frontend-next.version^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$ + qbit-quick-search.version^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$ + qbit-user-role-permissions.version^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$ + qbit-customizable-table-views.version^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$ + qbit-standard-process-trace.version^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$ + qbit-webhooks.version^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$ + qbit-workflows.version^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$ + qbit-geo-data.version^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$ + qbit-sftp-data-integration.version^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$ diff --git a/scripts/release_bundle.py b/scripts/release_bundle.py new file mode 100644 index 0000000..a57f2d4 --- /dev/null +++ b/scripts/release_bundle.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Package digest-pinned Compose with the tracked full-profile infrastructure.""" + +import argparse +import json +import os +import re +import shutil +import subprocess +import tarfile +from pathlib import Path + +from release_gate import validate_version + + +def create_bundle(root, version, sha, image, output): + validate_version(version) + if not re.fullmatch(r"[0-9a-f]{40}", sha): + raise ValueError("Source SHA must be a full commit ID") + if not re.fullmatch(r"ghcr\.io/qrun-io/qqq-all@sha256:[0-9a-f]{64}", image): + raise ValueError("Release image must be the qqq-all GHCR digest") + environment = {key: value for key, value in os.environ.items() if not key.startswith("COMPOSE_")} + rendered = subprocess.run( + ["docker", "compose", "--env-file", ".env.example", "--profile", "*", "-f", "compose.yaml", + "config", "--no-interpolate", "--no-env-resolution", "--no-path-resolution", "--format", "json"], + cwd=root, env=environment, capture_output=True, text=True, check=True, + ) + compose = json.loads(rendered.stdout) + compose.pop("x-app-image", None) + for name in ("app-core", "app-full"): + app = compose["services"][name] + app.pop("build", None) + app["image"] = image + app["pull_policy"] = "always" + compose["services"]["minio"]["image"] = "qqq-all-minio:" + version + name = "qqq-all-" + version + bundle = output / name + bundle.mkdir(parents=True, exist_ok=False) + (bundle / "compose.yaml").write_text(json.dumps(compose, indent=2) + "\n") + tracked = subprocess.check_output(["git", "-C", str(root), "ls-files", "--", "infra"], text=True).splitlines() + for filename in (".env.example", "LICENSE", "NOTICE", *tracked): + source = root / filename + if source.is_symlink() or not source.is_file(): + raise ValueError(f"Bundle input must be a regular file: {filename}") + target = bundle / filename + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + (bundle / "release.json").write_text(json.dumps( + {"version": version, "source_sha": sha, "image": image}, indent=2) + "\n") + (bundle / "README.md").write_text(f"""# qqq-all {version} + +Extract this complete bundle and run commands from this directory. The application uses +`{image}` (source `{sha}`). No application source, Maven or app build is required. +Docker with Compose v2 is required. Copy `.env.example` to `.env` and replace +the demonstration passwords before sharing the environment. All fixture values remain +demo-only; retain the infra directory next to compose.yaml. + +```bash +cp .env.example .env +docker compose -p qqq-all-demo --profile core up -d --wait --no-build +``` + +For the full profile, stop core first. The included MinIO Dockerfile still builds locally +and needs network access; PostgreSQL/MySQL/Mongo initialization and the Keycloak realm +use the bundled infra files. External service images also need network access. + +```bash +docker compose -p qqq-all-demo --profile core down +docker compose -p qqq-all-demo --profile full build minio +docker compose -p qqq-all-demo --profile full up -d --wait --no-build +``` + +Use a distinct Compose project name for each installation. The application listens on +http://localhost:8080 by default. The full-profile identity provider uses +http://keycloak.localhost:8081; that hostname must resolve to loopback on the client. +Ports and fixture settings are in `.env.example`. `docker compose ... down` retains +named data volumes; remove volumes only when intentionally resetting that installation. +The standalone compose.yaml asset needs this bundle's infra and environment template +for full-profile operation. See release.json for the exact application identity. +""") + archive = output / (name + ".tar.gz") + with tarfile.open(archive, "x:gz") as handle: + handle.add(bundle, arcname=name) + return archive + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--sha", required=True) + parser.add_argument("--image", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(create_bundle(args.root.resolve(), args.version, args.sha, args.image, args.output.resolve())) + + +if __name__ == "__main__": + main() diff --git a/scripts/release_gate.py b/scripts/release_gate.py index 922f01a..d2db81c 100644 --- a/scripts/release_gate.py +++ b/scripts/release_gate.py @@ -1,9 +1,12 @@ -"""Fail-closed checks for the manual 4.1.0 release workflow.""" +"""Fail-closed checks for the manual 4.1.0 GA / exact RC.1 release workflow.""" import argparse +import hashlib import json import re import sys +import urllib.error +import urllib.request from pathlib import Path from xml.etree import ElementTree @@ -13,6 +16,16 @@ QBITS = ( "standard-process-trace", "webhooks", "workflows", "geo-data", "sftp-data-integration", ) +RELEASE_VERSIONS = ("4.1.0", "4.1.0-RC.1") + + +def immutable_version(value): + return re.fullmatch(r"[0-9]+(?:\.[0-9]+){2,3}(?:-RC\.[1-9][0-9]*)?", value) is not None + + +def validate_version(version): + if version not in RELEASE_VERSIONS: + raise GateError("This release workflow is limited to 4.1.0 and 4.1.0-RC.1") class GateError(ValueError): @@ -64,7 +77,7 @@ def managed_dependencies(path): reference = re.fullmatch(r"\$\{([^}]+)\}", version) if reference: version = values.get(reference.group(1), "") - if not version or "${" in version or "SNAPSHOT" in version.upper(): + if not immutable_version(version): raise GateError(f"Unreleased managed dependency in {path}: " f"{field(dependency, 'artifactId')}") managed.append((field(dependency, "groupId"), field(dependency, "artifactId"), @@ -84,6 +97,9 @@ def validate_managed_bom(path, version): if entries.get(("com.kingsrook.qqq", "qqq-bom-pom")) != (version, "pom", "import"): raise GateError(f"QQQ BOM import differs from {version} in {path}") values = properties(path) + if entries.get(("com.kingsrook.qqq", "qqq-frontend-next")) != ( + values.get("qqq-frontend-next.version"), "jar", "compile"): + raise GateError(f"Managed Next differs from its release pin in {path}") for name in QBITS: artifact = f"qbit-{name}" if entries.get(("com.kingsrook.qbits", artifact)) != ( @@ -102,14 +118,14 @@ def validate_published_poms(parent_path, bom_path, version, local_bom): raise GateError(f"Published {artifact} POM is not the {version} release") published = properties(bom_path) if published.get("qqq.version") != version: - raise GateError("Published BOM does not import QQQ 4.1.0") + raise GateError(f"Published BOM does not import QQQ {version}") for name in QBITS: key = f"qbit-{name}.version" value = published.get(key, "") - if not value or "SNAPSHOT" in value.upper(): + if not immutable_version(value): raise GateError(f"Published BOM {key} is not a release") local = properties(local_bom) - for key in ("qqq.version", *(f"qbit-{name}.version" for name in QBITS)): + for key in ("qqq.version", "qqq-frontend-next.version", *(f"qbit-{name}.version" for name in QBITS)): if published.get(key) != local.get(key): raise GateError(f"Published BOM {key} differs from release source") if validate_managed_bom(bom_path, version) != validate_managed_bom(local_bom, version): @@ -117,20 +133,19 @@ def validate_published_poms(parent_path, bom_path, version, local_bom): def validate_poms(root, version): - if version != "4.1.0": - raise GateError("This release workflow is limited to 4.1.0") + validate_version(version) parent = properties(root / "pom.xml") bom = properties(root / "qqq-all-bom/pom.xml") if parent.get("revision") != version or bom.get("qqq.version") != version: - raise GateError("Parent revision and QQQ BOM pin must both equal 4.1.0") + raise GateError(f"Parent revision and QQQ BOM pin must both equal {version}") for name in QBITS: key = f"qbit-{name}.version" value = bom.get(key, "") - if not value or "SNAPSHOT" in value.upper(): + if not immutable_version(value): raise GateError(f"{key} must be a published non-snapshot version") for key, value in bom.items(): - if key.endswith(".version") and "SNAPSHOT" in value.upper(): - raise GateError(f"{key} still points to a snapshot") + if key.endswith(".version") and not immutable_version(value): + raise GateError(f"{key} must be an immutable release version") validate_managed_bom(root / "qqq-all-bom/pom.xml", version) @@ -156,6 +171,94 @@ def validate_smoke(run, jobs, sha): raise GateError("Smoke run must include successful core and full jobs") +def validate_upstream_release(release, version): + if (not immutable_version(version) or release.get("tag_name") != "v" + version + or release.get("draft") is not False + or release.get("prerelease") is not ("-RC." in version)): + raise GateError(f"Upstream release must be public v{version} with matching prerelease state") + + +def validate_tag(tag, version, sha, tag_sha): + validate_version(version) + if not all(re.fullmatch(r"[0-9a-f]{40}", value) for value in (sha, tag_sha)): + raise GateError("Tag and commit IDs must be full Git SHAs") + if (tag.get("sha") != tag_sha or tag.get("tag") != version + or tag.get("object", {}).get("type") != "commit" + or tag.get("object", {}).get("sha") != sha + or tag.get("verification", {}).get("verified") is not True + or tag.get("verification", {}).get("reason") != "valid"): + raise GateError("Release tag must be signed, verified, annotated and point to the exact release commit") + + +def download_public_inputs(bom, version, destination, central="https://repo.maven.apache.org/maven2"): + """Fetch the declared first-party inputs and their parents without credentials.""" + validate_version(version) + destination.mkdir(parents=True, exist_ok=True) + receipts = [] + visited = set() + + def fetch(group, artifact, pin, kind): + coordinate = (group, artifact, pin, kind) + if coordinate in visited: + return + if (not re.fullmatch(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*", group) + or not re.fullmatch(r"[A-Za-z0-9_-]+", artifact) or not immutable_version(pin)): + raise GateError(f"Invalid or mutable public coordinate: {coordinate}") + visited.add(coordinate) + relative = f"{group.replace('.', '/')}/{artifact}/{pin}/{artifact}-{pin}.{kind}" + url = central + "/" + relative + try: + with urllib.request.urlopen(url, timeout=30) as response: + data = response.read() + except (urllib.error.URLError, OSError) as error: + if isinstance(error, urllib.error.HTTPError): + error.close() + raise GateError(f"Public input unavailable: {relative}") from error + if not data: + raise GateError(f"Empty public input: {relative}") + path = destination / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + if kind == "pom": + if coordinates(path) != (group, artifact, pin): + raise GateError(f"Public POM coordinates differ: {relative}") + root = ElementTree.fromstring(data) + ns = xml_namespace(root) + parent = root.find(f"{ns}parent") + if parent is not None: + parent_group = parent.findtext(f"{ns}groupId", "") + parent_pin = parent.findtext(f"{ns}version", "") + if not immutable_version(parent_pin): + raise GateError(f"Mutable parent in {relative}") + if parent_group.startswith("com.kingsrook"): + fetch(parent_group, parent.findtext(f"{ns}artifactId", ""), parent_pin, "pom") + if artifact == "qbit-build-parent": + imports = root.findall(f"{ns}dependencyManagement/{ns}dependencies/{ns}dependency") + props = root.find(f"{ns}properties") + values = {child.tag.removeprefix(ns): child.text for child in props} if props is not None else {} + qqq_imports = [dep for dep in imports if dep.findtext(f"{ns}groupId") == "com.kingsrook.qqq" + and dep.findtext(f"{ns}artifactId") == "qqq-bom-pom"] + if len(qqq_imports) != 1: + raise GateError("Public qbit-build-parent must import the candidate QQQ BOM") + dep = qqq_imports[0] + actual = dep.findtext(f"{ns}version", "") + actual = values.get(actual[2:-1], "") if actual.startswith("${") else actual + if (actual != version or dep.findtext(f"{ns}type") != "pom" + or dep.findtext(f"{ns}scope") != "import"): + raise GateError("Public qbit-build-parent QQQ import differs from the candidate") + receipts.append({"url": url, "sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)}) + + for group, artifact, pin, kind, _, classifier in validate_managed_bom(bom, version): + if group.startswith("com.kingsrook"): + if classifier or kind not in ("jar", "pom"): + raise GateError(f"Unsupported first-party release input: {artifact}") + fetch(group, artifact, pin, "pom") + if kind == "jar": + fetch(group, artifact, pin, "jar") + (destination / "receipts.json").write_text(json.dumps(receipts, indent=2) + "\n") + return receipts + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, required=True) @@ -165,13 +268,23 @@ def main(): parser.add_argument("--smoke-jobs", type=Path, required=True) parser.add_argument("--published-parent", type=Path, required=True) parser.add_argument("--published-bom", type=Path, required=True) + parser.add_argument("--upstream-release", type=Path, required=True) + parser.add_argument("--next-release", type=Path, required=True) + parser.add_argument("--tag-object", type=Path, required=True) + parser.add_argument("--tag-sha", required=True) + parser.add_argument("--public-inputs", type=Path, required=True) args = parser.parse_args() try: validate_poms(args.root, args.version) + validate_upstream_release(json.loads(args.upstream_release.read_text()), args.version) + validate_upstream_release(json.loads(args.next_release.read_text()), + properties(args.root / "qqq-all-bom/pom.xml")["qqq-frontend-next.version"]) + validate_tag(json.loads(args.tag_object.read_text()), args.version, args.sha, args.tag_sha) validate_smoke(json.loads(args.smoke_run.read_text()), json.loads(args.smoke_jobs.read_text()), args.sha) validate_published_poms(args.published_parent, args.published_bom, args.version, args.root / "qqq-all-bom/pom.xml") + download_public_inputs(args.root / "qqq-all-bom/pom.xml", args.version, args.public_inputs) except (GateError, OSError, ElementTree.ParseError, json.JSONDecodeError) as error: print(f"Release gate failed: {error}", file=sys.stderr) return 1 diff --git a/scripts/test_release_bundle.py b/scripts/test_release_bundle.py new file mode 100644 index 0000000..55486f9 --- /dev/null +++ b/scripts/test_release_bundle.py @@ -0,0 +1,76 @@ +"""Exercise the delivered archive with Compose, without running any containers.""" + +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +IMAGE = "ghcr.io/qrun-io/qqq-all@sha256:" + "a" * 64 + + +class ReleaseBundleTest(unittest.TestCase): + def test_bundle_uses_digest_and_keeps_full_profile_infra_without_app_build(self): + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run(["python3", str(ROOT / "scripts/release_bundle.py"), + "--root", str(ROOT), "--version", "4.1.0-RC.1", + "--sha", "b" * 40, "--image", IMAGE, "--output", directory], + capture_output=True, text=True) + self.assertEqual(0, result.returncode, result.stderr) + archive = Path(directory) / "qqq-all-4.1.0-RC.1.tar.gz" + extracted = Path(directory) / "extracted" + with tarfile.open(archive) as bundle: + names = bundle.getnames() + self.assertIn("qqq-all-4.1.0-RC.1/infra/minio/Dockerfile", names) + self.assertIn("qqq-all-4.1.0-RC.1/infra/keycloak/realm.json", names) + self.assertIn("qqq-all-4.1.0-RC.1/.env.example", names) + self.assertNotIn("qqq-all-4.1.0-RC.1/.env", names) + self.assertNotIn("qqq-all-4.1.0-RC.1/Dockerfile", names) + bundle.extractall(extracted, filter="data") + root = extracted / "qqq-all-4.1.0-RC.1" + bundled = json.loads((root / "compose.yaml").read_text()) + self.assertEqual("/home/${SFTP_USER:-qqq}/upload", + bundled["services"]["sftp"]["volumes"][0]["target"]) + environment = {key: value for key, value in os.environ.items() if not key.startswith("COMPOSE_")} + result = subprocess.run(["docker", "compose", "--env-file", ".env.example", "--profile", "*", + "-f", "compose.yaml", "config", "--format", "json"], + cwd=root, env=environment, capture_output=True, text=True) + self.assertEqual(0, result.returncode, result.stderr) + config = json.loads(result.stdout) + original_result = subprocess.run(["docker", "compose", "--env-file", ".env.example", "--profile", "*", + "-f", "compose.yaml", "config", "--format", "json"], + cwd=ROOT, env=environment, capture_output=True, text=True, check=True) + original = json.loads(original_result.stdout.replace(str(ROOT.resolve()), str(root.resolve()))) + for name, service in original["services"].items(): + if name in ("app-core", "app-full"): + service.pop("build") + service.update(image=IMAGE, pull_policy="always") + if name == "minio": + service["image"] = "qqq-all-minio:4.1.0-RC.1" + self.assertEqual(service, config["services"][name], name) + for service in ("app-core", "app-full"): + self.assertEqual(IMAGE, config["services"][service]["image"]) + self.assertNotIn("build", config["services"][service]) + self.assertEqual((root / "infra/minio").resolve(), Path(config["services"]["minio"]["build"]["context"]).resolve()) + for service in config["services"].values(): + for volume in service.get("volumes", []): + if volume["type"] == "bind": + self.assertTrue(Path(volume["source"]).exists(), volume) + manifest = json.loads((root / "release.json").read_text()) + self.assertEqual("4.1.0-RC.1", manifest["version"]) + self.assertEqual("b" * 40, manifest["source_sha"]) + self.assertEqual(IMAGE, manifest["image"]) + + def test_bundle_rejects_mutable_image_and_unapproved_version(self): + with tempfile.TemporaryDirectory() as directory: + for version, image in (("4.1.0-RC.2", IMAGE), ("4.1.0-RC.1", "ghcr.io/qrun-io/qqq-all:develop"), + ("4.1.0", "ghcr.io/other/qqq-all@sha256:" + "a" * 64)): + result = subprocess.run(["python3", str(ROOT / "scripts/release_bundle.py"), + "--root", str(ROOT), "--version", version, "--sha", "b" * 40, + "--image", image, "--output", directory], capture_output=True, text=True) + self.assertNotEqual(0, result.returncode) + self.assertEqual([], list(Path(directory).iterdir())) diff --git a/scripts/test_release_gate.py b/scripts/test_release_gate.py index 105a949..0c4bb59 100644 --- a/scripts/test_release_gate.py +++ b/scripts/test_release_gate.py @@ -26,6 +26,10 @@ def management(qqq_version="${qqq.version}", qbit_version=None): f"qbit-{name}{qbit_version(name)}" for name in QBITS ) + dependencies += ( + "com.kingsrook.qqqqqq-frontend-next" + "1.0.0-RC.8" + ) return f"{dependencies}" @@ -44,6 +48,7 @@ class ReleaseGateTest(unittest.TestCase): (self.root / "qqq-all-bom/pom.xml").write_text( "" f"{qqq_version}" + "1.0.0-RC.8" + "".join(f"{qbit_version}" for name in QBITS) + "" + management() + "" ) @@ -59,6 +64,7 @@ class ReleaseGateTest(unittest.TestCase): "com.kingsrook.qqq" "qqq-all-parent4.1.0" "qqq-all-bom4.1.0" + "1.0.0-RC.8" + "".join(f"1.2.3" for name in QBITS) + "" + managed + "") return parent, bom @@ -97,6 +103,8 @@ class ReleaseGateTest(unittest.TestCase): ({**run, "head_sha": "b" * 40}, jobs), ({**run, "conclusion": "failure"}, jobs), ({**run, "path": ".github/workflows/other.yml"}, jobs), + ({**run, "event": "workflow_dispatch"}, jobs), + ({**run, "head_branch": "main"}, jobs), (run, {"jobs": [jobs["jobs"][0]]}), (run, {"jobs": [jobs["jobs"][0], {"name": "full", "conclusion": "skipped"}]}), ): diff --git a/scripts/test_release_rc.py b/scripts/test_release_rc.py new file mode 100644 index 0000000..aeac35e --- /dev/null +++ b/scripts/test_release_rc.py @@ -0,0 +1,151 @@ +"""RC publication must retain the GA boundary and fail before any upload.""" + +import functools +import http.server +import tempfile +import threading +import unittest +from pathlib import Path + +import release_gate +from release_gate import GateError +import test_release_gate + + +RC = "4.1.0-RC.1" +TAG_SHA = "b" * 40 +RELEASE_SHA = "a" * 40 + + +class QuietHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, format, *args): + pass + + +class ReleaseCandidateTest(unittest.TestCase): + def setUp(self): + self.fixture = test_release_gate.ReleaseGateTest() + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + self.root = self.fixture.root + + def test_exact_rc_and_ga_are_allowed_but_other_candidates_are_not(self): + for version in ("4.1.0", RC): + self.fixture.write_poms(version, version, "1.2.3-RC.1") + release_gate.validate_poms(self.root, version) + for version in ("4.1.0-RC.2", "4.1.0-SNAPSHOT", "4.1.1", "LATEST", ""): + with self.subTest(version=version), self.assertRaises(GateError): + self.fixture.write_poms(version, version, "1.2.3") + release_gate.validate_poms(self.root, version) + + def test_rc_rejects_wrong_framework_pin_and_mutable_dependencies(self): + for pin in ("4.1.0", "4.1.0-RC.2", "4.1.0-SNAPSHOT"): + self.fixture.write_poms(RC, pin, "1.2.3") + with self.assertRaises(GateError): + release_gate.validate_poms(self.root, RC) + for pin in ("LATEST", "RELEASE", "[1.0,2.0)", "1.2.3-SNAPSHOT", "${unknown}"): + self.fixture.write_poms(RC, RC, pin) + with self.subTest(pin=pin), self.assertRaises(GateError): + release_gate.validate_poms(self.root, RC) + self.fixture.write_poms(RC, RC, "1.2.3") + bom = self.root / "qqq-all-bom/pom.xml" + bom.write_text(bom.read_text().replace("1.0.0-RC.8", "LATEST")) + with self.assertRaises(GateError): + release_gate.validate_poms(self.root, RC) + + def test_public_rc_distribution_poms_must_match_source_including_next(self): + self.fixture.write_poms(RC, RC, "1.2.3") + parent, bom = self.fixture.published_poms(resolved=True) + for path in (parent, bom): + path.write_text(path.read_text().replace("4.1.0", RC)) + release_gate.validate_published_poms(parent, bom, RC, self.root / "qqq-all-bom/pom.xml") + original = bom.read_text() + for changed in (original.replace("1.0.0-RC.8", "1.0.0-RC.9"), + original.replace(RC, "4.1.0-RC.2"), + original.replace("1.2.3", "[1,2)")): + bom.write_text(changed) + with self.assertRaises(GateError): + release_gate.validate_published_poms(parent, bom, RC, self.root / "qqq-all-bom/pom.xml") + + def test_upstream_rc_requires_exact_public_prerelease_and_ga_requires_final(self): + for version, prerelease in ((RC, True), ("4.1.0", False)): + release = {"tag_name": "v" + version, "draft": False, "prerelease": prerelease} + release_gate.validate_upstream_release(release, version) + for changed in ({**release, "draft": True}, {**release, "prerelease": not prerelease}, + {**release, "tag_name": "v4.1.0-RC.2"}, {}): + with self.subTest(changed=changed), self.assertRaises(GateError): + release_gate.validate_upstream_release(changed, version) + + def test_tag_requires_verified_signature_exact_name_object_and_commit(self): + tag = {"sha": TAG_SHA, "tag": RC, "object": {"type": "commit", "sha": RELEASE_SHA}, + "verification": {"verified": True, "reason": "valid"}} + release_gate.validate_tag(tag, RC, RELEASE_SHA, TAG_SHA) + mutations = [ + {**tag, "tag": "4.1.0"}, {**tag, "sha": "c" * 40}, + {**tag, "object": {"type": "commit", "sha": "c" * 40}}, + {**tag, "object": {"type": "tag", "sha": RELEASE_SHA}}, + {**tag, "verification": {"verified": False, "reason": "unsigned"}}, + {**tag, "verification": {"verified": True, "reason": "unknown_key"}}, {}, + ] + for changed in mutations: + with self.subTest(changed=changed), self.assertRaises(GateError): + release_gate.validate_tag(changed, RC, RELEASE_SHA, TAG_SHA) + + def test_public_download_fails_on_missing_or_wrong_coordinates_and_parent(self): + self.fixture.write_poms(RC, RC, "1.2.3") + with tempfile.TemporaryDirectory() as directory: + central = Path(directory) + dependencies = [("com.kingsrook.qqq", "qqq-bom-pom", RC, "pom"), + ("com.kingsrook.qqq", "qqq-frontend-next", "1.0.0-RC.8", "jar")] + dependencies += [("com.kingsrook.qbits", "qbit-" + name, "1.2.3", "jar") + for name in release_gate.QBITS] + for group, artifact, version, kind in dependencies: + folder = central / group.replace(".", "/") / artifact / version + folder.mkdir(parents=True) + stem = folder / (artifact + "-" + version) + Path(str(stem) + ".pom").write_text( + f"{group}{artifact}" + f"{version}") + if kind == "jar": + Path(str(stem) + ".jar").write_bytes(b"published fixture jar") + handler = functools.partial(QuietHandler, directory=directory) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + base_url = f"http://127.0.0.1:{server.server_port}" + dest = self.root / "public" + bom = self.root / "qqq-all-bom/pom.xml" + receipts = release_gate.download_public_inputs(bom, RC, dest, base_url) + self.assertEqual(19, len(receipts)) + self.assertTrue(all(len(row["sha256"]) == 64 for row in receipts)) + public_pom = central / "com/kingsrook/qqq/qqq-frontend-next/1.0.0-RC.8/qqq-frontend-next-1.0.0-RC.8.pom" + original = public_pom.read_text() + public_pom.write_text(original.replace("1.0.0-RC.8", "1.0.0-RC.9")) + with self.assertRaises(GateError): + release_gate.download_public_inputs(bom, RC, dest, base_url) + public_pom.write_text(original.replace("", "com.kingsrook" + "qbit-build-parentLATEST")) + with self.assertRaises(GateError): + release_gate.download_public_inputs(bom, RC, dest, base_url) + public_pom.write_text(original.replace("", "com.kingsrook" + "qbit-build-parent2.1.0-RC.1")) + parent = central / "com/kingsrook/qbit-build-parent/2.1.0-RC.1/qbit-build-parent-2.1.0-RC.1.pom" + parent.parent.mkdir(parents=True) + parent.write_text("com.kingsrookqbit-build-parent" + "2.1.0-RC.1" + "com.kingsrook.qqqqqq-bom-pom" + "4.0.0pomimport" + "") + with self.assertRaises(GateError): + release_gate.download_public_inputs(bom, RC, dest, base_url) + parent.write_text(parent.read_text().replace("4.0.0", RC)) + self.assertEqual(20, len(release_gate.download_public_inputs(bom, RC, dest, base_url))) + public_pom.unlink() + with self.assertRaises(GateError): + release_gate.download_public_inputs(bom, RC, dest, base_url) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_release_workflow.py b/scripts/test_release_workflow.py new file mode 100644 index 0000000..a2481fd --- /dev/null +++ b/scripts/test_release_workflow.py @@ -0,0 +1,51 @@ +"""Run the workflow's publication shell with a recording CLI, never GitHub.""" + +import json +import os +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent + + +def workflow_shell(name): + text = (ROOT / ".github/workflows/release.yml").read_text() + step = text.split(" - name: " + name + "\n", 1)[1].split(" - ", 1)[0] + return textwrap.dedent(step.split(" run: |\n", 1)[1]) + + +class ReleaseWorkflowTest(unittest.TestCase): + def test_rc_is_prerelease_never_latest_and_ga_keeps_existing_release_behavior(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + gh = root / "gh" + gh.write_text("#!/usr/bin/env python3\nimport json, os, sys\n" + "with open(os.environ['CALLS'], 'w') as f: json.dump(sys.argv[1:], f)\n") + gh.chmod(0o755) + calls = root / "calls.json" + for version in ("4.1.0", "4.1.0-RC.1", "4.1.0-RC.2"): + calls.unlink(missing_ok=True) + environment = {**os.environ, "PATH": directory + os.pathsep + os.environ["PATH"], + "CALLS": str(calls), "VERSION": version, "TAG": version, + "RUNNER_TEMP": directory, "GITHUB_REPOSITORY": "QRun-IO/qqq-all"} + result = subprocess.run(["bash", "-c", workflow_shell("Publish GitHub release with Compose bundle")], + env=environment, capture_output=True, text=True) + if version == "4.1.0-RC.2": + self.assertNotEqual(0, result.returncode) + self.assertFalse(calls.exists()) + continue + self.assertEqual(0, result.returncode, result.stderr) + args = json.loads(calls.read_text()) + self.assertEqual(["release", "create", version], args[:3]) + self.assertIn(str(root / ("qqq-all-" + version + ".tar.gz")), args) + self.assertIn("--verify-tag", args) + if version == "4.1.0-RC.1": + self.assertIn("--prerelease", args) + self.assertIn("--latest=false", args) + else: + self.assertNotIn("--prerelease", args) + self.assertNotIn("--latest=false", args)