Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c80ce04963
|
||
|
|
2956e3ffce | ||
|
|
0afe2a231f | ||
|
|
65265e48a4 | ||
|
|
49399deb3d | ||
|
|
9fe4257490 | ||
|
|
0965a45653 | ||
|
|
bfd41c8e0d | ||
|
|
9c30a30754 | ||
|
|
ee8953bd93 | ||
|
|
5bc3167286 | ||
|
|
1a3368e119 | ||
|
|
1ddba883c9 | ||
|
|
da8044d6f2 | ||
|
|
67ba48da1f | ||
|
|
5196609974 | ||
|
|
49e09a6776 | ||
|
|
748016f105 | ||
|
|
6514aaa688 | ||
|
|
3ac47adc21 |
No files matched your search
@@ -61,6 +61,8 @@ jobs:
|
||||
python3 -m unittest discover -s tests -p test_generate_qctl_workflow.py
|
||||
python3 tests/test_rc_finalize_sweep.py
|
||||
python3 tests/test_rc_guard.py
|
||||
python3 -m unittest discover -s tests -p test_rc_timeout_recovery.py
|
||||
python3 -m unittest discover -s tests -p test_rc_ref_lookup.py
|
||||
python3 -m unittest discover -s tests -p test_generate_rc_workflow.py
|
||||
python3 -m unittest discover -s tests -p test_audit_estate.py
|
||||
python3 tests/test_all_profiles.py
|
||||
|
||||
@@ -19,6 +19,14 @@ on:
|
||||
description: Finalize an existing cut from repository_dispatch or an explicit retry
|
||||
type: boolean
|
||||
default: false
|
||||
recover:
|
||||
description: Verify an already-published upload after a Circle deployment timeout
|
||||
type: boolean
|
||||
default: false
|
||||
circle_build:
|
||||
description: Original Circle publisher build number (recovery only)
|
||||
type: string
|
||||
default: ''
|
||||
merge_sha:
|
||||
type: string
|
||||
default: ''
|
||||
@@ -32,6 +40,10 @@ on:
|
||||
type: string
|
||||
default: ''
|
||||
secrets:
|
||||
CENTRAL_USERNAME:
|
||||
required: false
|
||||
CENTRAL_PASSWORD:
|
||||
required: false
|
||||
QRUN_TAG_SIGNING_KEY:
|
||||
description: SSH private key that signs release tags
|
||||
required: false
|
||||
@@ -48,7 +60,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
cut-rc:
|
||||
if: ${{ !inputs.finalize }}
|
||||
if: ${{ !inputs.finalize && !inputs.recover }}
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
@@ -123,7 +135,7 @@ jobs:
|
||||
echo "merge_sha=$merge_sha"
|
||||
echo "merged_at=$merged_at"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "Pushed $branch; CircleCI publish_release_candidate cuts the RC." >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Pushed $branch; the CircleCI publisher cuts the RC." >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Release line: $branch. Waiting without a runner for qbit-rc-published." >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Persist trusted release intent
|
||||
if: steps.merge.outputs.merge_sha != ''
|
||||
@@ -135,7 +147,7 @@ jobs:
|
||||
retention-days: 90
|
||||
|
||||
tag-and-release:
|
||||
if: inputs.finalize
|
||||
if: inputs.finalize || inputs.recover
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
@@ -147,6 +159,12 @@ jobs:
|
||||
MERGE_SHA: ${{ inputs.merge_sha }}
|
||||
BUMP_SHA: ${{ inputs.bump_sha }}
|
||||
VERSION: ${{ inputs.version }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
RECOVER: ${{ inputs.recover }}
|
||||
CIRCLE_BUILD: ${{ inputs.circle_build }}
|
||||
RECOVERY_EVENT: ${{ github.event_name }}
|
||||
CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }}
|
||||
CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }}
|
||||
SIGNING_KEY: ${{ secrets.QRUN_TAG_SIGNING_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -156,6 +174,22 @@ jobs:
|
||||
fi
|
||||
workdir=$(mktemp -d)
|
||||
trap 'rm -rf "$workdir"' EXIT
|
||||
# BEGIN RC REF LOOKUP
|
||||
read_ref() {
|
||||
local body sha
|
||||
if body=$(gh api "repos/$REPO/git/ref/$1" 2>"$workdir/ref-error"); then
|
||||
sha=$(jq -er '.object.sha' <<< "$body") || return 1
|
||||
[[ "$sha" =~ ^[0-9a-f]{40}$ ]] || return 1
|
||||
printf '%s\n' "$sha"
|
||||
elif jq -se 'length == 1 and .[0].status == "404"' >/dev/null 2>&1 <<< "$body"; then
|
||||
# gh prints HTTP error JSON on stdout even when --jq was requested.
|
||||
return 0
|
||||
else
|
||||
cat "$workdir/ref-error" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
# END RC REF LOOKUP
|
||||
# The evaluator is inline so a pinned reusable workflow cannot fetch mutable code.
|
||||
cat > "$workdir/rc_guard.py" <<'PY'
|
||||
# BEGIN RC GUARD
|
||||
@@ -164,12 +198,12 @@ jobs:
|
||||
class ReleaseFailure(ValueError):
|
||||
pass
|
||||
|
||||
def evaluate(merge_sha, status, commit):
|
||||
def evaluate(merge_sha, status, commit, recovered_version=""):
|
||||
if status.get("sha") != merge_sha:
|
||||
raise ReleaseFailure("Status response does not identify the exact release merge")
|
||||
# Circle's workflow Check Run is stable even when job statuses receive -1/-2 suffixes.
|
||||
publishers = [x for x in status.get("check_runs", [])
|
||||
if x.get("name") == "publish_release_candidate" and x.get("app", {}).get("id") == 18001]
|
||||
if x.get("name") in ("publish_release_candidate", "verify") and x.get("app", {}).get("id") == 18001]
|
||||
if not publishers:
|
||||
return ""
|
||||
publisher = max(publishers, key=lambda x: x["id"])
|
||||
@@ -177,7 +211,7 @@ jobs:
|
||||
raise ReleaseFailure("Publisher Check Run identifies a different source")
|
||||
if publisher.get("status") != "completed":
|
||||
return ""
|
||||
if publisher.get("conclusion") != "success":
|
||||
if publisher.get("conclusion") != "success" and not recovered_version:
|
||||
raise ReleaseFailure("CircleCI failed for release merge: " + str(publisher.get("conclusion")))
|
||||
message = commit.get("commit", {}).get("message", "").split("\n")[0]
|
||||
match = re.fullmatch(r"Bump version to (\d+\.\d+\.\d+-RC\.\d+) \[skip ci\]", message)
|
||||
@@ -185,6 +219,8 @@ jobs:
|
||||
return ""
|
||||
if [x.get("sha") for x in commit.get("parents", [])] != [merge_sha]:
|
||||
raise ReleaseFailure("RC bump is not the direct child of the published merge")
|
||||
if recovered_version and recovered_version != match.group(1):
|
||||
raise ReleaseFailure("Recovery receipt and bump version differ")
|
||||
return match.group(1)
|
||||
|
||||
def validate_intent(run, intent, repo, merge, branch, run_id):
|
||||
@@ -223,24 +259,168 @@ jobs:
|
||||
raise ReleaseFailure("Central POM does not match the expected artifact/version")
|
||||
if parsedate_to_datetime(modified).timestamp() <= datetime.fromisoformat(merged.replace("Z", "+00:00")).timestamp():
|
||||
raise ReleaseFailure("Central artifact predates the exact release merge")
|
||||
def validate_timeout(job, publisher, repo, merge, branch, build, merged_at):
|
||||
from datetime import datetime
|
||||
if (job.get('username') + '/' + job.get('reponame'), job.get('vcs_revision'),
|
||||
job.get('branch'), str(job.get('build_num'))) != (repo, merge, branch, build):
|
||||
raise ReleaseFailure('Timeout job does not identify the exact cut')
|
||||
workflow = job.get('workflows', {})
|
||||
if workflow.get('workflow_name') not in ('publish_release_candidate', 'verify') or workflow.get('job_name') != 'qqq-orb/mvn_publish':
|
||||
raise ReleaseFailure('Unexpected Circle publication workflow/job')
|
||||
if (publisher.get('head_sha') != merge or publisher.get('app', {}).get('id') != 18001
|
||||
or publisher.get('name') != workflow.get('workflow_name')
|
||||
or publisher.get('status') != 'completed' or publisher.get('conclusion') != 'failure'):
|
||||
raise ReleaseFailure('Recovery requires the failed exact Circle publisher')
|
||||
from urllib.parse import urlsplit
|
||||
link = urlsplit(publisher.get('details_url', ''))
|
||||
if link.scheme != 'https' or link.netloc != 'app.circleci.com' or link.path != '/workflow/' + workflow.get('workflow_id', ''):
|
||||
raise ReleaseFailure('Circle check does not identify the timeout workflow')
|
||||
if job.get('status') != 'failed' or job.get('outcome') != 'failed':
|
||||
raise ReleaseFailure('Recovery requires a terminal failed job')
|
||||
actions = [action for step in job.get('steps', []) for action in step.get('actions', [])]
|
||||
if not actions or actions[-1].get('name') != 'Publish to Sonatype Central (releases and SNAPSHOTs)':
|
||||
raise ReleaseFailure('Deployment must be the last executed action')
|
||||
deploy = actions[-1]
|
||||
if deploy.get('status') != 'timedout' or deploy.get('timedout') is not True:
|
||||
raise ReleaseFailure('Only a deployment timeout can be recovered')
|
||||
if any(action.get('status') != 'success' for action in actions[:-1]):
|
||||
raise ReleaseFailure('Every pre-publication action must have passed')
|
||||
if not any(action.get('name') == 'Run Maven Verify' for action in actions[:-1]):
|
||||
raise ReleaseFailure('Successful Maven verification is missing')
|
||||
if datetime.fromisoformat(job['start_time'].replace('Z', '+00:00')) < datetime.fromisoformat(merged_at.replace('Z', '+00:00')):
|
||||
raise ReleaseFailure('Publisher predates the trusted cut')
|
||||
if any(type(deploy.get(key)) is not int or deploy[key] < 0 for key in ('step', 'index')):
|
||||
raise ReleaseFailure('Invalid deployment log coordinates')
|
||||
return deploy['step'], deploy['index']
|
||||
|
||||
|
||||
def upload_receipt(log, artifact, version):
|
||||
# Only the successful-upload receipt and actual Central staging paths count.
|
||||
deployments = re.findall(r'Uploaded bundle successfully,[^\n]*deploymentId: ([0-9a-f-]{36})\.', log)
|
||||
if len(deployments) != 1 or not re.fullmatch(r'[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', deployments[0]):
|
||||
raise ReleaseFailure('Expected exactly one successful Central upload receipt')
|
||||
paths = re.findall(r'/target/central-staging/(com/kingsrook/qbits)/(qbit-[a-z0-9-]+)/(\d+\.\d+\.\d+-RC\.\d+)/', log)
|
||||
coordinates = {(group.replace('/', '.'), name, revision) for group, name, revision in paths}
|
||||
if not coordinates or ('com.kingsrook.qbits', artifact, version) not in coordinates:
|
||||
raise ReleaseFailure('Receipt does not include the expected root artifact')
|
||||
if any(revision != version for _, _, revision in coordinates):
|
||||
raise ReleaseFailure('Mixed candidate versions in the uploaded bundle')
|
||||
return deployments[0], sorted('pkg:maven/' + group + '/' + name + '@' + revision for group, name, revision in coordinates)
|
||||
|
||||
|
||||
def validate_deployment(deployment, deployment_id, purls):
|
||||
if deployment.get('deploymentId') != deployment_id or deployment.get('deploymentState') != 'PUBLISHED':
|
||||
raise ReleaseFailure('The original Central deployment is not PUBLISHED')
|
||||
if sorted(deployment.get('purls', [])) != purls:
|
||||
raise ReleaseFailure('Central deployment coordinates differ from the original upload')
|
||||
|
||||
|
||||
def validate_recovery_bump(commit, source, actual, version, merge):
|
||||
if [p.get('sha') for p in commit.get('parents', [])] != [merge]:
|
||||
raise ReleaseFailure('Recovery bump must be a direct child of the cut')
|
||||
files = commit.get('files', [])
|
||||
if len(files) != 1 or files[0].get('filename') != 'pom.xml' or files[0].get('status') != 'modified':
|
||||
raise ReleaseFailure('Recovery bump may change only the root POM')
|
||||
revisions = re.findall(r'<revision>([^<]+)</revision>', source)
|
||||
if len(revisions) != 1 or not re.fullmatch(re.escape(version.split('-RC.')[0]) + r'-(SNAPSHOT|RC\.\d+)', revisions[0]):
|
||||
raise ReleaseFailure('Unexpected source POM revision')
|
||||
expected = source.replace('<revision>' + revisions[0] + '</revision>', '<revision>' + version + '</revision>')
|
||||
if actual != expected:
|
||||
raise ReleaseFailure('Recovery bump changed content beyond the verified version')
|
||||
|
||||
def verify_recovery(directory, repo, merge, branch, build, version):
|
||||
import base64, json, os, pathlib, urllib.request, xml.etree.ElementTree as ET
|
||||
if not re.fullmatch(r'QRun-IO/qbit-[a-z0-9-]+', repo) or not re.fullmatch(r'[1-9][0-9]*', build):
|
||||
raise ReleaseFailure('Invalid recovery repository or Circle build')
|
||||
directory = pathlib.Path(directory)
|
||||
status = json.loads((directory / 'status.json').read_text())
|
||||
intent = json.loads((directory / 'intent.json').read_text())
|
||||
publishers = [p for p in status['check_runs'] if p.get('name') in ('publish_release_candidate', 'verify') and p.get('app', {}).get('id') == 18001]
|
||||
if status.get('sha') != merge or not publishers:
|
||||
raise ReleaseFailure('Exact publisher evidence missing')
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
raise ReleaseFailure('Unexpected evidence redirect')
|
||||
opener = urllib.request.build_opener(NoRedirect())
|
||||
def request(url, token=None, post=False):
|
||||
headers = {'Accept': 'application/json'}
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
req = urllib.request.Request(url, headers=headers, data=b'' if post else None)
|
||||
with opener.open(req, timeout=30) as response:
|
||||
data = response.read(8_000_001)
|
||||
if len(data) > 8_000_000:
|
||||
raise ReleaseFailure('Oversized recovery evidence')
|
||||
return data, response.headers
|
||||
circle = 'https://circleci.com/api/v1.1/project/github/' + repo + '/' + build
|
||||
job = json.loads(request(circle)[0])
|
||||
step, index = validate_timeout(job, max(publishers, key=lambda p: p['id']), repo, merge, branch, build, intent['merged_at'])
|
||||
log = request(circle + '/output/' + str(step) + '/' + str(index) + '?file=true')[0].decode()
|
||||
# file=true can return either a raw log or JSON-encoded output.
|
||||
try:
|
||||
value = json.loads(log)
|
||||
except json.JSONDecodeError:
|
||||
value = log
|
||||
log = value if isinstance(value, str) else ''.join(part['message'] for part in value)
|
||||
token = os.environ.get('GH_TOKEN', '')
|
||||
source = json.loads(request('https://api.github.com/repos/' + repo + '/contents/pom.xml?ref=' + merge, token)[0])
|
||||
source_pom = base64.b64decode(source['content']).decode()
|
||||
pom = ET.fromstring(source_pom)
|
||||
ns = {'m': 'http://maven.apache.org/POM/4.0.0'}
|
||||
artifact = pom.findtext('m:artifactId', namespaces=ns)
|
||||
deployment_id, purls = upload_receipt(log, artifact, version)
|
||||
username, password = os.environ.get('CENTRAL_USERNAME'), os.environ.get('CENTRAL_PASSWORD')
|
||||
if not username or not password:
|
||||
raise ReleaseFailure('Existing Central publisher credentials are required for receipt verification')
|
||||
authorization = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
deployment = json.loads(request('https://central.sonatype.com/api/v1/publisher/status?id=' + deployment_id, authorization, post=True)[0])
|
||||
validate_deployment(deployment, deployment_id, purls)
|
||||
for purl in purls:
|
||||
coordinate, revision = purl.removeprefix('pkg:maven/').split('@')
|
||||
group, name = coordinate.split('/')
|
||||
url = 'https://repo1.maven.org/maven2/' + group.replace('.', '/') + '/' + name + '/' + revision + '/' + name + '-' + revision + '.pom'
|
||||
xml, headers = request(url)
|
||||
validate_central(revision, name, xml, headers.get('Last-Modified', ''), intent['merged_at'])
|
||||
(directory / 'source-pom.xml').write_text(source_pom)
|
||||
# This receipt is written only after all remote evidence has been verified.
|
||||
(directory / 'recovery.json').write_text(json.dumps({'merge_sha': merge, 'version': version,
|
||||
'circle_build': build, 'deployment_id': deployment_id, 'purls': purls}))
|
||||
|
||||
# END RC GUARD
|
||||
if __name__ == "__main__":
|
||||
import json, sys
|
||||
try:
|
||||
if sys.argv[1] == "lock":
|
||||
if sys.argv[1] == "recover":
|
||||
verify_recovery(*sys.argv[2:])
|
||||
elif sys.argv[1] == "lock":
|
||||
validate_lock(sys.argv[2], sys.argv[3])
|
||||
elif sys.argv[1] == "tag":
|
||||
print(tag_action(sys.argv[2], sys.argv[3], sys.argv[4] == "true", sys.argv[5]))
|
||||
elif sys.argv[1] == "central":
|
||||
validate_central(*sys.argv[2:4], open(sys.argv[4]).read(), *sys.argv[5:7])
|
||||
else:
|
||||
print(evaluate(sys.argv[1], json.load(open(sys.argv[2])), json.load(open(sys.argv[3]))))
|
||||
recovered = ""
|
||||
if len(sys.argv) == 5:
|
||||
receipt = json.load(open(sys.argv[4]))
|
||||
if receipt["merge_sha"] != sys.argv[1]:
|
||||
raise ReleaseFailure("Recovery receipt source mismatch")
|
||||
recovered = receipt["version"]
|
||||
import pathlib
|
||||
directory = pathlib.Path(sys.argv[4]).parent
|
||||
validate_recovery_bump(json.load(open(sys.argv[3])), (directory / 'source-pom.xml').read_text(),
|
||||
(directory / 'bump-pom.xml').read_text(), recovered, sys.argv[1])
|
||||
print(evaluate(sys.argv[1], json.load(open(sys.argv[2])), json.load(open(sys.argv[3])), recovered))
|
||||
except (ValueError, KeyError) as error:
|
||||
sys.exit(str(error))
|
||||
PY
|
||||
|
||||
# Dispatch payload is only a hint. Bind it to immutable successful cut-run evidence.
|
||||
[[ "$MERGE_SHA" =~ ^[0-9a-f]{40}$ && "$BUMP_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$MERGE_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
if [[ "$RECOVER" == true ]]; then
|
||||
[[ "$RECOVERY_EVENT" == workflow_dispatch && "$CIRCLE_BUILD" =~ ^[1-9][0-9]*$ ]]
|
||||
else
|
||||
[[ "$BUMP_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
fi
|
||||
[[ "$BRANCH" =~ ^release/[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]
|
||||
[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-RC\.[0-9]+$ ]]
|
||||
artifact_name="qbit-rc-intent-$MERGE_SHA"
|
||||
@@ -278,13 +458,53 @@ jobs:
|
||||
PY
|
||||
MERGED_AT=$(jq -er .merged_at "$workdir/intent.json")
|
||||
lock=automation/qbit-rc-pending
|
||||
current_lock=$(gh api "repos/$REPO/git/ref/heads/$lock" -q .object.sha 2>/dev/null || true)
|
||||
current_lock=$(read_ref "heads/$lock")
|
||||
# A duplicate after completed finalization is safe only after tag/source checks below.
|
||||
[[ -z "$current_lock" || "$current_lock" == "$MERGE_SHA" ]] || { echo "::error::Different release owns pending lock"; exit 1; }
|
||||
gh api "repos/$REPO/commits/$MERGE_SHA/check-runs?filter=latest&per_page=100" \
|
||||
| jq --arg sha "$MERGE_SHA" '{sha:$sha,check_runs:.check_runs}' > "$workdir/status.json"
|
||||
recovered_args=()
|
||||
if [[ "$RECOVER" == true ]]; then
|
||||
python3 "$workdir/rc_guard.py" lock "$current_lock" "$MERGE_SHA"
|
||||
python3 "$workdir/rc_guard.py" recover "$workdir" "$REPO" "$MERGE_SHA" "$BRANCH" "$CIRCLE_BUILD" "$VERSION"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "Recovery evidence verified; dry run leaves branch, tag, release and lock unchanged." >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
head=$(gh api "repos/$REPO/branches/$BRANCH" -q .commit.sha)
|
||||
if [[ "$head" == "$MERGE_SHA" ]]; then
|
||||
key="$workdir/recovery_key"
|
||||
install -m 600 /dev/null "$key"
|
||||
printf '%s\n' "$SIGNING_KEY" > "$key"
|
||||
git init -q "$workdir/recovery-repo"
|
||||
git -C "$workdir/recovery-repo" fetch -q --depth=1 "https://x-access-token:$GH_TOKEN@github.com/$REPO.git" "$MERGE_SHA"
|
||||
git -C "$workdir/recovery-repo" checkout -q -b "$BRANCH" FETCH_HEAD
|
||||
python3 - "$workdir/recovery-repo/pom.xml" "$VERSION" <<'PY'
|
||||
import pathlib, re, sys
|
||||
path, version = pathlib.Path(sys.argv[1]), sys.argv[2]
|
||||
source = path.read_text()
|
||||
revisions = re.findall(r'<revision>([^<]+)</revision>', source)
|
||||
assert len(revisions) == 1 and re.fullmatch(re.escape(version.split('-RC.')[0]) + r'-(SNAPSHOT|RC\.\d+)', revisions[0]), 'Unexpected source revision'
|
||||
path.write_text(source.replace('<revision>' + revisions[0] + '</revision>', '<revision>' + version + '</revision>'))
|
||||
PY
|
||||
git -C "$workdir/recovery-repo" add pom.xml
|
||||
git -C "$workdir/recovery-repo" -c user.name="James Maes" -c user.email=james@kof22.com -c gpg.format=ssh -c user.signingkey="$key" commit -q -S -m "Bump version to $VERSION [skip ci]"
|
||||
python3 "$workdir/rc_guard.py" lock "$(gh api "repos/$REPO/git/ref/heads/$lock" -q .object.sha)" "$MERGE_SHA"
|
||||
# A normal fast-forward push refuses a competing release-branch change.
|
||||
git -C "$workdir/recovery-repo" push -q "https://x-access-token:$GH_TOKEN@github.com/$REPO.git" "HEAD:$BRANCH"
|
||||
BUMP_SHA=$(git -C "$workdir/recovery-repo" rev-parse HEAD)
|
||||
rm -f "$key"
|
||||
else
|
||||
BUMP_SHA=$head
|
||||
fi
|
||||
recovered_args=("$workdir/recovery.json")
|
||||
echo "Verified original Central upload after Circle timeout; no artifact was uploaded again." >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
gh api "repos/$REPO/commits/$BUMP_SHA" > "$workdir/commit.json"
|
||||
version=$(python3 "$workdir/rc_guard.py" "$MERGE_SHA" "$workdir/status.json" "$workdir/commit.json")
|
||||
if [[ "$RECOVER" == true ]]; then
|
||||
gh api "repos/$REPO/contents/pom.xml?ref=$BUMP_SHA" -q .content | base64 -d > "$workdir/bump-pom.xml"
|
||||
fi
|
||||
version=$(python3 "$workdir/rc_guard.py" "$MERGE_SHA" "$workdir/status.json" "$workdir/commit.json" "${recovered_args[@]}")
|
||||
[[ -n "$version" && "$version" == "$VERSION" ]] || { echo "::error::Publisher not successful or callback version mismatch"; exit 1; }
|
||||
bump_sha=$BUMP_SHA
|
||||
[[ "$(gh api "repos/$REPO/branches/$BRANCH" -q .commit.sha)" == "$bump_sha" ]] || { echo "::error::Release branch moved away from exact bump"; exit 1; }
|
||||
@@ -316,7 +536,7 @@ jobs:
|
||||
python3 "$workdir/rc_guard.py" central "$version" "$artifact" "$workdir/central.pom" "$last_modified" "$MERGED_AT"
|
||||
|
||||
# 3. Signed annotated tag on the exact bump; duplicate runs must not accept a different tag.
|
||||
existing=$(gh api "repos/$REPO/git/ref/tags/v$version" --jq '.object.sha' 2>/dev/null || true)
|
||||
existing=$(read_ref "tags/v$version")
|
||||
target=""; release_exists=false
|
||||
if [[ -n "$existing" ]]; then
|
||||
target=$(gh api "repos/$REPO/git/tags/$existing" --jq '.object.sha')
|
||||
|
||||
@@ -61,11 +61,23 @@ parser is pinned to PyYAML 6.0.3 and uses its safe loader.
|
||||
|
||||
| Event | Workflow | Job contract |
|
||||
| --- | --- | --- |
|
||||
| develop | publish_snapshot | existing mvn_publish, snapshot |
|
||||
| release/* | publish_release_candidate | existing mvn_publish, release_candidate |
|
||||
| vX.Y.Z tag resolving to the checkout | publish_release | existing mvn_publish, release |
|
||||
| main, feature/*, hotfix/*, other/unknown branch | test_only | existing mvn_test_only |
|
||||
| Disabled publication or unrecognized/unproven tag | test_only | full verification, no publication |
|
||||
| develop | verify | existing mvn_publish, snapshot |
|
||||
| release/* | verify | existing mvn_publish, release_candidate |
|
||||
| vX.Y.Z tag resolving to the checkout | verify | existing mvn_publish, release |
|
||||
| main, feature/*, hotfix/*, other/unknown branch | verify | existing mvn_test_only |
|
||||
| Disabled publication or unrecognized/unproven tag | verify | full verification, no publication |
|
||||
|
||||
Generated `qbit`, `core`, `frontend-core`, and `frontend-material` pipelines use
|
||||
the stable workflow name `verify`. All selected jobs, dependencies and filters
|
||||
remain intact; this adds no executor or duplicate build. Other profiles retain
|
||||
their existing workflow names and native checks.
|
||||
|
||||
When upgrading from legacy workflow names, upgrade the immutable RC worker pin
|
||||
alongside the orb so finalization recognizes both names. Confirm the CircleCI
|
||||
app's successful `verify` check on the exact migration commit before replacing
|
||||
legacy required contexts. Require `setup` and `verify` on managed branches,
|
||||
preserving other required checks and controls. Human approval is not required;
|
||||
automated checks authorize the migration.
|
||||
|
||||
The pipeline SHA must match the checkout when Circle supplies it. A tagged release
|
||||
must resolve to that checkout; missing local tags are fetched individually. A main
|
||||
@@ -182,7 +194,7 @@ manifest validation already ran in setup. It adds no new security scanner and
|
||||
makes no security-scanning claim. Keep docs-only disabled if those paths are build
|
||||
inputs or packaged application content.
|
||||
|
||||
The light route keeps workflow `test_only` and names its job
|
||||
The light route keeps workflow `verify` and names its job
|
||||
`qqq-orb/mvn_test_only`. Verify actual CircleCI/GitHub status contexts on the canary
|
||||
before changing any protection rule; workflow/job identity alone is not proof of
|
||||
the hosted context name. Current protections must not be removed speculatively.
|
||||
|
||||
+39
-1
@@ -26,7 +26,8 @@ https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-t
|
||||
- Finalization retrieves the unique unexpired intent artifact and verifies its
|
||||
originating successful `release-rc.yml` run, repository, trigger and attempt.
|
||||
Callback fields must match that immutable intent.
|
||||
- The exact merge SHA must have a successful `publish_release_candidate` Check Run
|
||||
- The exact merge SHA must have a successful `verify` (or legacy
|
||||
`publish_release_candidate`) Check Run
|
||||
from CircleCI app ID `18001`; unrelated job statuses cannot authorize release.
|
||||
Terminal publisher failure stops finalization immediately. Pending/absent publication cannot authorize a tag.
|
||||
The RC bump must have exactly that one parent and the release branch must still
|
||||
@@ -80,3 +81,40 @@ release-captain reconciliation; the sweep does not invent provenance.
|
||||
syntax. Full actionlint with Windows native ShellCheck hung locally and was
|
||||
stopped; Linux actionlint/ShellCheck and end-to-end credential/callback rollout
|
||||
verification remain required. The orb has separate sweep/dispatch fixture tests.
|
||||
|
||||
### Published upload timeout recovery
|
||||
|
||||
A Circle deployment step can time out after Central has accepted and published its
|
||||
bundle. Use the canonical caller's explicit manual `recover` input with the existing
|
||||
`merge_sha`, `branch`, `version` and original `circle_build`. Run with `dry_run: true`
|
||||
first to verify evidence without changing the branch, tag, release or pending lock.
|
||||
This mode never builds, uploads, cuts a new candidate, or changes the failed Circle check.
|
||||
|
||||
The pinned orb worker requires the original successful cut intent and unchanged lock,
|
||||
the exact failed Circle workflow/check, a final deployment timeout after all preceding
|
||||
actions (including Maven Verify) succeeded, exactly one successful upload receipt,
|
||||
and an authenticated Central `PUBLISHED` response matching every staged coordinate.
|
||||
It also checks public POM coordinates and publication timestamps after the cut.
|
||||
Existing organization `CENTRAL_USERNAME` and `CENTRAL_PASSWORD` secrets authenticate
|
||||
only the Central status request; Circle evidence uses its public API. Redirects are
|
||||
rejected and credentials are never included in the recovery receipt.
|
||||
|
||||
After verification, recovery creates only the missing signed version bump as a direct
|
||||
child of the original release merge, using a normal fast-forward push. An existing
|
||||
bump must pass the same parent/version checks. The existing signed-tag/prerelease
|
||||
finalizer then completes the candidate and compares lock ownership before cleanup.
|
||||
A later explicit recovery retry verifies the original receipt again. Once the lock
|
||||
is gone, the completed release needs no recovery dispatch. Ordinary callbacks and
|
||||
scheduled reconciliation retain the requirement for a successful Circle publisher.
|
||||
|
||||
Central status API: https://central.sonatype.org/publish/publish-portal-api/
|
||||
|
||||
### Auditing independent worker revisions
|
||||
|
||||
The estate audit accepts `rc_worker_sha` in a profile or repository policy. For
|
||||
example, set `profiles.qbit.rc_worker_sha` to the approved immutable RC worker
|
||||
commit and keep `--worker-sha` at the approved revision for unchanged native
|
||||
workers. The audit still compares every caller to its exact canonical template
|
||||
and rejects mutable or mismatched pins. This avoids a fleet-wide native caller
|
||||
update when only RC release behavior changes. Required-check policy is separate;
|
||||
omitting it continues to report enforcement as unknown.
|
||||
@@ -441,6 +441,12 @@ def main():
|
||||
if expected and (not re.fullmatch(r"[0-9a-fA-F]{40}", expected) or git("rev-parse", "HEAD") != (expected + "\n").encode()):
|
||||
raise ManifestError("Checkout does not match the pipeline commit")
|
||||
output = PROFILE_TEMPLATES[config["profile"]](config, os.environ.get("CIRCLE_BRANCH", ""), os.environ.get("CIRCLE_TAG", ""))
|
||||
if config["profile"] in ("qbit", "core", "frontend-core", "frontend-material"):
|
||||
# One whole-workflow check follows the same commit from feature to release.
|
||||
# Keep every selected job, dependency and filter; add no runner or duplicate build.
|
||||
if len(output["workflows"]) != 1:
|
||||
raise ManifestError("Stable verification requires exactly one selected workflow")
|
||||
output["workflows"] = {"verify": next(iter(output["workflows"].values()))}
|
||||
parameters = continuation_parameters(config["profile"])
|
||||
if config["profile"] == "core":
|
||||
output["parameters"] = copy.deepcopy(CORE_TEMPLATE["parameters"])
|
||||
|
||||
@@ -17,12 +17,16 @@ def canonical(worker_sha):
|
||||
"dry_run": {"description": "Plan a cut without changing release state", "type": "boolean", "default": False},
|
||||
"finalize": {"description": "Retry finalization of an existing cut; do not cut another RC", "type": "boolean", "default": False},
|
||||
}
|
||||
inputs['recover'] = {'description': 'Recover the same published RC after a deployment timeout; never reupload', 'type': 'boolean', 'default': False}
|
||||
inputs['circle_build'] = {'description': 'Original Circle publisher build number (recovery only)', 'type': 'string', 'default': ''}
|
||||
for field in ("merge_sha", "bump_sha", "branch", "version"):
|
||||
inputs[field] = {"description": "Existing cut " + field + " (finalize retry only)", "type": "string", "default": ""}
|
||||
forwarded = {
|
||||
"dry_run": "${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false }}",
|
||||
"finalize": "${{ github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_dispatch' && inputs.finalize) }}",
|
||||
}
|
||||
forwarded['recover'] = "${{ github.event_name == 'workflow_dispatch' && inputs.recover || false }}"
|
||||
forwarded['circle_build'] = "${{ github.event_name == 'workflow_dispatch' && inputs.circle_build || '' }}"
|
||||
for field in ("merge_sha", "bump_sha", "branch", "version"):
|
||||
forwarded[field] = "${{ github.event_name == 'repository_dispatch' && github.event.client_payload." + field + " || inputs." + field + " || '' }}"
|
||||
return {
|
||||
|
||||
@@ -441,6 +441,12 @@ def main():
|
||||
if expected and (not re.fullmatch(r"[0-9a-fA-F]{40}", expected) or git("rev-parse", "HEAD") != (expected + "\n").encode()):
|
||||
raise ManifestError("Checkout does not match the pipeline commit")
|
||||
output = PROFILE_TEMPLATES[config["profile"]](config, os.environ.get("CIRCLE_BRANCH", ""), os.environ.get("CIRCLE_TAG", ""))
|
||||
if config["profile"] in ("qbit", "core", "frontend-core", "frontend-material"):
|
||||
# One whole-workflow check follows the same commit from feature to release.
|
||||
# Keep every selected job, dependency and filter; add no runner or duplicate build.
|
||||
if len(output["workflows"]) != 1:
|
||||
raise ManifestError("Stable verification requires exactly one selected workflow")
|
||||
output["workflows"] = {"verify": next(iter(output["workflows"].values()))}
|
||||
parameters = continuation_parameters(config["profile"])
|
||||
if config["profile"] == "core":
|
||||
output["parameters"] = copy.deepcopy(CORE_TEMPLATE["parameters"])
|
||||
|
||||
@@ -116,7 +116,7 @@ def reconcile(api, repo):
|
||||
validate_run(run, intent, repo, merge, run_id)
|
||||
checks = api.request(prefix + 'commits/' + merge + '/check-runs?filter=latest&per_page=100')
|
||||
publishers = [c for c in checks.get('check_runs', [])
|
||||
if c.get('name') == 'publish_release_candidate' and c.get('app', {}).get('id') == 18001]
|
||||
if c.get('name') in ('publish_release_candidate', 'verify') and c.get('app', {}).get('id') == 18001]
|
||||
if not publishers:
|
||||
return 'publisher-running'
|
||||
publisher = max(publishers, key=lambda c: c['id'])
|
||||
|
||||
@@ -74,6 +74,25 @@ class AuditTests(unittest.TestCase):
|
||||
report = audit.audit(entry, Snapshot({'.qqq-ci.yml': audit.generator.yaml.safe_dump(manifest)}), VERSION, SHA, POLICY)
|
||||
self.assertIn('missing_rc_bridge', [f['code'] for f in report['findings']])
|
||||
|
||||
def test_rc_pin_policy_does_not_force_unchanged_auxiliary_worker_updates(self):
|
||||
entry = dict(ENTRY, name='qbit-authentik', nameWithOwner='QRun-IO/qbit-authentik')
|
||||
rc_sha = 'b' * 40
|
||||
definition = audit.auxiliary.BRIDGES['qbit-schema-drift']
|
||||
manifest = {'profile': 'qbit', 'orb_version': VERSION, 'default_branch': 'develop', 'release_completion': True}
|
||||
files = {audit.rc_worker.CALLER_PATH: audit.rc_worker.generate(rc_sha),
|
||||
definition['path']: audit.generator.yaml.safe_dump(audit.auxiliary.canonical('qbit-schema-drift', SHA), sort_keys=False),
|
||||
'.qqq-ci.yml': audit.generator.yaml.safe_dump(manifest),
|
||||
'.circleci/config.yml': audit.generator.yaml.safe_dump(audit.standard.canonical_caller('qbit', VERSION))}
|
||||
policy = dict(POLICY, rc_worker_sha=rc_sha)
|
||||
self.assertEqual('pass', audit.audit(entry, Snapshot(files), VERSION, SHA, policy)['status'])
|
||||
for pin in (SHA, 'main', ''):
|
||||
with self.subTest(pin=pin):
|
||||
report = audit.audit(entry, Snapshot(files), VERSION, SHA, dict(POLICY, rc_worker_sha=pin))
|
||||
self.assertTrue(report['findings'])
|
||||
files[definition['path']] = files[definition['path']].replace(SHA, rc_sha)
|
||||
report = audit.audit(entry, Snapshot(files), VERSION, SHA, policy)
|
||||
self.assertIn('native_maintenance_bridge_drift', [f['code'] for f in report['findings']])
|
||||
|
||||
def test_qbit_rc_bridge_missing_drift_pin_and_duplicate(self):
|
||||
entry = dict(ENTRY, name='qbit-test', nameWithOwner='QRun-IO/qbit-test')
|
||||
path = audit.rc_worker.CALLER_PATH
|
||||
|
||||
@@ -192,10 +192,12 @@ class PipelineTests(unittest.TestCase):
|
||||
return result
|
||||
|
||||
def assert_full(self, result, workflow="test_only"):
|
||||
self.assertEqual(list(result["workflows"]), [workflow])
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
self.assertNotIn("docs_only", result.get("jobs", {}))
|
||||
job, options = next(iter(result["workflows"][workflow]["jobs"][0].items()))
|
||||
job, options = next(iter(result["workflows"]["verify"]["jobs"][0].items()))
|
||||
self.assertEqual(job, "qqq-orb/mvn_test_only" if workflow == "test_only" else "qqq-orb/mvn_publish")
|
||||
if workflow != "test_only":
|
||||
self.assertEqual(options["branch_type"], workflow.removeprefix("publish_"))
|
||||
self.assertEqual(options["context"], ["qqq-maven-registry-credentials"])
|
||||
self.assertFalse(options["compile_before_verify"])
|
||||
return options
|
||||
@@ -298,8 +300,8 @@ class PipelineTests(unittest.TestCase):
|
||||
self.change_docs()
|
||||
for packed in ([False, True] if PACKED else [False]):
|
||||
result = self.render(packed=packed)
|
||||
self.assertEqual(list(result["workflows"]), ["test_only"])
|
||||
options = result["workflows"]["test_only"]["jobs"][0]["docs_only"]
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
options = result["workflows"]["verify"]["jobs"][0]["docs_only"]
|
||||
self.assertEqual(options["name"], "qqq-orb/mvn_test_only")
|
||||
self.assertIn("git diff --check", result["jobs"]["docs_only"]["steps"][1]["run"]["command"])
|
||||
(self.repo / "App.java").write_text("changed\n", encoding="utf-8")
|
||||
@@ -360,8 +362,8 @@ class PipelineTests(unittest.TestCase):
|
||||
for branch, tag, workflow in events:
|
||||
with self.subTest(workflow=workflow):
|
||||
result = self.render(CIRCLE_BRANCH=branch, CIRCLE_TAG=tag)
|
||||
self.assertEqual(list(result["workflows"]), [workflow])
|
||||
actual = result["workflows"][workflow]["jobs"]
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
actual = result["workflows"]["verify"]["jobs"]
|
||||
expected = baseline["workflows"][workflow]["jobs"]
|
||||
self.assertEqual(len(actual), len(expected))
|
||||
for before, after in zip(expected, actual):
|
||||
@@ -422,8 +424,8 @@ class PipelineTests(unittest.TestCase):
|
||||
with self.subTest(branch=branch, enabled=enabled):
|
||||
result = self.render(CIRCLE_BRANCH=branch, QQQ_CI_RUN_PLAYWRIGHT=str(enabled).lower())
|
||||
workflow = base + ("_with_playwright" if enabled else "")
|
||||
self.assertEqual(list(result["workflows"]), [workflow])
|
||||
jobs = result["workflows"][workflow]["jobs"]
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
jobs = result["workflows"]["verify"]["jobs"]
|
||||
self.assertEqual(len(jobs), 2 if enabled else 1)
|
||||
maven = [j for j in jobs if next(iter(j)).startswith("qqq-orb/mvn_frontend_")]
|
||||
self.assertEqual(len(maven), 1)
|
||||
@@ -454,8 +456,10 @@ class PipelineTests(unittest.TestCase):
|
||||
("hotfix/fix", "", "publish_hotfix_release"), ("", "publish-fixture", "publish_feature"),
|
||||
("", "v1.0.0", "publish_release")]:
|
||||
result = self.render(CIRCLE_BRANCH=branch, CIRCLE_TAG=tag)
|
||||
self.assertEqual(list(result["workflows"]), [workflow])
|
||||
options = next(iter(result["workflows"][workflow]["jobs"][0].values()))
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
options = next(iter(result["workflows"]["verify"]["jobs"][0].values()))
|
||||
expected_kind = {"publish_hotfix_release": "hotfix"}.get(workflow, workflow.removeprefix("publish_"))
|
||||
self.assertEqual(options["branch_type"], expected_kind)
|
||||
if profile == "frontend-core":
|
||||
self.assertTrue(options["npm_trusted_publishing"])
|
||||
self.assertEqual(options["node_version"], "24.21")
|
||||
@@ -463,9 +467,9 @@ class PipelineTests(unittest.TestCase):
|
||||
else:
|
||||
self.assertEqual(options["context"], ["qqq-maven-registry-credentials"])
|
||||
main = self.render(CIRCLE_BRANCH="main")
|
||||
self.assertEqual(list(main["workflows"]), ["publish_release" if profile == "frontend-core" else "test_only"])
|
||||
self.assertEqual(list(main["workflows"]), ["verify"])
|
||||
rc_tag = self.render(CIRCLE_BRANCH="", CIRCLE_TAG="v1.0.0-RC.1")
|
||||
self.assertEqual(list(rc_tag["workflows"]), ["publish_release" if profile == "frontend-core" else "test_only"])
|
||||
self.assertEqual(list(rc_tag["workflows"]), ["verify"])
|
||||
|
||||
def test_profile_specific_options_rejected(self):
|
||||
for profile, options in [("qbit", {"node_version": "22"}), ("frontend-core", {"browsers": False}),
|
||||
@@ -532,7 +536,8 @@ class PipelineTests(unittest.TestCase):
|
||||
{"CIRCLE_BRANCH": "integration-example"}):
|
||||
result = self.render(**environment)
|
||||
if environment.get("CIRCLE_BRANCH") != "integration-example":
|
||||
self.assertIn("publish_release", result["workflows"])
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
self.assertEqual(result["workflows"]["verify"]["jobs"][0]["qqq-orb/mvn_publish"]["branch_type"], "release")
|
||||
if PACKED:
|
||||
self.assertEqual(result, self.render(packed=True, **environment))
|
||||
|
||||
|
||||
@@ -35,13 +35,16 @@ class RcBridgeTests(unittest.TestCase):
|
||||
data = rc.canonical(SHA)
|
||||
inputs = data["on"]["workflow_dispatch"]["inputs"]
|
||||
passed = data["jobs"]["release-rc"]["with"]
|
||||
self.assertEqual({"dry_run", "finalize", "merge_sha", "bump_sha", "branch", "version"}, set(inputs))
|
||||
self.assertEqual({"dry_run", "finalize", "recover", "circle_build", "merge_sha", "bump_sha", "branch", "version"}, set(inputs))
|
||||
self.assertEqual(set(inputs), set(passed))
|
||||
for name in ("dry_run", "finalize"):
|
||||
for name in ("dry_run", "finalize", "recover"):
|
||||
self.assertEqual("boolean", inputs[name]["type"])
|
||||
self.assertIs(False, inputs[name]["default"])
|
||||
self.assertEqual("${{ github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_dispatch' && inputs.finalize) }}", passed["finalize"])
|
||||
self.assertEqual("${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false }}", passed["dry_run"])
|
||||
self.assertEqual("${{ github.event_name == 'workflow_dispatch' && inputs.recover || false }}", passed['recover'])
|
||||
self.assertEqual("${{ github.event_name == 'workflow_dispatch' && inputs.circle_build || '' }}", passed['circle_build'])
|
||||
self.assertEqual({'description': 'Original Circle publisher build number (recovery only)', 'type': 'string', 'default': ''}, inputs['circle_build'])
|
||||
for name in ("merge_sha", "bump_sha", "branch", "version"):
|
||||
self.assertEqual("string", inputs[name]["type"])
|
||||
self.assertEqual("", inputs[name]["default"])
|
||||
|
||||
@@ -58,6 +58,17 @@ class FixtureAPI:
|
||||
|
||||
|
||||
class SweepTests(unittest.TestCase):
|
||||
def test_stable_gate_and_mixed_name_latest_failure(self):
|
||||
api = FixtureAPI()
|
||||
api.status['check_runs'][0]['name'] = 'verify'
|
||||
self.assertEqual('dispatched', sweep.reconcile(api, REPO))
|
||||
api.sent.clear()
|
||||
api.status['check_runs'].append(dict(api.status['check_runs'][0],
|
||||
id=10, name='publish_release_candidate', conclusion='failure'))
|
||||
with self.assertRaises(sweep.InvalidIntent):
|
||||
sweep.reconcile(api, REPO)
|
||||
self.assertEqual([], api.sent)
|
||||
|
||||
def test_idle_never_dispatches_or_reads_artifacts(self):
|
||||
api = FixtureAPI(); api.lock = None
|
||||
self.assertEqual('idle', sweep.reconcile(api, REPO))
|
||||
|
||||
@@ -23,6 +23,17 @@ def bump(parent=SHA, version='1.2.3-RC.4'):
|
||||
|
||||
|
||||
class ReleaseGuardTests(unittest.TestCase):
|
||||
def test_stable_workflow_preserves_exact_source_and_latest_attempt_checks(self):
|
||||
result = status()
|
||||
result['check_runs'][0]['name'] = 'verify'
|
||||
self.assertEqual('1.2.3-RC.4', evaluate(SHA, result, bump()))
|
||||
result['check_runs'].append(dict(result['check_runs'][0], id=10, conclusion='failure'))
|
||||
with self.assertRaisesRegex(Failure, 'CircleCI failed'):
|
||||
evaluate(SHA, result, bump())
|
||||
result['check_runs'] = [dict(result['check_runs'][0], head_sha=OTHER)]
|
||||
with self.assertRaises(Failure):
|
||||
evaluate(SHA, result, bump())
|
||||
|
||||
def test_success_exact_built_source(self):
|
||||
self.assertEqual('1.2.3-RC.4', evaluate(SHA, status(), bump()))
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Exercise actual shell handling of gh error JSON, including the observed 404."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
SOURCE = Path(__file__).parents[1].joinpath('.github/workflows/qbit-release-rc.yml').read_text()
|
||||
FUNCTION = textwrap.dedent(SOURCE.split(' # BEGIN RC REF LOOKUP\n')[1].split(' # END RC REF LOOKUP')[0])
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which('bash') and shutil.which('jq'), 'Bash and jq required')
|
||||
class RefLookupTests(unittest.TestCase):
|
||||
def lookup(self, body, code):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
env = dict(os.environ, BODY=body, CODE=str(code), REPO='QRun-IO/qbit-alerts', workdir=directory)
|
||||
script = 'set -euo pipefail\ngh() { printf "%s" "$BODY"; return "$CODE"; }\n' + FUNCTION + '\nresult=$(read_ref tags/v0.1.0-RC.17)\nprintf "%s" "$result"\n'
|
||||
return subprocess.run(['bash', '-c', script], env=env, text=True, capture_output=True)
|
||||
|
||||
def test_absent_tag_returns_empty_not_error_json(self):
|
||||
result = self.lookup('{"message":"Not Found","documentation_url":"https://docs.github.com/rest/git/refs#get-a-reference","status":"404"}', 1)
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertEqual('', result.stdout)
|
||||
|
||||
def test_existing_tag_returns_only_valid_object_sha(self):
|
||||
result = self.lookup('{"object":{"sha":"' + 'a' * 40 + '"}}', 0)
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertEqual('a' * 40, result.stdout)
|
||||
|
||||
def test_permission_network_and_server_errors_are_not_missing_refs(self):
|
||||
for body in ('{"status":"403"}', '{"status":"500"}', '', ' ', '\n', 'not json'):
|
||||
with self.subTest(body=body):
|
||||
self.assertNotEqual(0, self.lookup(body, 1).returncode)
|
||||
|
||||
def test_malformed_success_response_is_rejected(self):
|
||||
for body in ('{}', '{"object":{"sha":null}}', '{"object":{"sha":"not-a-sha"}}'):
|
||||
with self.subTest(body=body):
|
||||
self.assertNotEqual(0, self.lookup(body, 0).returncode)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,136 @@
|
||||
"""A timeout recovery must prove the original upload, not forgive a failed build."""
|
||||
import copy
|
||||
import unittest
|
||||
|
||||
from test_rc_guard import scope, SHA, OTHER, Failure, bump, status, evaluate
|
||||
|
||||
REPO = 'QRun-IO/qbit-alerts'
|
||||
WORKFLOW = '86efe478-29d6-41a4-93f0-d74b178bf329'
|
||||
DEPLOYMENT = '993e5a6d-23ce-4696-8e5a-9176ae19e670'
|
||||
VERSION = '0.1.0-RC.17'
|
||||
PURL = 'pkg:maven/com.kingsrook.qbits/qbit-alerts@' + VERSION
|
||||
|
||||
|
||||
def evidence():
|
||||
publisher = status('failure')['check_runs'][0]
|
||||
publisher['details_url'] = 'https://app.circleci.com/workflow/' + WORKFLOW + '?utm_source=github'
|
||||
job = {'username': 'QRun-IO', 'reponame': 'qbit-alerts', 'vcs_revision': SHA,
|
||||
'branch': 'release/0.1', 'build_num': 148, 'status': 'failed', 'outcome': 'failed',
|
||||
'start_time': '2026-10-21T00:01:00Z',
|
||||
'workflows': {'workflow_name': 'publish_release_candidate',
|
||||
'job_name': 'qqq-orb/mvn_publish', 'workflow_id': WORKFLOW},
|
||||
'steps': [{'actions': [{'name': 'Run Maven Verify', 'status': 'success'}]},
|
||||
{'actions': [{'name': 'Publish to Sonatype Central (releases and SNAPSHOTs)',
|
||||
'status': 'timedout', 'timedout': True, 'step': 124, 'index': 0}]}]}
|
||||
return job, publisher
|
||||
|
||||
|
||||
def validate(job, publisher):
|
||||
return scope['validate_timeout'](job, publisher, REPO, SHA, 'release/0.1', '148', '2026-10-21T00:00:00Z')
|
||||
|
||||
|
||||
class TimeoutRecoveryTests(unittest.TestCase):
|
||||
def test_stable_gate_must_match_the_actual_publishing_workflow(self):
|
||||
job, publisher = evidence()
|
||||
job['workflows']['workflow_name'] = 'verify'
|
||||
publisher['name'] = 'verify'
|
||||
validate(job, publisher)
|
||||
publisher['name'] = 'publish_release_candidate'
|
||||
with self.assertRaises(Failure):
|
||||
validate(job, publisher)
|
||||
|
||||
def test_exact_upload_timeout_after_successful_verify(self):
|
||||
self.assertEqual((124, 0), validate(*evidence()))
|
||||
|
||||
def test_other_source_branch_repository_or_attempt_is_rejected(self):
|
||||
for key, value in [('vcs_revision', OTHER), ('branch', 'develop'),
|
||||
('username', 'untrusted'), ('reponame', 'other'), ('build_num', 149)]:
|
||||
with self.subTest(key=key), self.assertRaises(Failure):
|
||||
job, publisher = evidence()
|
||||
job[key] = value
|
||||
validate(job, publisher)
|
||||
|
||||
def test_non_circle_or_unrelated_workflow_cannot_authorize_recovery(self):
|
||||
for key, value in [('head_sha', OTHER), ('app', {'id': 1}), ('status', 'in_progress'),
|
||||
('conclusion', 'success'), ('name', 'test_only'),
|
||||
('details_url', 'https://example.com/workflow/' + WORKFLOW),
|
||||
('details_url', 'https://app.circleci.com/workflow/another')]:
|
||||
with self.subTest(key=key, value=value), self.assertRaises(Failure):
|
||||
job, publisher = evidence()
|
||||
publisher[key] = value
|
||||
validate(job, publisher)
|
||||
|
||||
def test_prior_failure_or_missing_verify_never_becomes_release_success(self):
|
||||
for action in [{'name': 'Run Maven Verify', 'status': 'failed'},
|
||||
{'name': 'Run Maven Verify', 'status': 'skipped'},
|
||||
{'name': 'Some other build', 'status': 'success'}]:
|
||||
with self.subTest(action=action), self.assertRaises(Failure):
|
||||
job, publisher = evidence()
|
||||
job['steps'][0]['actions'][0] = action
|
||||
validate(job, publisher)
|
||||
|
||||
def test_only_a_terminal_deployment_timeout_is_eligible(self):
|
||||
for change in [{'name': 'Run Maven Verify'}, {'timedout': False}, {'status': 'failed'},
|
||||
{'step': -1}, {'index': '0'}]:
|
||||
with self.subTest(change=change), self.assertRaises(Failure):
|
||||
job, publisher = evidence()
|
||||
job['steps'][-1]['actions'][0].update(change)
|
||||
validate(job, publisher)
|
||||
|
||||
def test_failed_later_git_push_is_not_an_upload_timeout(self):
|
||||
job, publisher = evidence()
|
||||
job['steps'].append({'actions': [{'name': 'Commit version', 'status': 'failed'}]})
|
||||
with self.assertRaises(Failure):
|
||||
validate(job, publisher)
|
||||
|
||||
def test_publisher_cannot_predate_the_cut(self):
|
||||
job, publisher = evidence()
|
||||
job['start_time'] = '2026-10-20T23:59:00Z'
|
||||
with self.assertRaises(Failure):
|
||||
validate(job, publisher)
|
||||
|
||||
def test_receipt_requires_one_upload_and_expected_staged_coordinates(self):
|
||||
log = ('[INFO] Installing x to /home/circleci/project/target/central-staging/'
|
||||
'com/kingsrook/qbits/qbit-alerts/' + VERSION + '/qbit-alerts-' + VERSION + '.pom\n'
|
||||
'[INFO] Uploaded bundle successfully, deployment name: Deployment, deploymentId: '
|
||||
+ DEPLOYMENT + '. Deployment will publish automatically\n')
|
||||
self.assertEqual((DEPLOYMENT, [PURL]), scope['upload_receipt'](log, 'qbit-alerts', VERSION))
|
||||
for invalid in [log + log, log.replace('Uploaded bundle successfully', 'Upload failed'),
|
||||
log.replace('qbit-alerts', 'qbit-other'), log.replace(VERSION, '0.1.0-RC.16')]:
|
||||
with self.subTest(invalid=invalid), self.assertRaises(Failure):
|
||||
scope['upload_receipt'](invalid, 'qbit-alerts', VERSION)
|
||||
|
||||
def test_central_must_confirm_the_exact_receipt_and_all_coordinates(self):
|
||||
result = {'deploymentId': DEPLOYMENT, 'deploymentState': 'PUBLISHED', 'purls': [PURL]}
|
||||
scope['validate_deployment'](result, DEPLOYMENT, [PURL])
|
||||
for key, value in [('deploymentId', 'other'), ('deploymentState', 'PUBLISHING'),
|
||||
('deploymentState', 'FAILED'), ('purls', []), ('purls', [PURL, 'other'])]:
|
||||
with self.subTest(key=key, value=value), self.assertRaises(Failure):
|
||||
invalid = copy.deepcopy(result)
|
||||
invalid[key] = value
|
||||
scope['validate_deployment'](invalid, DEPLOYMENT, [PURL])
|
||||
|
||||
def test_retry_bump_cannot_hide_other_source_changes(self):
|
||||
source = '<project><revision>0.1.0-RC.16</revision><name>Alerts</name></project>'
|
||||
actual = source.replace('0.1.0-RC.16', VERSION)
|
||||
commit = bump(version=VERSION)
|
||||
commit['files'] = [{'filename': 'pom.xml', 'status': 'modified'}]
|
||||
scope['validate_recovery_bump'](commit, source, actual, VERSION, SHA)
|
||||
for changed in [actual.replace('Alerts', 'Other'), actual + '<extra/>', source]:
|
||||
with self.subTest(changed=changed), self.assertRaises(Failure):
|
||||
scope['validate_recovery_bump'](commit, source, changed, VERSION, SHA)
|
||||
commit['files'].append({'filename': 'other.java', 'status': 'modified'})
|
||||
with self.assertRaises(Failure):
|
||||
scope['validate_recovery_bump'](commit, source, actual, VERSION, SHA)
|
||||
|
||||
def test_recovery_still_requires_exact_bump_parent_and_verified_version(self):
|
||||
self.assertEqual(VERSION, evaluate(SHA, status('failure'), bump(version=VERSION), VERSION))
|
||||
for commit in [bump(parent=OTHER, version=VERSION), bump(version='0.1.0-RC.18')]:
|
||||
with self.subTest(commit=commit), self.assertRaises(Failure):
|
||||
evaluate(SHA, status('failure'), commit, VERSION)
|
||||
with self.assertRaises(Failure):
|
||||
evaluate(SHA, status('failure'), bump(version=VERSION))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -6,6 +6,8 @@ Example: python tools/audit_estate.py --inventory inventory.json --orb-version 0
|
||||
Policy: {"profiles":{"qbit":{"required_checks":{"main":["observed context"],
|
||||
"develop":["observed context"]}}},"repositories":{"QRun-IO/example":{...}}}.
|
||||
Only use contexts verified on hosted canaries. Absent policy is unknown, not pass.
|
||||
A profile/repository policy may set rc_worker_sha independently of --worker-sha
|
||||
so an RC-only worker change does not require updating unchanged native workers.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
@@ -209,11 +211,12 @@ def audit(entry, snapshot, version, worker_sha, policy):
|
||||
if profile == 'qbit' and (rc_enabled or existing_rc):
|
||||
path = rc_worker.CALLER_PATH
|
||||
raw = rc_source
|
||||
rc_sha = policy.get("rc_worker_sha", worker_sha)
|
||||
if raw is None:
|
||||
finding('missing_rc_bridge', path)
|
||||
elif not re.fullmatch(r'[0-9a-f]{40}', worker_sha or ''):
|
||||
elif not re.fullmatch(r'[0-9a-f]{40}', rc_sha or ''):
|
||||
finding('native_pin_policy_missing', path)
|
||||
elif raw != rc_worker.generate(worker_sha):
|
||||
elif raw != rc_worker.generate(rc_sha):
|
||||
finding('rc_bridge_drift', path)
|
||||
else:
|
||||
native_paths.add(path)
|
||||
|
||||
Reference in new issue
Block a user