Compare commits
37
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
66b93dccce | ||
|
|
a53af9ca33 | ||
|
|
978d6939d7 | ||
|
|
0131a1f415 | ||
|
|
5bdda2f8c9 | ||
|
|
7b1756818f | ||
|
|
68f17d3b09 | ||
|
|
130bc35ab0 | ||
|
|
9b7161c3c8 | ||
|
|
482d72ce21 | ||
|
|
45a439796f | ||
|
|
02f0b93e41 | ||
|
|
5e433ec629 | ||
|
|
cec7a76f12 | ||
|
|
8717d3536e | ||
|
|
8229a66bc5 | ||
|
|
9687feb36c | ||
|
|
0fefe6a064 | ||
|
|
2956e3ffce | ||
|
|
0afe2a231f | ||
|
|
65265e48a4 | ||
|
|
49399deb3d | ||
|
|
9fe4257490 | ||
|
|
0965a45653 | ||
|
|
bfd41c8e0d | ||
|
|
9c30a30754 | ||
|
|
ee8953bd93 | ||
|
|
5bc3167286 | ||
|
|
1a3368e119 | ||
|
|
1ddba883c9 | ||
|
|
da8044d6f2 | ||
|
|
67ba48da1f | ||
|
|
5196609974 | ||
|
|
49e09a6776 | ||
|
|
748016f105 | ||
|
|
6514aaa688 | ||
|
|
3ac47adc21 |
No files matched your search
+34
-1
@@ -4,6 +4,9 @@ parameters:
|
||||
run_rc_finalize_sweep:
|
||||
type: boolean
|
||||
default: false
|
||||
run_enrollment_reconcile:
|
||||
type: boolean
|
||||
default: false
|
||||
orbs:
|
||||
orb-tools: circleci/orb-tools@12.3
|
||||
shellcheck: circleci/shellcheck@3.2
|
||||
@@ -13,6 +16,23 @@ filters: &filters
|
||||
only: /.*/
|
||||
|
||||
jobs:
|
||||
enrollment-reconcile:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
resource_class: small
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Install reconciliation tools
|
||||
command: |
|
||||
bash tools/install_reconcile_gh.sh
|
||||
python3 -m pip install --disable-pip-version-check PyYAML==6.0.3
|
||||
- run:
|
||||
name: Reconcile current QQQ branches
|
||||
no_output_timeout: 21m
|
||||
command: python3 tools/run_enrollment.py
|
||||
- store_artifacts:
|
||||
path: artifacts/enrollment
|
||||
rc-finalize-sweep:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
@@ -28,7 +48,10 @@ jobs:
|
||||
workflows:
|
||||
lint-pack:
|
||||
when:
|
||||
not: << pipeline.parameters.run_rc_finalize_sweep >>
|
||||
not:
|
||||
or:
|
||||
- << pipeline.parameters.run_rc_finalize_sweep >>
|
||||
- << pipeline.parameters.run_enrollment_reconcile >>
|
||||
jobs:
|
||||
- orb-tools/lint:
|
||||
filters: *filters
|
||||
@@ -56,3 +79,13 @@ workflows:
|
||||
only: main
|
||||
tags:
|
||||
ignore: /.*/
|
||||
reconcile-enrollment:
|
||||
when: << pipeline.parameters.run_enrollment_reconcile >>
|
||||
jobs:
|
||||
- enrollment-reconcile:
|
||||
context: qqq-maven-registry-credentials
|
||||
filters:
|
||||
branches:
|
||||
only: main
|
||||
tags:
|
||||
ignore: /.*/
|
||||
@@ -31,6 +31,9 @@ jobs:
|
||||
- run:
|
||||
name: Verify resolved multi-module publication POMs
|
||||
command: python3 tests/test_maven_publication_poms.py
|
||||
- run:
|
||||
name: Verify calculated versions match installed Maven coordinates
|
||||
command: python3 tests/test_effective_publication_version.py
|
||||
pipeline-contract-test:
|
||||
resource_class: small
|
||||
docker:
|
||||
@@ -49,10 +52,23 @@ jobs:
|
||||
python3 tests/test_cli_ci_contract.py
|
||||
python3 tests/test_maven_ci_contract.py
|
||||
python3 tests/test_standardize_qbit.py
|
||||
python3 tests/test_qbit_hotfix_profile.py
|
||||
python3 tests/test_qbit_feature_profile.py
|
||||
python3 tests/test_ci_maintenance.py
|
||||
python3 tests/test_rollout_version.py
|
||||
python3 tests/test_enforce_orb_checks.py
|
||||
python3 tests/test_enroll_consumers.py
|
||||
python3 tests/test_reconcile_enrollment.py
|
||||
python3 tests/test_discover_enrollment.py
|
||||
python3 tests/test_finish_enrollment.py
|
||||
python3 tests/test_run_enrollment.py
|
||||
python3 tests/test_static_analysis_exit.py
|
||||
python3 tests/test_validate_standard.py
|
||||
python3 tests/test_voyage_profile.py
|
||||
python3 tests/test_next_native.py
|
||||
python3 tests/test_auxiliary_workflows.py
|
||||
python3 tests/test_qbit_codeql.py
|
||||
python3 tests/test_repository_policy.py
|
||||
python3 tests/test_intellij_plugin.py
|
||||
python3 tests/test_workflow_lint.py
|
||||
python3 tests/test_business_e2e.py
|
||||
@@ -60,6 +76,8 @@ jobs:
|
||||
python3 -m unittest discover -s tests -p test_generate_qctl_workflow.py
|
||||
python3 tests/test_rc_finalize_sweep.py
|
||||
python3 tests/test_rc_guard.py
|
||||
python3 -m unittest discover -s tests -p test_rc_timeout_recovery.py
|
||||
python3 -m unittest discover -s tests -p test_rc_ref_lookup.py
|
||||
python3 -m unittest discover -s tests -p test_generate_rc_workflow.py
|
||||
python3 -m unittest discover -s tests -p test_audit_estate.py
|
||||
python3 tests/test_all_profiles.py
|
||||
@@ -74,6 +92,8 @@ jobs:
|
||||
bash tests/test_maven_ci_cache.sh
|
||||
bash tests/test_maven_test_results.sh
|
||||
bash tests/test_maven_ci_install.sh
|
||||
bash tests/test_browser_install.sh
|
||||
bash tests/test_frontend_build_reuse.sh
|
||||
bash tests/test_snapshot_refresh.sh
|
||||
bash tests/test_maven_settings_output.sh
|
||||
# Create jobs to test the commands of your orbs.
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
name: QBit CodeQL
|
||||
'on':
|
||||
workflow_call:
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze branch source
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Select languages present in this checkout
|
||||
id: languages
|
||||
run: |
|
||||
python3 <<'PY'
|
||||
import os
|
||||
from pathlib import PurePosixPath
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
files = subprocess.check_output(['git', 'ls-files', '-z']).decode('utf-8').split('\0')
|
||||
extensions = {PurePosixPath(name).suffix.lower() for name in files if name}
|
||||
languages = []
|
||||
if extensions & {'.java', '.kt', '.kts'}:
|
||||
languages.append('java-kotlin')
|
||||
if extensions & {'.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts', '.cts'}:
|
||||
languages.append('javascript-typescript')
|
||||
if '.py' in extensions:
|
||||
languages.append('python')
|
||||
if any((name.startswith('.github/workflows/') and PurePosixPath(name).suffix in {'.yml', '.yaml'})
|
||||
or PurePosixPath(name).name in {'action.yml', 'action.yaml'} for name in files if name):
|
||||
languages.append('actions')
|
||||
if not languages:
|
||||
sys.exit('No supported source in this branch; classify placeholders before enabling QBit scanning')
|
||||
build_mode = 'autobuild' if extensions & {'.kt', '.kts'} else 'none'
|
||||
with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output:
|
||||
output.write('languages=' + ','.join(languages) + '\n')
|
||||
output.write('build-mode=' + build_mode + '\n')
|
||||
PY
|
||||
- uses: actions/setup-java@v4
|
||||
if: steps.languages.outputs.build-mode == 'autobuild'
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: '21'
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ steps.languages.outputs.languages }}
|
||||
build-mode: ${{ steps.languages.outputs.build-mode }}
|
||||
- name: Build Kotlin sources
|
||||
if: steps.languages.outputs.build-mode == 'autobuild'
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
- name: Analyze source
|
||||
uses: github/codeql-action/analyze@v4
|
||||
@@ -19,6 +19,14 @@ on:
|
||||
description: Finalize an existing cut from repository_dispatch or an explicit retry
|
||||
type: boolean
|
||||
default: false
|
||||
recover:
|
||||
description: Verify an already-published upload after a Circle deployment timeout
|
||||
type: boolean
|
||||
default: false
|
||||
circle_build:
|
||||
description: Original Circle publisher build number (recovery only)
|
||||
type: string
|
||||
default: ''
|
||||
merge_sha:
|
||||
type: string
|
||||
default: ''
|
||||
@@ -32,6 +40,10 @@ on:
|
||||
type: string
|
||||
default: ''
|
||||
secrets:
|
||||
CENTRAL_USERNAME:
|
||||
required: false
|
||||
CENTRAL_PASSWORD:
|
||||
required: false
|
||||
QRUN_TAG_SIGNING_KEY:
|
||||
description: SSH private key that signs release tags
|
||||
required: false
|
||||
@@ -48,7 +60,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
cut-rc:
|
||||
if: ${{ !inputs.finalize }}
|
||||
if: ${{ !inputs.finalize && !inputs.recover }}
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
@@ -123,7 +135,7 @@ jobs:
|
||||
echo "merge_sha=$merge_sha"
|
||||
echo "merged_at=$merged_at"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "Pushed $branch; CircleCI publish_release_candidate cuts the RC." >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Pushed $branch; the CircleCI publisher cuts the RC." >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Release line: $branch. Waiting without a runner for qbit-rc-published." >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Persist trusted release intent
|
||||
if: steps.merge.outputs.merge_sha != ''
|
||||
@@ -135,7 +147,7 @@ jobs:
|
||||
retention-days: 90
|
||||
|
||||
tag-and-release:
|
||||
if: inputs.finalize
|
||||
if: inputs.finalize || inputs.recover
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
@@ -147,6 +159,12 @@ jobs:
|
||||
MERGE_SHA: ${{ inputs.merge_sha }}
|
||||
BUMP_SHA: ${{ inputs.bump_sha }}
|
||||
VERSION: ${{ inputs.version }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
RECOVER: ${{ inputs.recover }}
|
||||
CIRCLE_BUILD: ${{ inputs.circle_build }}
|
||||
RECOVERY_EVENT: ${{ github.event_name }}
|
||||
CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }}
|
||||
CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }}
|
||||
SIGNING_KEY: ${{ secrets.QRUN_TAG_SIGNING_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -156,6 +174,22 @@ jobs:
|
||||
fi
|
||||
workdir=$(mktemp -d)
|
||||
trap 'rm -rf "$workdir"' EXIT
|
||||
# BEGIN RC REF LOOKUP
|
||||
read_ref() {
|
||||
local body sha
|
||||
if body=$(gh api "repos/$REPO/git/ref/$1" 2>"$workdir/ref-error"); then
|
||||
sha=$(jq -er '.object.sha' <<< "$body") || return 1
|
||||
[[ "$sha" =~ ^[0-9a-f]{40}$ ]] || return 1
|
||||
printf '%s\n' "$sha"
|
||||
elif jq -se 'length == 1 and .[0].status == "404"' >/dev/null 2>&1 <<< "$body"; then
|
||||
# gh prints HTTP error JSON on stdout even when --jq was requested.
|
||||
return 0
|
||||
else
|
||||
cat "$workdir/ref-error" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
# END RC REF LOOKUP
|
||||
# The evaluator is inline so a pinned reusable workflow cannot fetch mutable code.
|
||||
cat > "$workdir/rc_guard.py" <<'PY'
|
||||
# BEGIN RC GUARD
|
||||
@@ -164,12 +198,12 @@ jobs:
|
||||
class ReleaseFailure(ValueError):
|
||||
pass
|
||||
|
||||
def evaluate(merge_sha, status, commit):
|
||||
def evaluate(merge_sha, status, commit, recovered_version=""):
|
||||
if status.get("sha") != merge_sha:
|
||||
raise ReleaseFailure("Status response does not identify the exact release merge")
|
||||
# Circle's workflow Check Run is stable even when job statuses receive -1/-2 suffixes.
|
||||
publishers = [x for x in status.get("check_runs", [])
|
||||
if x.get("name") == "publish_release_candidate" and x.get("app", {}).get("id") == 18001]
|
||||
if x.get("name") in ("publish_release_candidate", "verify") and x.get("app", {}).get("id") == 18001]
|
||||
if not publishers:
|
||||
return ""
|
||||
publisher = max(publishers, key=lambda x: x["id"])
|
||||
@@ -177,7 +211,7 @@ jobs:
|
||||
raise ReleaseFailure("Publisher Check Run identifies a different source")
|
||||
if publisher.get("status") != "completed":
|
||||
return ""
|
||||
if publisher.get("conclusion") != "success":
|
||||
if publisher.get("conclusion") != "success" and not recovered_version:
|
||||
raise ReleaseFailure("CircleCI failed for release merge: " + str(publisher.get("conclusion")))
|
||||
message = commit.get("commit", {}).get("message", "").split("\n")[0]
|
||||
match = re.fullmatch(r"Bump version to (\d+\.\d+\.\d+-RC\.\d+) \[skip ci\]", message)
|
||||
@@ -185,6 +219,8 @@ jobs:
|
||||
return ""
|
||||
if [x.get("sha") for x in commit.get("parents", [])] != [merge_sha]:
|
||||
raise ReleaseFailure("RC bump is not the direct child of the published merge")
|
||||
if recovered_version and recovered_version != match.group(1):
|
||||
raise ReleaseFailure("Recovery receipt and bump version differ")
|
||||
return match.group(1)
|
||||
|
||||
def validate_intent(run, intent, repo, merge, branch, run_id):
|
||||
@@ -223,24 +259,168 @@ jobs:
|
||||
raise ReleaseFailure("Central POM does not match the expected artifact/version")
|
||||
if parsedate_to_datetime(modified).timestamp() <= datetime.fromisoformat(merged.replace("Z", "+00:00")).timestamp():
|
||||
raise ReleaseFailure("Central artifact predates the exact release merge")
|
||||
def validate_timeout(job, publisher, repo, merge, branch, build, merged_at):
|
||||
from datetime import datetime
|
||||
if (job.get('username') + '/' + job.get('reponame'), job.get('vcs_revision'),
|
||||
job.get('branch'), str(job.get('build_num'))) != (repo, merge, branch, build):
|
||||
raise ReleaseFailure('Timeout job does not identify the exact cut')
|
||||
workflow = job.get('workflows', {})
|
||||
if workflow.get('workflow_name') not in ('publish_release_candidate', 'verify') or workflow.get('job_name') != 'qqq-orb/mvn_publish':
|
||||
raise ReleaseFailure('Unexpected Circle publication workflow/job')
|
||||
if (publisher.get('head_sha') != merge or publisher.get('app', {}).get('id') != 18001
|
||||
or publisher.get('name') != workflow.get('workflow_name')
|
||||
or publisher.get('status') != 'completed' or publisher.get('conclusion') != 'failure'):
|
||||
raise ReleaseFailure('Recovery requires the failed exact Circle publisher')
|
||||
from urllib.parse import urlsplit
|
||||
link = urlsplit(publisher.get('details_url', ''))
|
||||
if link.scheme != 'https' or link.netloc != 'app.circleci.com' or link.path != '/workflow/' + workflow.get('workflow_id', ''):
|
||||
raise ReleaseFailure('Circle check does not identify the timeout workflow')
|
||||
if job.get('status') != 'failed' or job.get('outcome') != 'failed':
|
||||
raise ReleaseFailure('Recovery requires a terminal failed job')
|
||||
actions = [action for step in job.get('steps', []) for action in step.get('actions', [])]
|
||||
if not actions or actions[-1].get('name') != 'Publish to Sonatype Central (releases and SNAPSHOTs)':
|
||||
raise ReleaseFailure('Deployment must be the last executed action')
|
||||
deploy = actions[-1]
|
||||
if deploy.get('status') != 'timedout' or deploy.get('timedout') is not True:
|
||||
raise ReleaseFailure('Only a deployment timeout can be recovered')
|
||||
if any(action.get('status') != 'success' for action in actions[:-1]):
|
||||
raise ReleaseFailure('Every pre-publication action must have passed')
|
||||
if not any(action.get('name') == 'Run Maven Verify' for action in actions[:-1]):
|
||||
raise ReleaseFailure('Successful Maven verification is missing')
|
||||
if datetime.fromisoformat(job['start_time'].replace('Z', '+00:00')) < datetime.fromisoformat(merged_at.replace('Z', '+00:00')):
|
||||
raise ReleaseFailure('Publisher predates the trusted cut')
|
||||
if any(type(deploy.get(key)) is not int or deploy[key] < 0 for key in ('step', 'index')):
|
||||
raise ReleaseFailure('Invalid deployment log coordinates')
|
||||
return deploy['step'], deploy['index']
|
||||
|
||||
|
||||
def upload_receipt(log, artifact, version):
|
||||
# Only the successful-upload receipt and actual Central staging paths count.
|
||||
deployments = re.findall(r'Uploaded bundle successfully,[^\n]*deploymentId: ([0-9a-f-]{36})\.', log)
|
||||
if len(deployments) != 1 or not re.fullmatch(r'[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', deployments[0]):
|
||||
raise ReleaseFailure('Expected exactly one successful Central upload receipt')
|
||||
paths = re.findall(r'/target/central-staging/(com/kingsrook/qbits)/(qbit-[a-z0-9-]+)/(\d+\.\d+\.\d+-RC\.\d+)/', log)
|
||||
coordinates = {(group.replace('/', '.'), name, revision) for group, name, revision in paths}
|
||||
if not coordinates or ('com.kingsrook.qbits', artifact, version) not in coordinates:
|
||||
raise ReleaseFailure('Receipt does not include the expected root artifact')
|
||||
if any(revision != version for _, _, revision in coordinates):
|
||||
raise ReleaseFailure('Mixed candidate versions in the uploaded bundle')
|
||||
return deployments[0], sorted('pkg:maven/' + group + '/' + name + '@' + revision for group, name, revision in coordinates)
|
||||
|
||||
|
||||
def validate_deployment(deployment, deployment_id, purls):
|
||||
if deployment.get('deploymentId') != deployment_id or deployment.get('deploymentState') != 'PUBLISHED':
|
||||
raise ReleaseFailure('The original Central deployment is not PUBLISHED')
|
||||
if sorted(deployment.get('purls', [])) != purls:
|
||||
raise ReleaseFailure('Central deployment coordinates differ from the original upload')
|
||||
|
||||
|
||||
def validate_recovery_bump(commit, source, actual, version, merge):
|
||||
if [p.get('sha') for p in commit.get('parents', [])] != [merge]:
|
||||
raise ReleaseFailure('Recovery bump must be a direct child of the cut')
|
||||
files = commit.get('files', [])
|
||||
if len(files) != 1 or files[0].get('filename') != 'pom.xml' or files[0].get('status') != 'modified':
|
||||
raise ReleaseFailure('Recovery bump may change only the root POM')
|
||||
revisions = re.findall(r'<revision>([^<]+)</revision>', source)
|
||||
if len(revisions) != 1 or not re.fullmatch(re.escape(version.split('-RC.')[0]) + r'-(SNAPSHOT|RC\.\d+)', revisions[0]):
|
||||
raise ReleaseFailure('Unexpected source POM revision')
|
||||
expected = source.replace('<revision>' + revisions[0] + '</revision>', '<revision>' + version + '</revision>')
|
||||
if actual != expected:
|
||||
raise ReleaseFailure('Recovery bump changed content beyond the verified version')
|
||||
|
||||
def verify_recovery(directory, repo, merge, branch, build, version):
|
||||
import base64, json, os, pathlib, urllib.request, xml.etree.ElementTree as ET
|
||||
if not re.fullmatch(r'QRun-IO/qbit-[a-z0-9-]+', repo) or not re.fullmatch(r'[1-9][0-9]*', build):
|
||||
raise ReleaseFailure('Invalid recovery repository or Circle build')
|
||||
directory = pathlib.Path(directory)
|
||||
status = json.loads((directory / 'status.json').read_text())
|
||||
intent = json.loads((directory / 'intent.json').read_text())
|
||||
publishers = [p for p in status['check_runs'] if p.get('name') in ('publish_release_candidate', 'verify') and p.get('app', {}).get('id') == 18001]
|
||||
if status.get('sha') != merge or not publishers:
|
||||
raise ReleaseFailure('Exact publisher evidence missing')
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
raise ReleaseFailure('Unexpected evidence redirect')
|
||||
opener = urllib.request.build_opener(NoRedirect())
|
||||
def request(url, token=None, post=False):
|
||||
headers = {'Accept': 'application/json'}
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
req = urllib.request.Request(url, headers=headers, data=b'' if post else None)
|
||||
with opener.open(req, timeout=30) as response:
|
||||
data = response.read(8_000_001)
|
||||
if len(data) > 8_000_000:
|
||||
raise ReleaseFailure('Oversized recovery evidence')
|
||||
return data, response.headers
|
||||
circle = 'https://circleci.com/api/v1.1/project/github/' + repo + '/' + build
|
||||
job = json.loads(request(circle)[0])
|
||||
step, index = validate_timeout(job, max(publishers, key=lambda p: p['id']), repo, merge, branch, build, intent['merged_at'])
|
||||
log = request(circle + '/output/' + str(step) + '/' + str(index) + '?file=true')[0].decode()
|
||||
# file=true can return either a raw log or JSON-encoded output.
|
||||
try:
|
||||
value = json.loads(log)
|
||||
except json.JSONDecodeError:
|
||||
value = log
|
||||
log = value if isinstance(value, str) else ''.join(part['message'] for part in value)
|
||||
token = os.environ.get('GH_TOKEN', '')
|
||||
source = json.loads(request('https://api.github.com/repos/' + repo + '/contents/pom.xml?ref=' + merge, token)[0])
|
||||
source_pom = base64.b64decode(source['content']).decode()
|
||||
pom = ET.fromstring(source_pom)
|
||||
ns = {'m': 'http://maven.apache.org/POM/4.0.0'}
|
||||
artifact = pom.findtext('m:artifactId', namespaces=ns)
|
||||
deployment_id, purls = upload_receipt(log, artifact, version)
|
||||
username, password = os.environ.get('CENTRAL_USERNAME'), os.environ.get('CENTRAL_PASSWORD')
|
||||
if not username or not password:
|
||||
raise ReleaseFailure('Existing Central publisher credentials are required for receipt verification')
|
||||
authorization = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
deployment = json.loads(request('https://central.sonatype.com/api/v1/publisher/status?id=' + deployment_id, authorization, post=True)[0])
|
||||
validate_deployment(deployment, deployment_id, purls)
|
||||
for purl in purls:
|
||||
coordinate, revision = purl.removeprefix('pkg:maven/').split('@')
|
||||
group, name = coordinate.split('/')
|
||||
url = 'https://repo1.maven.org/maven2/' + group.replace('.', '/') + '/' + name + '/' + revision + '/' + name + '-' + revision + '.pom'
|
||||
xml, headers = request(url)
|
||||
validate_central(revision, name, xml, headers.get('Last-Modified', ''), intent['merged_at'])
|
||||
(directory / 'source-pom.xml').write_text(source_pom)
|
||||
# This receipt is written only after all remote evidence has been verified.
|
||||
(directory / 'recovery.json').write_text(json.dumps({'merge_sha': merge, 'version': version,
|
||||
'circle_build': build, 'deployment_id': deployment_id, 'purls': purls}))
|
||||
|
||||
# END RC GUARD
|
||||
if __name__ == "__main__":
|
||||
import json, sys
|
||||
try:
|
||||
if sys.argv[1] == "lock":
|
||||
if sys.argv[1] == "recover":
|
||||
verify_recovery(*sys.argv[2:])
|
||||
elif sys.argv[1] == "lock":
|
||||
validate_lock(sys.argv[2], sys.argv[3])
|
||||
elif sys.argv[1] == "tag":
|
||||
print(tag_action(sys.argv[2], sys.argv[3], sys.argv[4] == "true", sys.argv[5]))
|
||||
elif sys.argv[1] == "central":
|
||||
validate_central(*sys.argv[2:4], open(sys.argv[4]).read(), *sys.argv[5:7])
|
||||
else:
|
||||
print(evaluate(sys.argv[1], json.load(open(sys.argv[2])), json.load(open(sys.argv[3]))))
|
||||
recovered = ""
|
||||
if len(sys.argv) == 5:
|
||||
receipt = json.load(open(sys.argv[4]))
|
||||
if receipt["merge_sha"] != sys.argv[1]:
|
||||
raise ReleaseFailure("Recovery receipt source mismatch")
|
||||
recovered = receipt["version"]
|
||||
import pathlib
|
||||
directory = pathlib.Path(sys.argv[4]).parent
|
||||
validate_recovery_bump(json.load(open(sys.argv[3])), (directory / 'source-pom.xml').read_text(),
|
||||
(directory / 'bump-pom.xml').read_text(), recovered, sys.argv[1])
|
||||
print(evaluate(sys.argv[1], json.load(open(sys.argv[2])), json.load(open(sys.argv[3])), recovered))
|
||||
except (ValueError, KeyError) as error:
|
||||
sys.exit(str(error))
|
||||
PY
|
||||
|
||||
# Dispatch payload is only a hint. Bind it to immutable successful cut-run evidence.
|
||||
[[ "$MERGE_SHA" =~ ^[0-9a-f]{40}$ && "$BUMP_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$MERGE_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
if [[ "$RECOVER" == true ]]; then
|
||||
[[ "$RECOVERY_EVENT" == workflow_dispatch && "$CIRCLE_BUILD" =~ ^[1-9][0-9]*$ ]]
|
||||
else
|
||||
[[ "$BUMP_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
fi
|
||||
[[ "$BRANCH" =~ ^release/[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]
|
||||
[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-RC\.[0-9]+$ ]]
|
||||
artifact_name="qbit-rc-intent-$MERGE_SHA"
|
||||
@@ -278,13 +458,53 @@ jobs:
|
||||
PY
|
||||
MERGED_AT=$(jq -er .merged_at "$workdir/intent.json")
|
||||
lock=automation/qbit-rc-pending
|
||||
current_lock=$(gh api "repos/$REPO/git/ref/heads/$lock" -q .object.sha 2>/dev/null || true)
|
||||
current_lock=$(read_ref "heads/$lock")
|
||||
# A duplicate after completed finalization is safe only after tag/source checks below.
|
||||
[[ -z "$current_lock" || "$current_lock" == "$MERGE_SHA" ]] || { echo "::error::Different release owns pending lock"; exit 1; }
|
||||
gh api "repos/$REPO/commits/$MERGE_SHA/check-runs?filter=latest&per_page=100" \
|
||||
| jq --arg sha "$MERGE_SHA" '{sha:$sha,check_runs:.check_runs}' > "$workdir/status.json"
|
||||
recovered_args=()
|
||||
if [[ "$RECOVER" == true ]]; then
|
||||
python3 "$workdir/rc_guard.py" lock "$current_lock" "$MERGE_SHA"
|
||||
python3 "$workdir/rc_guard.py" recover "$workdir" "$REPO" "$MERGE_SHA" "$BRANCH" "$CIRCLE_BUILD" "$VERSION"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "Recovery evidence verified; dry run leaves branch, tag, release and lock unchanged." >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
head=$(gh api "repos/$REPO/branches/$BRANCH" -q .commit.sha)
|
||||
if [[ "$head" == "$MERGE_SHA" ]]; then
|
||||
key="$workdir/recovery_key"
|
||||
install -m 600 /dev/null "$key"
|
||||
printf '%s\n' "$SIGNING_KEY" > "$key"
|
||||
git init -q "$workdir/recovery-repo"
|
||||
git -C "$workdir/recovery-repo" fetch -q --depth=1 "https://x-access-token:$GH_TOKEN@github.com/$REPO.git" "$MERGE_SHA"
|
||||
git -C "$workdir/recovery-repo" checkout -q -b "$BRANCH" FETCH_HEAD
|
||||
python3 - "$workdir/recovery-repo/pom.xml" "$VERSION" <<'PY'
|
||||
import pathlib, re, sys
|
||||
path, version = pathlib.Path(sys.argv[1]), sys.argv[2]
|
||||
source = path.read_text()
|
||||
revisions = re.findall(r'<revision>([^<]+)</revision>', source)
|
||||
assert len(revisions) == 1 and re.fullmatch(re.escape(version.split('-RC.')[0]) + r'-(SNAPSHOT|RC\.\d+)', revisions[0]), 'Unexpected source revision'
|
||||
path.write_text(source.replace('<revision>' + revisions[0] + '</revision>', '<revision>' + version + '</revision>'))
|
||||
PY
|
||||
git -C "$workdir/recovery-repo" add pom.xml
|
||||
git -C "$workdir/recovery-repo" -c user.name="James Maes" -c user.email=james@kof22.com -c gpg.format=ssh -c user.signingkey="$key" commit -q -S -m "Bump version to $VERSION [skip ci]"
|
||||
python3 "$workdir/rc_guard.py" lock "$(gh api "repos/$REPO/git/ref/heads/$lock" -q .object.sha)" "$MERGE_SHA"
|
||||
# A normal fast-forward push refuses a competing release-branch change.
|
||||
git -C "$workdir/recovery-repo" push -q "https://x-access-token:$GH_TOKEN@github.com/$REPO.git" "HEAD:$BRANCH"
|
||||
BUMP_SHA=$(git -C "$workdir/recovery-repo" rev-parse HEAD)
|
||||
rm -f "$key"
|
||||
else
|
||||
BUMP_SHA=$head
|
||||
fi
|
||||
recovered_args=("$workdir/recovery.json")
|
||||
echo "Verified original Central upload after Circle timeout; no artifact was uploaded again." >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
gh api "repos/$REPO/commits/$BUMP_SHA" > "$workdir/commit.json"
|
||||
version=$(python3 "$workdir/rc_guard.py" "$MERGE_SHA" "$workdir/status.json" "$workdir/commit.json")
|
||||
if [[ "$RECOVER" == true ]]; then
|
||||
gh api "repos/$REPO/contents/pom.xml?ref=$BUMP_SHA" -q .content | base64 -d > "$workdir/bump-pom.xml"
|
||||
fi
|
||||
version=$(python3 "$workdir/rc_guard.py" "$MERGE_SHA" "$workdir/status.json" "$workdir/commit.json" "${recovered_args[@]}")
|
||||
[[ -n "$version" && "$version" == "$VERSION" ]] || { echo "::error::Publisher not successful or callback version mismatch"; exit 1; }
|
||||
bump_sha=$BUMP_SHA
|
||||
[[ "$(gh api "repos/$REPO/branches/$BRANCH" -q .commit.sha)" == "$bump_sha" ]] || { echo "::error::Release branch moved away from exact bump"; exit 1; }
|
||||
@@ -316,7 +536,7 @@ jobs:
|
||||
python3 "$workdir/rc_guard.py" central "$version" "$artifact" "$workdir/central.pom" "$last_modified" "$MERGED_AT"
|
||||
|
||||
# 3. Signed annotated tag on the exact bump; duplicate runs must not accept a different tag.
|
||||
existing=$(gh api "repos/$REPO/git/ref/tags/v$version" --jq '.object.sha' 2>/dev/null || true)
|
||||
existing=$(read_ref "tags/v$version")
|
||||
target=""; release_exists=false
|
||||
if [[ -n "$existing" ]]; then
|
||||
target=$(gh api "repos/$REPO/git/tags/$existing" --jq '.object.sha')
|
||||
|
||||
@@ -58,10 +58,21 @@ test-pipelines: pack
|
||||
bash tests/test_cli_ci_behavior.sh
|
||||
python3 tests/test_cli_ci_contract.py
|
||||
python3 tests/test_standardize_qbit.py
|
||||
python3 tests/test_qbit_hotfix_profile.py
|
||||
python3 tests/test_ci_maintenance.py
|
||||
python3 tests/test_rollout_version.py
|
||||
python3 tests/test_enforce_orb_checks.py
|
||||
python3 tests/test_enroll_consumers.py
|
||||
python3 tests/test_reconcile_enrollment.py
|
||||
python3 tests/test_discover_enrollment.py
|
||||
python3 tests/test_finish_enrollment.py
|
||||
python3 tests/test_static_analysis_exit.py
|
||||
python3 tests/test_validate_standard.py
|
||||
python3 tests/test_voyage_profile.py
|
||||
python3 tests/test_next_native.py
|
||||
python3 tests/test_auxiliary_workflows.py
|
||||
python3 tests/test_qbit_codeql.py
|
||||
python3 tests/test_repository_policy.py
|
||||
python3 tests/test_intellij_plugin.py
|
||||
python3 tests/test_workflow_lint.py
|
||||
python3 tests/test_business_e2e.py
|
||||
@@ -215,3 +226,4 @@ publish-release: check-clean check-branch lint clean dev test-all
|
||||
# an isolated temporary repository. Never publishes or uses signing credentials.
|
||||
test-publication-poms:
|
||||
python3 tests/test_maven_publication_poms.py
|
||||
python3 tests/test_effective_publication_version.py
|
||||
@@ -4,6 +4,8 @@ CircleCI orb for QQQ framework projects.
|
||||
|
||||
For centralized pipeline ownership, see [standardized manifest-driven pipelines](docs/GENERATED-PIPELINES.md).
|
||||
The initial QBit profile uses a small `.qqq-ci.yml` and a setup entry point; existing direct job callers remain supported.
|
||||
For centrally managed patch adoption and exact-version rollback, see
|
||||
[managed patch updates](docs/managed-patch-updates.md).
|
||||
|
||||
**For:** Teams using CircleCI to build, test, and publish QQQ applications
|
||||
**Status:** Stable (v0.6.x)
|
||||
@@ -87,6 +89,11 @@ real publication, not a dry run, verifies the trust relationship.
|
||||
| `hotfix/1.5.1` | `1.5.1` |
|
||||
| `feature/*` | `1.5.0-feature-abc1234-SNAPSHOT` |
|
||||
|
||||
The Maven version step verifies the effective `project.version`, not only the
|
||||
`revision` property. For older POMs with a literal project version, it uses
|
||||
Maven's version plugin to align project and reactor coordinates with the
|
||||
calculated version. A mismatch after updating fails before publication.
|
||||
|
||||
### Java Version
|
||||
|
||||
Maven jobs run on Ubuntu machine executors with Temurin JDK. Default is Java 21. To use Java 17:
|
||||
@@ -114,6 +121,21 @@ Maven jobs use machine executors with Docker pre-installed. Tests using Testcont
|
||||
|
||||
## Orb Publishing
|
||||
|
||||
### QBit CI maintenance
|
||||
|
||||
The QBit profile runs Maven verification without publishing when a single-parent
|
||||
`ci: ...` or `ci(scope): ...` commit changes only `.qqq-ci.yml`,
|
||||
`.circleci/config.yml`, `.github/workflows/codeql.yml`, or
|
||||
`.github/workflows/release-rc.yml`. This applies on
|
||||
main, develop, release, and hotfix branches. The normal test job still runs,
|
||||
including on branches normally excluded from feature tests, and reports through
|
||||
the required `verify` workflow.
|
||||
|
||||
Tags, merge commits (including release automation), application changes, other
|
||||
files, empty diffs, and unavailable history retain normal publishing behavior.
|
||||
This lets central configuration updates avoid unnecessary application versions
|
||||
without introducing per-repository maintenance workflows.
|
||||
|
||||
The orb auto-publishes via CircleCI:
|
||||
|
||||
| Branch | Action |
|
||||
|
||||
@@ -3,6 +3,18 @@
|
||||
`tools/audit_estate.py` classifies repositories from explicit mappings and observed source.
|
||||
It never changes consumers, branch rules or publication settings.
|
||||
|
||||
Source auditing reads the repository's current default branch unless
|
||||
`--live --source-branch main` (or an explicit `release/...` or `hotfix/...` branch)
|
||||
is supplied. Run separate audits for each maintained source branch: a clean
|
||||
default-branch report does not establish conformity on main or release branches.
|
||||
The selected branch is resolved to an immutable commit before any files are read.
|
||||
Reports identify both `source_branch` and the repository's `default_branch`;
|
||||
the manifest's `default_branch` must still match the repository default.
|
||||
Unavailable selected branches remain unknown without falling back to default.
|
||||
Explicit branch selection requires live lookup and cannot relabel an offline SHA.
|
||||
Required-check policy still controls which branch settings are inspected;
|
||||
`--source-branch` changes source selection, not that enforcement policy.
|
||||
|
||||
- Circle profiles include android, mcp-policy and marketing. Their manifest and canonical
|
||||
setup caller remain mandatory; a missing configuration is a finding.
|
||||
- MCP, business-platform and the organization `.github` repository use approved native
|
||||
@@ -65,3 +77,67 @@ actionlint .github/workflows/workflow-lint.yml
|
||||
Tests cover exact no-CI allowlists and drift, unknown repositories, missing configured
|
||||
profiles/native bridges, retained legacy workflows, bootstrap dependency/gate removal,
|
||||
required-check unknown states, and the lint image/tool/command/caller contract.
|
||||
|
||||
## Batch orb-version rollout
|
||||
|
||||
For a standard QBit without a manifest, start directly on the managed patch
|
||||
track using `python tools/standardize_qbit.py /checkout --orb-version 0.8
|
||||
--default-branch main`. Inspect the preview, then repeat with `--write`.
|
||||
This preserves the inferred publication and branch-filter policy while writing
|
||||
the same track to the manifest and setup caller. Use `rollout_version.py` below
|
||||
for repositories that already have a manifest. Neither command configures
|
||||
repository protection or automatically enrolls newly created repositories.
|
||||
|
||||
Upgrade existing canonical CircleCI consumers with one command:
|
||||
|
||||
```sh
|
||||
python tools/rollout_version.py --orb-version 0.7.11 /checkouts/qbit-one /checkouts/qqq-cli
|
||||
python tools/rollout_version.py --orb-version 0.7.11 --write /checkouts/qbit-one /checkouts/qqq-cli
|
||||
```
|
||||
|
||||
The first command previews the batch. The second updates only `.qqq-ci.yml` and
|
||||
`.circleci/config.yml`, after every supplied checkout passes the existing manifest
|
||||
and canonical-caller validators. It preserves publication opt-outs, release filters,
|
||||
credential context names and forwarded profile parameters. Already-current files
|
||||
are unchanged. Duplicate checkouts, custom jobs, invalid options and mismatched
|
||||
pins reject the batch before writing; resolve these in the central profile first.
|
||||
|
||||
Use isolated checkouts of the intended repository heads, inspect the resulting
|
||||
patches, then commit and push through the normal automated gates. The tool does not
|
||||
commit, push, publish packages, change GitHub rules or modify native/RC worker pins.
|
||||
A version update does not prove a publication succeeded: retain exact-head hosted
|
||||
verification and artifact evidence. The requested version must already be published.
|
||||
|
||||
```sh
|
||||
python3 -m unittest discover -s tests -p test_rollout_version.py
|
||||
```
|
||||
|
||||
## Live organization discovery
|
||||
|
||||
Use `--organization QRun-IO --live` instead of `--inventory inventory.json` to
|
||||
include newly created repositories on each audit. Discovery uses the existing
|
||||
`gh` authentication and all API pages, retaining archived entries for explicit
|
||||
skip reporting. Unknown repositories receive no automatic exemption. Duplicate,
|
||||
empty, malformed or failed discovery aborts instead of reusing a stale inventory.
|
||||
|
||||
```sh
|
||||
python tools/audit_estate.py --organization QRun-IO --live --orb-version 0.8 \
|
||||
--worker-sha <40SHA> --policy observed-checks.json --output audit.json
|
||||
```
|
||||
|
||||
The organization mode has no fixed repository-count ceiling. Supply
|
||||
`--expected-active N` for an independent count check when needed; saved-inventory
|
||||
mode preserves its default of 68. Reports identify the inventory source and
|
||||
credential visibility limit: inaccessible repositories cannot be certified.
|
||||
This is discovery and audit, not automatic enrollment or branch-rule mutation.
|
||||
|
||||
For a qualified batch of canonical orb branches, use the shared
|
||||
[branch-check enforcement command](branch-enforcement.md). It adds the observed
|
||||
CircleCI setup/verify gates while preserving other protections. Source enrollment,
|
||||
scanner transitions and scheduled reconciliation remain separate requirements.
|
||||
Scheduling and authenticated onboarding still need to be configured centrally.
|
||||
|
||||
## Batch enrollment
|
||||
|
||||
Use [the shared enrollment command](consumer-enrollment.md) to convert recognized
|
||||
legacy QBits and update existing canonical consumers in one validated batch.
|
||||
@@ -61,11 +61,23 @@ parser is pinned to PyYAML 6.0.3 and uses its safe loader.
|
||||
|
||||
| Event | Workflow | Job contract |
|
||||
| --- | --- | --- |
|
||||
| develop | publish_snapshot | existing mvn_publish, snapshot |
|
||||
| release/* | publish_release_candidate | existing mvn_publish, release_candidate |
|
||||
| vX.Y.Z tag resolving to the checkout | publish_release | existing mvn_publish, release |
|
||||
| main, feature/*, hotfix/*, other/unknown branch | test_only | existing mvn_test_only |
|
||||
| Disabled publication or unrecognized/unproven tag | test_only | full verification, no publication |
|
||||
| develop | verify | existing mvn_publish, snapshot |
|
||||
| release/* | verify | existing mvn_publish, release_candidate |
|
||||
| vX.Y.Z tag resolving to the checkout | verify | existing mvn_publish, release |
|
||||
| main, feature/*, hotfix/*, other/unknown branch | verify | existing mvn_test_only |
|
||||
| Disabled publication or unrecognized/unproven tag | verify | full verification, no publication |
|
||||
|
||||
Generated `qbit`, `core`, `frontend-core`, and `frontend-material` pipelines use
|
||||
the stable workflow name `verify`. All selected jobs, dependencies and filters
|
||||
remain intact; this adds no executor or duplicate build. Other profiles retain
|
||||
their existing workflow names and native checks.
|
||||
|
||||
When upgrading from legacy workflow names, upgrade the immutable RC worker pin
|
||||
alongside the orb so finalization recognizes both names. Confirm the CircleCI
|
||||
app's successful `verify` check on the exact migration commit before replacing
|
||||
legacy required contexts. Require `setup` and `verify` on managed branches,
|
||||
preserving other required checks and controls. Human approval is not required;
|
||||
automated checks authorize the migration.
|
||||
|
||||
The pipeline SHA must match the checkout when Circle supplies it. A tagged release
|
||||
must resolve to that checkout; missing local tags are fetched individually. A main
|
||||
@@ -99,8 +111,10 @@ The independent baseline fixture pins job sets, dependency edges, contexts,
|
||||
coverage requirements and actual verification commands across all six workflows.
|
||||
|
||||
Core additionally supports `publish.feature` and `publish.hotfix` (both true by
|
||||
default), preserving `publish*` tag and hotfix branch publication. QBit does not
|
||||
accept these keys. Snapshot, RC and GA behavior remains distinct; RC/GA/hotfix
|
||||
default), preserving `publish*` tag and hotfix branch publication. QBit supports
|
||||
both keys as explicit opt-ins (false by default). QBit feature publishing requires
|
||||
a `publish*` tag resolving to the checked-out commit; feature branch pushes remain
|
||||
test-only. Snapshot, RC and GA behavior remains distinct; RC/GA/hotfix
|
||||
publication still requires the complete source coverage gate. Cold bootstrap and
|
||||
fresh consumer Maven repositories remain uncached. Shared installer policy replaces
|
||||
local package preparation; preliminary compilation is disabled before clean
|
||||
@@ -182,7 +196,7 @@ manifest validation already ran in setup. It adds no new security scanner and
|
||||
makes no security-scanning claim. Keep docs-only disabled if those paths are build
|
||||
inputs or packaged application content.
|
||||
|
||||
The light route keeps workflow `test_only` and names its job
|
||||
The light route keeps workflow `verify` and names its job
|
||||
`qqq-orb/mvn_test_only`. Verify actual CircleCI/GitHub status contexts on the canary
|
||||
before changing any protection rule; workflow/job identity alone is not proof of
|
||||
the hosted context name. Current protections must not be removed speculatively.
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# Shared QBit source scanning
|
||||
|
||||
Generate the thin caller with `python3 src/scripts/generate_auxiliary_workflow.py qbit-codeql --ref <immutable-worker-sha>`.
|
||||
The reusable worker selects supported languages from tracked files in the actual checked-out branch. Historical Java branches therefore do not request an absent Actions database. Java, JavaScript/TypeScript, Python and Actions share one job; Kotlin selects an actual build instead of silently omitting Kotlin from no-build Java analysis.
|
||||
|
||||
The caller scans pull requests, main/develop/release/hotfix pushes and weekly, cancels superseded PR scans, and contains no scan implementation. Set `codeql_worker_sha` in the estate enforcement policy to require the exact canonical caller independently of other native worker pins. Existing default-setup repositories remain unchanged until migrated.
|
||||
|
||||
Roll out to both active default and maintained branches. Verify the worker on representative branch contents and preserve existing query coverage before replacing GitHub default setup; do not run duplicate default and advanced scanners permanently. Default setup cannot select languages independently for historical branches. A branch with no supported application or workflow source is a placeholder to classify, not a passing application scan.
|
||||
|
||||
Local tests execute the embedded language-selection step against real git indexes, including absent Actions, untracked generated files, mixed source, Kotlin and placeholders. Hosted CodeQL database creation and SARIF upload must also pass before claiming rollout completion.
|
||||
|
||||
References: [GitHub setup types](https://docs.github.com/en/code-security/concepts/code-scanning/setup-types) and [CodeQL build modes](https://docs.github.com/en/code-security/reference/code-scanning/codeql/build-options-for-compiled-languages).
|
||||
@@ -8,11 +8,16 @@ recognized contracts:
|
||||
|---|---|---|
|
||||
| `release_on_main` | boolean, default false | Main pushes select the existing release job when release publishing is enabled. |
|
||||
| `release_tag_policy` | `stable`, `v-prefix`, `none` | Stable `v1.2.3`, the existing broad `/v.*/` contract, or no tags. |
|
||||
| `test_branch_policy` | `all`, `non-release`, `non-release-no-integration`, `feature`, `non-release-no-main` | Selects one of the five audited test branch exclusions. |
|
||||
| `test_branch_policy` | `all`, `non-release`, `non-release-no-integration`, `feature`, `non-release-no-main`, `non-release-no-hotfix`, `non-release-no-main-hotfix`, `feature-no-hotfix` | Selects an audited test branch exclusion. |
|
||||
| `publish.hotfix` | boolean, default false | Runs the existing Maven hotfix publisher for nonempty `hotfix/*` branch events only when explicitly enabled. |
|
||||
|
||||
`feature` preserves the original exclusion of develop, main, release branches,
|
||||
and integration-prefixed branches; it is not a new feature-prefix-only rule.
|
||||
All policies continue to exclude the RC coordination lock branch.
|
||||
The three hotfix exclusions preserve the corresponding legacy policies while
|
||||
the single shared publisher handles hotfix events. Existing manifests do not
|
||||
enable hotfix publication implicitly; migration enables it only when the source
|
||||
configuration contains the recognized hotfix publisher and branch filter.
|
||||
No consumer-supplied regex, command, job, or arbitrary branch-publication rule
|
||||
is accepted.
|
||||
|
||||
@@ -32,10 +37,27 @@ fail migration rather than silently changing release behavior.
|
||||
|
||||
Use `tools/standardize_qbit.py --write` for the two-file migration. API callers
|
||||
must pass the inferred policy to
|
||||
`canonical_caller(version, manifest["release_tag_policy"])`; the legacy one-argument
|
||||
`canonical_caller(version, manifest["release_tag_policy"], manifest["publish"].get("feature", False))`; the legacy one-argument
|
||||
form is only for manifests without an explicit policy.
|
||||
|
||||
`publish.feature` is false by default. Opting in enables the audited `publish*`
|
||||
tag contract, with the tag required to resolve to the checked-out commit. The
|
||||
canonical setup admits those tags alongside the declared release tag policy.
|
||||
Feature branch pushes remain test-only. Migration accepts Workflows' exact legacy
|
||||
literal `feature/.*` branch selector (not a regular expression); Git forbids `*`
|
||||
in branch names, so that selector cannot publish a branch. Unknown feature
|
||||
filters require a central profile change rather than silently broadening triggers.
|
||||
|
||||
Focused tests cover the immutable inventory, unknown-filter rejection, canonical
|
||||
caller agreement, main opt-in, broad/stable tag selection, HEAD provenance,
|
||||
publication opt-out and test exclusions. The packed generator suite also exercises
|
||||
main, broad RC-shaped tags, and excluded integration branches.
|
||||
Fifteen additional main-branch fixtures exercise 195 branch/tag cases,
|
||||
checking both test and publication selection. Source and packed generator tests
|
||||
also verify hotfix opt-in, its boolean validation, and unchanged main behavior.
|
||||
CMS's original YAML is retained with its immutable source SHA: its unquoted
|
||||
stable-tag regex is invalid YAML. The fixture repairs only that quoting. Both
|
||||
audited spellings of the stable regex map to the same ASCII-digit tag policy;
|
||||
arbitrary regexes remain rejected.
|
||||
|
||||
CI-only commits on publishing branches retain Maven verification without publishing when their changes are limited to the manifest, CircleCI caller, shared scanner caller, RC caller, or schema-drift caller. The commit must have one parent and a conventional ci title. Tags, application POMs, schema resources, tests, and other application inputs retain normal publication behavior. Canonical caller enforcement separately verifies the shared workflow definition and immutable worker pin.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Application-free branch verification
|
||||
|
||||
Use `profile: repository-policy` only on a branch containing repository policies,
|
||||
documentation, workflow configuration and optional unused Maven wrappers. It uses
|
||||
the same canonical setup caller as other profiles and one small Python verification
|
||||
job. It checks tracked YAML/JSON syntax and rejects application manifests, source
|
||||
files, unknown file types and symlinks. It never builds or publishes an application.
|
||||
|
||||
The manifest accepts only `profile`, `orb_version` and `default_branch`. The latter
|
||||
remains the repository's actual default branch. A future application promotion must
|
||||
select its real build profile in the same change; adding source without doing so fails.
|
||||
|
||||
Estate policy can set `branch_profiles: {main: repository-policy}` without changing
|
||||
the default branch's application profile. Require the CircleCI `setup` and `verify`
|
||||
checks on the policy branch. This profile does not establish application release
|
||||
readiness or replace the build/release checks on source-bearing branches.
|
||||
@@ -0,0 +1,66 @@
|
||||
# Shared orb check enforcement
|
||||
|
||||
`tools/enforce_orb_checks.py` replaces repository-specific scripts for the shared
|
||||
CircleCI `setup` and `verify` branch gates. One invocation processes an explicit
|
||||
batch of current `main` and `develop` branches using existing `gh` authentication.
|
||||
|
||||
Create a targets file:
|
||||
|
||||
```json
|
||||
[
|
||||
{"repository": "QRun-IO/qbit-example", "branch": "main"},
|
||||
{"repository": "QRun-IO/qbit-example", "branch": "develop"}
|
||||
]
|
||||
```
|
||||
|
||||
Preview, then apply the same batch:
|
||||
|
||||
```sh
|
||||
python tools/enforce_orb_checks.py --targets targets.json --orb-version 0.8 --output enforcement-plan.json
|
||||
python tools/enforce_orb_checks.py --targets targets.json --orb-version 0.8 --output enforcement-result.json --apply
|
||||
```
|
||||
|
||||
Every branch must already contain the exact canonical caller for its manifest and
|
||||
have successful `setup` and `verify` checks from CircleCI app 18001 at its current
|
||||
commit. Supported profiles are QBit, Core, frontend-core, frontend-material and repository-policy.
|
||||
The complete batch is inspected before any mutation. Source/head/protection drift,
|
||||
unavailable APIs and unknown profiles fail closed. Existing repository rulesets are verified as described below.
|
||||
|
||||
The tool appends the app-bound checks, enables strict checks and administrator
|
||||
enforcement, and reads the result back. Existing required checks remain required.
|
||||
For protected branches it uses the narrow status-check/admin endpoints, preserving
|
||||
reviews, signatures, linear history, restrictions and other settings. For a
|
||||
previously unprotected branch it creates protection without adding human reviews.
|
||||
It never changes scanner configuration, repository visibility, billing, or source.
|
||||
|
||||
Output records the planned head and original protection, then verification as each
|
||||
branch completes. A batch is not transactional: if an API call fails, inspect the
|
||||
output and rerun; successful protections are retained rather than rolled back.
|
||||
An already-correct branch needs no mutation.
|
||||
|
||||
This command covers the shared orb gate pair only. Continue to run the estate
|
||||
audit for scanner/native-worker requirements and other policy. Automatic repository
|
||||
discovery, source enrollment and scheduled reconciliation remain separate work.
|
||||
|
||||
GitHub API reference: [protected branches](https://docs.github.com/en/rest/branches/branch-protection).
|
||||
|
||||
```sh
|
||||
python -m unittest discover -s tests -p test_enforce_orb_checks.py -v
|
||||
```
|
||||
|
||||
## Existing rulesets
|
||||
|
||||
When effective branch rules come from repository rulesets, the tool can verify
|
||||
existing enforcement without adding classic protection. Each contributing ruleset
|
||||
must be active for branches, have no bypass actors and require both setup and
|
||||
verify from CircleCI app18001. The current branch must also have successful gate
|
||||
runs. The tool rechecks the branch, effective rules and full ruleset records before
|
||||
reporting verified. Rule drift or an unbound/missing gate stops verification.
|
||||
|
||||
This path writes nothing. It preserves scanner requirements, conditions and the
|
||||
existing up-to-date-branch policy, including a false strict setting that avoids
|
||||
unnecessary rebuilds. Reports identify protection_mode=rulesets and the observed
|
||||
strict value. Unsupported organization-level rulesets, bypass actors and missing
|
||||
gates remain unresolved rather than being rewritten or ignored.
|
||||
|
||||
For classic protection with an empty push allowlist and admin enforcement disabled, the tool leaves the entire classic policy unchanged. After current-head gates pass, it creates a narrowly scoped active repository ruleset requiring app-bound setup and verify with no bypass actors. This enforces CI even for administrators without changing their existing branch access or locking all actors out. The ruleset retains the fleet freshness policy (strict=false); it does not weaken any existing classic policy. A same-named ruleset causes rejection rather than overwrite or duplication. Readback verifies the effective gates and unchanged classic protection; subsequent runs recognize the existing ruleset without writes.
|
||||
@@ -0,0 +1,152 @@
|
||||
# Batch consumer enrollment
|
||||
|
||||
Use one target file for canonical consumers of every supported Circle profile and
|
||||
legacy QBits whose jobs and publication filters the shared converter recognizes:
|
||||
|
||||
```json
|
||||
[
|
||||
{"checkout": "/work/qbit-example", "default_branch": "main"},
|
||||
{"checkout": "/work/qqq", "default_branch": "develop"}
|
||||
]
|
||||
```
|
||||
|
||||
The branch values must come from repository inventory. Run on clean isolated
|
||||
checkouts of the branches being migrated.
|
||||
|
||||
```sh
|
||||
python tools/enroll_consumers.py targets.json --orb-version 0.8
|
||||
python tools/enroll_consumers.py targets.json --orb-version 0.8 --write
|
||||
```
|
||||
|
||||
The command validates the entire batch before writing. Existing manifests keep
|
||||
all publication and profile options; only the managed orb version changes.
|
||||
Recognized legacy QBits receive the canonical caller and inferred publication
|
||||
policy. An unknown job, filter, profile option, duplicate target, symlink, or
|
||||
concurrent declaration edit rejects the batch. Repeating a completed enrollment
|
||||
makes no changes. Filesystem errors during the write phase can leave a partial
|
||||
batch; rerun after correcting the error. This is not a filesystem transaction.
|
||||
|
||||
Only `.circleci/config.yml` and `.qqq-ci.yml` are changed. The tool does not delete
|
||||
other workflows, infer publication from repository names, or enroll unknown
|
||||
custom pipelines. Their behavior must first be represented by a central profile.
|
||||
After committing and qualifying the exact source, apply the shared
|
||||
[branch enforcement tool](branch-enforcement.md) to supported gate profiles.
|
||||
|
||||
This command provides batch source conversion. Use the coordinator below for remote signed submissions. Local conversion alone does not prove fleet deployment or release success.
|
||||
|
||||
## Remote signed submissions
|
||||
|
||||
Supply observed current branches, without cloning each repository yourself:
|
||||
|
||||
```json
|
||||
[
|
||||
{"repository": "QRun-IO/qbit-example", "branch": "main"},
|
||||
{"repository": "QRun-IO/qqq", "branch": "develop"}
|
||||
]
|
||||
```
|
||||
|
||||
```sh
|
||||
python tools/reconcile_enrollment.py --targets remote-targets.json --orb-version 0.8 --output plan.json
|
||||
python tools/reconcile_enrollment.py --targets remote-targets.json --orb-version 0.8 --output result.json --apply
|
||||
```
|
||||
|
||||
Use existing `gh` authentication with repository write access. The coordinator
|
||||
never installs credentials or changes authentication. Every target is inspected
|
||||
at an immutable head and validated before any remote mutation. No repository
|
||||
clone, SSH key or local signing key is required. A moved target stops submission;
|
||||
refresh the plan instead of overwriting it.
|
||||
|
||||
An automation branch derives from the source commit and proposed declarations.
|
||||
The coordinator creates the branch at the observed base and uses GitHub's
|
||||
`createCommitOnBranch` API with `expectedHeadOid` for an atomic append. GitHub
|
||||
signs the commit when supported by the authenticated identity; the coordinator
|
||||
requires remote signature verification, exact parent, changed paths and
|
||||
declaration bytes before opening a PR. It never force-updates a branch.
|
||||
An interrupted branch creation, commit or PR creation can be resumed:
|
||||
an existing branch still at the planned base receives the missing commit;
|
||||
matching signed branches and PRs are reused. A mismatched branch is rejected.
|
||||
Closed existing PRs are reported as existing; they are not silently reopened.
|
||||
Reports include repository, branch, immutable head and submission URL, without
|
||||
embedding declaration contents or credentials. Keep the report if a later target
|
||||
fails; earlier valid submissions remain intact.
|
||||
|
||||
API permission, signature or commit-conflict failures stop the run. A timeout
|
||||
can occur after GitHub accepted a mutation, so inspect remote state or repeat
|
||||
the same invocation; do not create a different automation branch to retry.
|
||||
See [GitHub commit API documentation](https://docs.github.com/en/graphql/reference/commits#createcommitonbranch).
|
||||
|
||||
This handles remote target lists and signed submissions. Organization discovery is described below. Qualification-aware merging and classic branch enforcement are described below; scheduling and ruleset reconciliation remain separate work. A submitted PR is not a completed migration. Unknown profiles and
|
||||
native workflows require their own central conversion, not deletion by this tool.
|
||||
|
||||
## Organization discovery
|
||||
|
||||
The coordinator can select observed current branches directly from GitHub:
|
||||
|
||||
```sh
|
||||
python tools/reconcile_enrollment.py --organization QRun-IO --orb-version 0.8 --output organization-plan.json
|
||||
python tools/reconcile_enrollment.py --organization QRun-IO --orb-version 0.8 --output organization-result.json --apply
|
||||
```
|
||||
|
||||
Repository and branch inventories are paginated. Only existing main/develop
|
||||
branches are considered; historical release branches are excluded. Archived
|
||||
repositories and the orb's own bootstrap repository are reported separately.
|
||||
Each supported branch receives the same immutable preflight as explicit targets.
|
||||
Unknown workflows, missing entrypoints, unsupported branch layouts and unavailable
|
||||
APIs stay `unresolved` in the output. They are never counted as conforming or
|
||||
silently converted. No-CI and native-only repositories may therefore appear as
|
||||
unresolved here; use the estate auditor for their separate policy classification.
|
||||
|
||||
Discovery finishes before any submission. `--apply` processes qualified plans
|
||||
and retains unresolved entries for follow-up; one unsupported repository does
|
||||
not prevent independent valid submissions. An explicit `--targets` batch keeps
|
||||
its stricter behavior: any preflight failure stops the whole batch. By default neither mode merges PRs or changes branch protections. Use the explicit completion flags below.
|
||||
|
||||
## Qualification and enforcement
|
||||
|
||||
```sh
|
||||
python tools/reconcile_enrollment.py --targets remote-targets.json --orb-version 0.8 --output result.json --apply --merge-ready --enforce
|
||||
```
|
||||
|
||||
Both flags require `--apply` and are optional. `--merge-ready` revalidates the
|
||||
signed submission, exact parent/files/content and current base, requires successful
|
||||
CircleCI-app `setup` and `verify` checks and every advertised latest check/status,
|
||||
then requests a normal exact-head squash merge. It never uses admin override or
|
||||
force pushes. The resulting signed merge must have the qualified source tree.
|
||||
Pending/failed checks and changed bases are reported without merging. Repeated
|
||||
runs can advance the same PR once checks complete; there is no polling loop that
|
||||
holds a CI executor open.
|
||||
|
||||
`--enforce` independently checks the actual current target branch, which must
|
||||
pass its own orb gates before shared classic protection enforcement. Existing
|
||||
protections and other required checks are preserved. Supported gate profiles are
|
||||
QBit, Core, frontend-core, frontend-material and repository-policy. Other profiles
|
||||
remain explicitly unsupported for completion even when source enrollment works.
|
||||
Existing active repository rulesets with app-bound gates and no bypass actors are verified without writes. Unsupported or incomplete rulesets remain unresolved; none are rewritten or layered with classic protection. API/plan restrictions remain visible.
|
||||
|
||||
A merged source update and an enforced branch are separate reported outcomes.
|
||||
Neither proves artifact publication or deployment. Scheduling and ruleset
|
||||
reconciliation remain outstanding.
|
||||
|
||||
## Central hosted pass
|
||||
|
||||
The orb owner's CircleCI config accepts `run_enrollment_reconcile: true` on
|
||||
`main`. This selects one small Python container instead of the normal orb
|
||||
build/publication workflow. It uses the existing
|
||||
`qqq-maven-registry-credentials` context's GitHub token through `gh`.
|
||||
No consumer-specific runner configuration or local signing key is needed.
|
||||
|
||||
`tools/run_enrollment.py` verifies the owner repository and main branch, runs
|
||||
organization discovery, signed enrollment, qualified merging and enforcement,
|
||||
then exits. The pass has a 20-minute execution limit. Pending checks are reported
|
||||
for the next pass rather than polled while a paid executor waits. Incremental
|
||||
results and execution status are retained under `artifacts/enrollment`, including
|
||||
when execution fails or times out. A completed pass does not mean every repository
|
||||
conforms: unresolved and waiting entries remain explicit follow-up work.
|
||||
|
||||
Create a single daily CircleCI schedule for the owner project only after a manual
|
||||
main-branch pass verifies its hosted credentials and results. Set `branch: main`
|
||||
and `run_enrollment_reconcile: true` in the schedule parameters. Scheduling is
|
||||
an external project setting; merging this config does not itself enable a schedule.
|
||||
Keep the existing RC-finalization parameter false for these runs. A timed-out
|
||||
GitHub request may already have succeeded remotely; the next pass resumes from
|
||||
the deterministic branch and verified source rather than forcing updates.
|
||||
@@ -0,0 +1,37 @@
|
||||
# Managed patch updates
|
||||
|
||||
The canonical caller and manifest accept either an exact stable version such as
|
||||
`0.8.0` or a quoted minor track such as `"0.8"`. CircleCI resolves a minor track
|
||||
to its latest published patch. A major-only version, `latest`, `volatile`, and
|
||||
wildcards are rejected. The caller import, setup parameter, and manifest must
|
||||
still agree exactly.
|
||||
|
||||
Use a minor track after qualifying that release line. Future patch fixes then
|
||||
arrive on the repository's next pipeline without a version-bump PR or an extra
|
||||
fleet-wide build. Existing exact pins remain supported and are not changed by
|
||||
publishing this capability. YAML minor versions must be quoted to remain strings.
|
||||
|
||||
The central batch tool applies the same update to all supplied checkouts:
|
||||
|
||||
```sh
|
||||
python3 tools/rollout_version.py --orb-version 0.8 --write /path/to/repo1 /path/to/repo2
|
||||
```
|
||||
|
||||
This changes only the two canonical CI declarations. Publication policies,
|
||||
contexts, branch/tag filters, and profile options stay unchanged. Normal source
|
||||
and publication checks still run when a pipeline is triggered; this setting does
|
||||
not skip tests. GitHub reusable-worker commit pins are separate and unchanged.
|
||||
|
||||
Automatic patch adoption trades exact-version reproducibility for centrally
|
||||
managed fixes. Keep patch releases backward compatible, qualify central changes
|
||||
before production publication, and use an exact stable pin for a canary or
|
||||
rollback. For example, the same tool can move a consumer from `0.8` back to
|
||||
`0.8.0`. Historical pipeline reruns can resolve a newer patch; choose an exact
|
||||
pin when reproducing an earlier run. Setup and continuation each resolve their
|
||||
orb imports, so patch compatibility also matters when publication overlaps a run.
|
||||
|
||||
Changing minor tracks is an explicit migration. Do not use this feature to push
|
||||
breaking changes into a patch release or to bypass repository checks.
|
||||
|
||||
Reference: [CircleCI orb semantic versioning](https://circleci.com/docs/orbs/use/orb-concepts/).
|
||||
|
||||
+39
-1
@@ -26,7 +26,8 @@ https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-t
|
||||
- Finalization retrieves the unique unexpired intent artifact and verifies its
|
||||
originating successful `release-rc.yml` run, repository, trigger and attempt.
|
||||
Callback fields must match that immutable intent.
|
||||
- The exact merge SHA must have a successful `publish_release_candidate` Check Run
|
||||
- The exact merge SHA must have a successful `verify` (or legacy
|
||||
`publish_release_candidate`) Check Run
|
||||
from CircleCI app ID `18001`; unrelated job statuses cannot authorize release.
|
||||
Terminal publisher failure stops finalization immediately. Pending/absent publication cannot authorize a tag.
|
||||
The RC bump must have exactly that one parent and the release branch must still
|
||||
@@ -80,3 +81,40 @@ release-captain reconciliation; the sweep does not invent provenance.
|
||||
syntax. Full actionlint with Windows native ShellCheck hung locally and was
|
||||
stopped; Linux actionlint/ShellCheck and end-to-end credential/callback rollout
|
||||
verification remain required. The orb has separate sweep/dispatch fixture tests.
|
||||
|
||||
### Published upload timeout recovery
|
||||
|
||||
A Circle deployment step can time out after Central has accepted and published its
|
||||
bundle. Use the canonical caller's explicit manual `recover` input with the existing
|
||||
`merge_sha`, `branch`, `version` and original `circle_build`. Run with `dry_run: true`
|
||||
first to verify evidence without changing the branch, tag, release or pending lock.
|
||||
This mode never builds, uploads, cuts a new candidate, or changes the failed Circle check.
|
||||
|
||||
The pinned orb worker requires the original successful cut intent and unchanged lock,
|
||||
the exact failed Circle workflow/check, a final deployment timeout after all preceding
|
||||
actions (including Maven Verify) succeeded, exactly one successful upload receipt,
|
||||
and an authenticated Central `PUBLISHED` response matching every staged coordinate.
|
||||
It also checks public POM coordinates and publication timestamps after the cut.
|
||||
Existing organization `CENTRAL_USERNAME` and `CENTRAL_PASSWORD` secrets authenticate
|
||||
only the Central status request; Circle evidence uses its public API. Redirects are
|
||||
rejected and credentials are never included in the recovery receipt.
|
||||
|
||||
After verification, recovery creates only the missing signed version bump as a direct
|
||||
child of the original release merge, using a normal fast-forward push. An existing
|
||||
bump must pass the same parent/version checks. The existing signed-tag/prerelease
|
||||
finalizer then completes the candidate and compares lock ownership before cleanup.
|
||||
A later explicit recovery retry verifies the original receipt again. Once the lock
|
||||
is gone, the completed release needs no recovery dispatch. Ordinary callbacks and
|
||||
scheduled reconciliation retain the requirement for a successful Circle publisher.
|
||||
|
||||
Central status API: https://central.sonatype.org/publish/publish-portal-api/
|
||||
|
||||
### Auditing independent worker revisions
|
||||
|
||||
The estate audit accepts `rc_worker_sha` in a profile or repository policy. For
|
||||
example, set `profiles.qbit.rc_worker_sha` to the approved immutable RC worker
|
||||
commit and keep `--worker-sha` at the approved revision for unchanged native
|
||||
workers. The audit still compares every caller to its exact canonical template
|
||||
and rejects mutable or mismatched pins. This avoids a fleet-wide native caller
|
||||
update when only RC release behavior changes. Required-check policy is separate;
|
||||
omitting it continues to report enforcement as unknown.
|
||||
@@ -12,8 +12,9 @@ text = source.read_text(encoding="utf-8")
|
||||
start = text.index("# BEGIN PROFILE RESOURCES")
|
||||
end = text.index("# END PROFILE RESOURCES") + len("# END PROFILE RESOURCES")
|
||||
embedded = ["# Generated by scripts/pack_pipeline.py; edit source/resources, then regenerate.", "import base64", "import zlib"]
|
||||
for variable, name in (("CORE_TEMPLATE", "core"), ("MATERIAL_TEMPLATE", "frontend-material"), ("VOYAGE_TEMPLATE", "voyage"), ("NEXT_TEMPLATE", "frontend-next"), ("ALL_TEMPLATE", "all"), ("STARTER_TEMPLATE", "app-starter"), ("ANDROID_TEMPLATE", "android"), ("POLICY_TEMPLATE", "mcp-policy"), ("MARKETING_TEMPLATE", "marketing")):
|
||||
payload = base64.b64encode(zlib.compress((source.parent / "templates" / (name + ".json")).read_bytes(), level=9)).decode("ascii")
|
||||
for variable, name in (("CORE_TEMPLATE", "core"), ("MATERIAL_TEMPLATE", "frontend-material"), ("VOYAGE_TEMPLATE", "voyage"), ("NEXT_TEMPLATE", "frontend-next"), ("ALL_TEMPLATE", "all"), ("STARTER_TEMPLATE", "app-starter"), ("ANDROID_TEMPLATE", "android"), ("POLICY_TEMPLATE", "mcp-policy"), ("REPOSITORY_POLICY_TEMPLATE", "repository-policy"), ("MARKETING_TEMPLATE", "marketing")):
|
||||
resource = (source.parent / "templates" / (name + ".json")).read_text(encoding="utf-8").encode("utf-8")
|
||||
payload = base64.b64encode(zlib.compress(resource, level=9)).decode("ascii")
|
||||
embedded.append(f'{variable} = json.loads(zlib.decompress(base64.b64decode("{payload}")))')
|
||||
packed = text[:start] + "\n".join(embedded) + text[end:]
|
||||
if len(packed.encode("utf-8")) > 100_000:
|
||||
|
||||
@@ -10,6 +10,11 @@ description: >
|
||||
- Collects JUnit XML test results
|
||||
- Stores test results and screenshots
|
||||
- Saves Maven cache for future builds
|
||||
parameters:
|
||||
build_before_test:
|
||||
type: boolean
|
||||
default: true
|
||||
description: "Build the React app first; combined jobs already build it with frontend_build"
|
||||
steps:
|
||||
# Ensure no stale Chrome/Driver is shadowing the PATH (your base image had Chrome 109)
|
||||
- run:
|
||||
@@ -17,7 +22,7 @@ steps:
|
||||
command: <<include(scripts/frontend_test_cleanup.sh)>>
|
||||
|
||||
# Install current Chrome; DO NOT install chromedriver (let Selenium 4 auto-manage it)
|
||||
- browser-tools/install-chrome
|
||||
- install_chrome
|
||||
|
||||
# System libs Chrome needs in headless CI
|
||||
- run:
|
||||
@@ -52,6 +57,8 @@ steps:
|
||||
# GPG signing will happen later in mvn_jar_deploy when deploying to Maven Central.
|
||||
- run:
|
||||
name: Build app, start server, run tests
|
||||
environment:
|
||||
PARAM_BUILD_BEFORE_TEST: << parameters.build_before_test >>
|
||||
command: <<include(scripts/frontend_test_run.sh)>>
|
||||
|
||||
# Always collect and publish test results & screenshots
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
description: Install stable Chrome using only its signed repository and existing Ubuntu dependency indexes.
|
||||
steps:
|
||||
- run:
|
||||
name: Install Google Chrome
|
||||
command: <<include(scripts/install_chrome.sh)>>
|
||||
@@ -21,7 +21,7 @@ steps:
|
||||
- when:
|
||||
condition: << parameters.install_browsers >>
|
||||
steps:
|
||||
- browser-tools/install-chrome
|
||||
- install_chrome
|
||||
- browser-tools/install-chromedriver
|
||||
- when:
|
||||
condition: << parameters.compile_before_verify >>
|
||||
|
||||
@@ -27,7 +27,8 @@ steps:
|
||||
- git_full_checkout
|
||||
- mvn_manage_version
|
||||
- frontend_build
|
||||
- frontend_test
|
||||
- frontend_test:
|
||||
build_before_test: false
|
||||
- frontend_package
|
||||
- mvn_jar_deploy
|
||||
- git_operations:
|
||||
|
||||
@@ -23,4 +23,5 @@ steps:
|
||||
node_version: << parameters.node_version >>
|
||||
- git_full_checkout
|
||||
- frontend_build
|
||||
- frontend_test
|
||||
- frontend_test:
|
||||
build_before_test: false
|
||||
@@ -161,7 +161,7 @@ set_version() {
|
||||
##############################################
|
||||
## Use Maven versions plugin to set version ##
|
||||
##############################################
|
||||
mvn versions:set-property -Dproperty=revision -DnewVersion="$new_version" -DgenerateBackupPoms=false
|
||||
mvn -f "$POM_FILE" versions:set-property -Dproperty=revision -DnewVersion="$new_version" -DgenerateBackupPoms=false
|
||||
|
||||
#######################
|
||||
## Verify the change ##
|
||||
@@ -175,6 +175,26 @@ set_version() {
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_project_version() {
|
||||
local expected=$1 actual
|
||||
# A revision property can exist without being used by project.version.
|
||||
# Ask Maven for the effective coordinate, including inherited configuration.
|
||||
actual=$(mvn -f "$POM_FILE" -q -Dstyle.color=never \
|
||||
org.apache.maven.plugins:maven-help-plugin:3.5.1:evaluate \
|
||||
-Dexpression=project.version -DforceStdout)
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
mvn -f "$POM_FILE" -B versions:set -DnewVersion="$expected" -DgenerateBackupPoms=false
|
||||
actual=$(mvn -f "$POM_FILE" -q -Dstyle.color=never \
|
||||
org.apache.maven.plugins:maven-help-plugin:3.5.1:evaluate \
|
||||
-Dexpression=project.version -DforceStdout)
|
||||
fi
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "ERROR: Effective project version is $actual; expected $expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Verified effective project version: $actual"
|
||||
}
|
||||
|
||||
#############################################
|
||||
## Branch Detection and Routing Functions ##
|
||||
#############################################
|
||||
@@ -434,6 +454,9 @@ main() {
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" != "true" ]]; then
|
||||
ensure_project_version "$NEW_VERSION"
|
||||
fi
|
||||
echo "=== Version calculation complete ==="
|
||||
}
|
||||
|
||||
|
||||
@@ -11,10 +11,20 @@
|
||||
## Output: Installs required system libraries
|
||||
############################################################################
|
||||
|
||||
set -e
|
||||
set -euo pipefail
|
||||
|
||||
echo "Installing browser dependencies"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
sudo apt-get update
|
||||
ubuntu_sources=/etc/apt/sources.list
|
||||
if [[ ! -s "$ubuntu_sources" ]]; then
|
||||
ubuntu_sources=/etc/apt/sources.list.d/ubuntu.sources
|
||||
fi
|
||||
if [[ ! -s "$ubuntu_sources" ]]; then
|
||||
echo 'Cannot locate Ubuntu package sources' >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get update \
|
||||
-o "Dir::Etc::sourcelist=$ubuntu_sources" \
|
||||
-o Dir::Etc::sourceparts=- -o APT::Get::List-Cleanup=0
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
libnss3-dev libgdk-pixbuf2.0-dev libgtk-3-dev libxss-dev
|
||||
@@ -15,7 +15,9 @@ set -e
|
||||
|
||||
echo "Building app, starting server, running tests"
|
||||
echo "HTTPS=true" >> ./.env
|
||||
npm run build
|
||||
if [[ "${PARAM_BUILD_BEFORE_TEST:-true}" == true ]]; then
|
||||
npm run build
|
||||
fi
|
||||
export REACT_APP_PROXY_LOCALHOST_PORT=8001
|
||||
export PORT=3001
|
||||
npm run start &
|
||||
|
||||
@@ -23,6 +23,7 @@ ALL_TEMPLATE = json.loads((_TEMPLATE_DIR / "all.json").read_text(encoding="utf-8
|
||||
STARTER_TEMPLATE = json.loads((_TEMPLATE_DIR / "app-starter.json").read_text(encoding="utf-8"))
|
||||
ANDROID_TEMPLATE = json.loads((_TEMPLATE_DIR / "android.json").read_text(encoding="utf-8"))
|
||||
POLICY_TEMPLATE = json.loads((_TEMPLATE_DIR / "mcp-policy.json").read_text(encoding="utf-8"))
|
||||
REPOSITORY_POLICY_TEMPLATE = json.loads((_TEMPLATE_DIR / "repository-policy.json").read_text(encoding="utf-8"))
|
||||
MARKETING_TEMPLATE = json.loads((_TEMPLATE_DIR / "marketing.json").read_text(encoding="utf-8"))
|
||||
# END PROFILE RESOURCES
|
||||
|
||||
@@ -69,7 +70,7 @@ def load_manifest(path, setup_version):
|
||||
if not isinstance(data, dict):
|
||||
raise ManifestError("Manifest must be a mapping")
|
||||
profile = data.get("profile")
|
||||
if profile not in ("qbit", "core", "frontend-material", "frontend-core", "cli", "voyage", "frontend-next", "all", "app-starter", "android", "mcp-policy", "marketing"):
|
||||
if profile not in ("qbit", "core", "frontend-material", "frontend-core", "cli", "voyage", "frontend-next", "all", "app-starter", "android", "mcp-policy", "repository-policy", "marketing"):
|
||||
raise ManifestError("Unsupported profile")
|
||||
allowed = ("profile", "orb_version", "default_branch", "docs_only", "publish")
|
||||
if profile in ("qbit", "core"):
|
||||
@@ -78,7 +79,7 @@ def load_manifest(path, setup_version):
|
||||
allowed += ("release_completion", "release_on_main", "release_tag_policy", "test_branch_policy")
|
||||
elif profile == "cli":
|
||||
allowed += ("publish_context",)
|
||||
elif profile in ("voyage", "frontend-next", "app-starter", "android", "mcp-policy", "marketing"):
|
||||
elif profile in ("voyage", "frontend-next", "app-starter", "android", "mcp-policy", "repository-policy", "marketing"):
|
||||
allowed = ("profile", "orb_version", "default_branch")
|
||||
elif profile == "all":
|
||||
allowed = ("profile", "orb_version", "default_branch", "publish", "publish_context")
|
||||
@@ -88,8 +89,8 @@ def load_manifest(path, setup_version):
|
||||
allowed += ("java_version",)
|
||||
mapping(data, allowed, "manifest")
|
||||
version = data.get("orb_version")
|
||||
if not isinstance(version, str) or not re.fullmatch(r"(?:\d+\.\d+\.\d+|dev:[A-Za-z0-9_-]+)", version):
|
||||
raise ManifestError("orb_version must be an explicit stable or development version")
|
||||
if not isinstance(version, str) or not re.fullmatch(r"(?:\d+\.\d+(?:\.\d+)?|dev:[A-Za-z0-9_-]+)", version):
|
||||
raise ManifestError("orb_version must be a quoted minor track, exact stable pin, or development version")
|
||||
if version != setup_version:
|
||||
raise ManifestError("Manifest orb_version must match the setup job orb_version")
|
||||
default_branch = data.get("default_branch")
|
||||
@@ -110,6 +111,8 @@ def load_manifest(path, setup_version):
|
||||
publish_keys = ("release",)
|
||||
if profile in ("qbit", "core"):
|
||||
publish_keys += ("asciidoc",)
|
||||
if profile == "qbit":
|
||||
publish_keys += ("hotfix", "feature")
|
||||
if profile not in ("qbit", "cli", "all", "app-starter", "voyage", "frontend-next"):
|
||||
publish_keys += ("feature", "hotfix")
|
||||
publishing = mapping(data.get("publish", {}), publish_keys, "publish")
|
||||
@@ -126,7 +129,8 @@ def load_manifest(path, setup_version):
|
||||
if tag_policy not in ("stable", "v-prefix", "none"):
|
||||
raise ManifestError("Unsupported QBit release tag policy")
|
||||
test_policy = data.get("test_branch_policy", "all")
|
||||
if test_policy not in ("all", "non-release", "non-release-no-integration", "feature", "non-release-no-main"):
|
||||
if test_policy not in ("all", "non-release", "non-release-no-integration", "feature", "non-release-no-main",
|
||||
"non-release-no-hotfix", "non-release-no-hotfix-integration", "non-release-no-main-hotfix", "feature-no-hotfix"):
|
||||
raise ManifestError("Unsupported QBit test branch policy")
|
||||
return {"profile": profile, "orb_version": version, "default_branch": default_branch,
|
||||
"java_version": java, "node_version": node, "browsers": boolean(data, "browsers", True),
|
||||
@@ -136,7 +140,8 @@ def load_manifest(path, setup_version):
|
||||
"release_tag_policy": tag_policy, "test_branch_policy": test_policy,
|
||||
"docs_only": boolean(data, "docs_only", False),
|
||||
"publish_context": publish_context,
|
||||
"publish": {key: boolean(publishing, key, key != "asciidoc" and profile not in ("cli", "all"))
|
||||
"publish": {key: boolean(publishing, key, key != "asciidoc" and profile not in ("cli", "all", "repository-policy")
|
||||
and not (profile == "qbit" and key in ("hotfix", "feature")))
|
||||
for key in publish_keys}}
|
||||
|
||||
|
||||
@@ -171,9 +176,28 @@ def proven_docs_only(config, branch, tag):
|
||||
return base.decode().strip() if eligible else False
|
||||
|
||||
|
||||
def proven_ci_maintenance(branch, tag):
|
||||
"""Keep application tests, but do not publish for isolated CI declarations."""
|
||||
if tag or not (branch == "main" or branch == "develop" or branch.startswith(("release/", "hotfix/"))):
|
||||
return False
|
||||
subject = git("log", "-1", "--format=%s")
|
||||
if not subject or not re.match(rb"ci(?:\([^\r\n()]+\))?: ", subject):
|
||||
return False
|
||||
parents = git("rev-list", "--parents", "-n", "1", "HEAD")
|
||||
if not parents or not re.fullmatch(rb"[0-9a-f]{40} [0-9a-f]{40}\n", parents):
|
||||
return False
|
||||
base = parents.split()[1].decode("ascii")
|
||||
changed = git("diff", "--no-ext-diff", "--no-renames", "--name-only", "-z", base, "HEAD")
|
||||
if not changed:
|
||||
return False
|
||||
allowed = {b".qqq-ci.yml", b".circleci/config.yml", b".github/workflows/codeql.yml",
|
||||
b".github/workflows/release-rc.yml", b".github/workflows/schema-drift.yml"}
|
||||
return all(path in allowed for path in changed.rstrip(b"\0").split(b"\0"))
|
||||
|
||||
|
||||
def event_kind(branch, tag, release_on_main=False, release_tag_policy="stable"):
|
||||
if tag:
|
||||
if release_tag_policy == "none" or not re.fullmatch(r"v.*" if release_tag_policy == "v-prefix" else r"v\d+\.\d+\.\d+", tag):
|
||||
if release_tag_policy == "none" or not re.fullmatch(r"v.*" if release_tag_policy == "v-prefix" else r"v\d+\.\d+\.\d+", tag, flags=re.ASCII):
|
||||
return "test"
|
||||
tagged = git("rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}")
|
||||
if not tagged and git("fetch", "--no-tags", "origin", f"refs/tags/{tag}:refs/tags/{tag}") is not None:
|
||||
@@ -191,21 +215,36 @@ def event_kind(branch, tag, release_on_main=False, release_tag_policy="stable"):
|
||||
def qbit_template(config, branch, tag):
|
||||
"""Profile-owned template; add future profiles as reviewed renderers, not raw YAML."""
|
||||
kind = event_kind(branch, tag, config.get("release_on_main", False), config.get("release_tag_policy", "stable"))
|
||||
if tag.startswith("publish") and config["publish"].get("feature", False):
|
||||
tagged = git("rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}")
|
||||
if not tagged and git("fetch", "--no-tags", "origin", f"refs/tags/{tag}:refs/tags/{tag}") is not None:
|
||||
tagged = git("rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}")
|
||||
if tagged and tagged == git("rev-parse", "HEAD"):
|
||||
kind = "feature"
|
||||
if not tag and re.fullmatch(r"hotfix/.+", branch) and config["publish"].get("hotfix", False):
|
||||
kind = "hotfix"
|
||||
if kind != "test" and not config["publish"][kind]:
|
||||
kind = "test"
|
||||
maintenance = proven_ci_maintenance(branch, tag)
|
||||
if maintenance:
|
||||
kind = "test"
|
||||
workflow = "test_only" if kind == "test" else "publish_" + kind
|
||||
result = {"version": 2.1, "orbs": {"qqq-orb": "kingsrook/qqq-orb@" + config["orb_version"]}}
|
||||
filters = ({"branches": {"ignore": "/.*/"}, "tags": {"only": "/.*/"}} if tag else
|
||||
{"branches": {"ignore": "automation/qbit-rc-pending"}, "tags": {"ignore": "/.*/"}})
|
||||
if kind == "test" and not tag:
|
||||
if kind == "test" and not tag and not maintenance:
|
||||
ignores = {"all": None, "non-release": "/(develop|release/.*)/",
|
||||
"non-release-no-integration": "/(develop|release/.*|integration.*)/",
|
||||
"feature": "/(develop|main|release/.*|integration.*)/",
|
||||
"non-release-no-main": "/(develop|main|release/.*)/"}
|
||||
"non-release-no-main": "/(develop|main|release/.*)/",
|
||||
"non-release-no-hotfix": "/(develop|release/.*|hotfix/.*)/",
|
||||
"non-release-no-hotfix-integration": "/(develop|release/.*|hotfix/.*|integration.*)/",
|
||||
"non-release-no-main-hotfix": "/(develop|main|release/.*|hotfix/.*)/",
|
||||
"feature-no-hotfix": "/(develop|main|release/.*|hotfix/.*|integration.*)/"}
|
||||
ignored = ignores[config.get("test_branch_policy", "all")]
|
||||
if ignored:
|
||||
filters["branches"]["ignore"] = ["automation/qbit-rc-pending", ignored]
|
||||
docs_base = proven_docs_only(config, branch, tag) if kind == "test" else False
|
||||
docs_base = proven_docs_only(config, branch, tag) if kind == "test" and not maintenance else False
|
||||
if docs_base:
|
||||
# Keep the existing workflow/result identity even when Maven is not needed.
|
||||
result["jobs"] = {"docs_only": {"resource_class": "small", "docker": [{"image": "cimg/base:2026.03"}],
|
||||
@@ -304,7 +343,10 @@ def frontend_template(config, branch, tag):
|
||||
kind = "release"
|
||||
if kind != "test" and not config["publish"][kind]:
|
||||
kind = "test"
|
||||
if kind == "test" and proven_docs_only(config, branch, tag):
|
||||
maintenance = proven_ci_maintenance(branch, tag)
|
||||
if maintenance:
|
||||
kind = "test"
|
||||
if kind == "test" and not maintenance and proven_docs_only(config, branch, tag):
|
||||
result = qbit_template(config, branch, tag)
|
||||
result["workflows"]["test_only"]["jobs"][0]["docs_only"]["name"] = (
|
||||
"qqq-orb/mvn_frontend_test_only" if material else "qqq-orb/node_test_only")
|
||||
@@ -407,7 +449,13 @@ def voyage_template(config, branch, tag):
|
||||
return json.loads(json.dumps(VOYAGE_TEMPLATE).replace("__ORB_VERSION__", config["orb_version"]))
|
||||
|
||||
|
||||
def repository_policy_template(config, branch, tag):
|
||||
del config, branch, tag
|
||||
return copy.deepcopy(REPOSITORY_POLICY_TEMPLATE)
|
||||
|
||||
|
||||
PROFILE_TEMPLATES = {"qbit": qbit_template, "core": core_template,
|
||||
"repository-policy": repository_policy_template,
|
||||
"frontend-material": frontend_template, "frontend-core": frontend_template,
|
||||
"cli": cli_template, "voyage": voyage_template, "frontend-next": next_template, "all": all_template, "app-starter": starter_template, "android": android_template, "mcp-policy": policy_template, "marketing": marketing_template}
|
||||
|
||||
@@ -441,6 +489,12 @@ def main():
|
||||
if expected and (not re.fullmatch(r"[0-9a-fA-F]{40}", expected) or git("rev-parse", "HEAD") != (expected + "\n").encode()):
|
||||
raise ManifestError("Checkout does not match the pipeline commit")
|
||||
output = PROFILE_TEMPLATES[config["profile"]](config, os.environ.get("CIRCLE_BRANCH", ""), os.environ.get("CIRCLE_TAG", ""))
|
||||
if config["profile"] in ("qbit", "core", "frontend-core", "frontend-material"):
|
||||
# One whole-workflow check follows the same commit from feature to release.
|
||||
# Keep every selected job, dependency and filter; add no runner or duplicate build.
|
||||
if len(output["workflows"]) != 1:
|
||||
raise ManifestError("Stable verification requires exactly one selected workflow")
|
||||
output["workflows"] = {"verify": next(iter(output["workflows"].values()))}
|
||||
parameters = continuation_parameters(config["profile"])
|
||||
if config["profile"] == "core":
|
||||
output["parameters"] = copy.deepcopy(CORE_TEMPLATE["parameters"])
|
||||
|
||||
@@ -17,12 +17,16 @@ def canonical(worker_sha):
|
||||
"dry_run": {"description": "Plan a cut without changing release state", "type": "boolean", "default": False},
|
||||
"finalize": {"description": "Retry finalization of an existing cut; do not cut another RC", "type": "boolean", "default": False},
|
||||
}
|
||||
inputs['recover'] = {'description': 'Recover the same published RC after a deployment timeout; never reupload', 'type': 'boolean', 'default': False}
|
||||
inputs['circle_build'] = {'description': 'Original Circle publisher build number (recovery only)', 'type': 'string', 'default': ''}
|
||||
for field in ("merge_sha", "bump_sha", "branch", "version"):
|
||||
inputs[field] = {"description": "Existing cut " + field + " (finalize retry only)", "type": "string", "default": ""}
|
||||
forwarded = {
|
||||
"dry_run": "${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false }}",
|
||||
"finalize": "${{ github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_dispatch' && inputs.finalize) }}",
|
||||
}
|
||||
forwarded['recover'] = "${{ github.event_name == 'workflow_dispatch' && inputs.recover || false }}"
|
||||
forwarded['circle_build'] = "${{ github.event_name == 'workflow_dispatch' && inputs.circle_build || '' }}"
|
||||
for field in ("merge_sha", "bump_sha", "branch", "version"):
|
||||
forwarded[field] = "${{ github.event_name == 'repository_dispatch' && github.event.client_payload." + field + " || inputs." + field + " || '' }}"
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Machine executors are Ubuntu amd64. Preserve an existing browser for Maven
|
||||
# consumers; frontend_test deliberately removes its stale browser beforehand.
|
||||
if command -v google-chrome >/dev/null 2>&1; then
|
||||
google-chrome --version
|
||||
exit 0
|
||||
fi
|
||||
architecture=$(dpkg --print-architecture)
|
||||
if [[ "$architecture" != amd64 ]]; then
|
||||
echo "Google Chrome requires an amd64 machine executor" >&2
|
||||
exit 1
|
||||
fi
|
||||
curl --fail --silent --show-error --location https://dl.google.com/linux/linux_signing_key.pub \
|
||||
| sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/qqq-google-chrome.gpg
|
||||
printf '%s\n' 'deb [arch=amd64 signed-by=/usr/share/keyrings/qqq-google-chrome.gpg] https://dl.google.com/linux/chrome/deb/ stable main' \
|
||||
| sudo tee /etc/apt/sources.list.d/qqq-google-chrome.list >/dev/null
|
||||
# Ubuntu dependency indexes were refreshed by install_java. Only the newly added
|
||||
# signed Google repository needs refreshing; image PPAs are unrelated to Chrome.
|
||||
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get update \
|
||||
-o Dir::Etc::sourcelist=/etc/apt/sources.list.d/qqq-google-chrome.list \
|
||||
-o Dir::Etc::sourceparts=- -o APT::Get::List-Cleanup=0
|
||||
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y google-chrome-stable
|
||||
google-chrome --version
|
||||
@@ -20,13 +20,30 @@ sudo install -d /etc/needrestart/conf.d
|
||||
# shellcheck disable=SC2016
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
# Retry transient repository downloads within APT, including NodeSource's child process.
|
||||
# Exhausted downloads retain APT's nonzero exit status and fail this build step.
|
||||
printf '%s\n' 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-qqq-download-retries >/dev/null
|
||||
|
||||
# Ubuntu archive carries the same signed security suite; avoid the flaky HTTP mirror.
|
||||
if [[ -f /etc/apt/sources.list ]]; then
|
||||
sudo sed -i 's|http://security.ubuntu.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list
|
||||
fi
|
||||
|
||||
# Add Eclipse Temurin repository
|
||||
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get update -qq
|
||||
# Base tools come from Ubuntu, not the unrelated PPAs bundled in machine images.
|
||||
# Support both the 22.04 list and the newer deb822 Ubuntu source file. Preserve
|
||||
# other indexes for later jobs without contacting their repositories here.
|
||||
ubuntu_sources=/etc/apt/sources.list
|
||||
if [[ ! -s "$ubuntu_sources" ]]; then
|
||||
ubuntu_sources=/etc/apt/sources.list.d/ubuntu.sources
|
||||
fi
|
||||
if [[ ! -s "$ubuntu_sources" ]]; then
|
||||
echo 'Cannot locate Ubuntu package sources' >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get update -qq \
|
||||
-o "Dir::Etc::sourcelist=$ubuntu_sources" \
|
||||
-o Dir::Etc::sourceparts=- \
|
||||
-o APT::Get::List-Cleanup=0
|
||||
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y -qq wget apt-transport-https gpg maven curl
|
||||
|
||||
# Add Temurin GPG key
|
||||
@@ -35,8 +52,13 @@ wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | sudo g
|
||||
# Add repository
|
||||
echo "deb [signed-by=/usr/share/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/adoptium.list
|
||||
|
||||
# Install Java
|
||||
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get update -qq
|
||||
# Refresh only the repository just added. Re-fetching unrelated image PPAs here
|
||||
# caused otherwise successful setup to fail on Launchpad 503 responses. Keep
|
||||
# Ubuntu indexes from the first refresh so Java dependencies remain resolvable.
|
||||
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get update -qq \
|
||||
-o Dir::Etc::sourcelist=/etc/apt/sources.list.d/adoptium.list \
|
||||
-o Dir::Etc::sourceparts=- \
|
||||
-o APT::Get::List-Cleanup=0
|
||||
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y -qq "temurin-${JAVA_VERSION}-jdk"
|
||||
|
||||
# Set JAVA_HOME
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
set -eo pipefail
|
||||
|
||||
echo "Running PMD static analysis..."
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
set -eo pipefail
|
||||
|
||||
echo "Running SpotBugs static analysis..."
|
||||
|
||||
|
||||
@@ -16,14 +16,15 @@ import yaml
|
||||
# Generated by scripts/pack_pipeline.py; edit source/resources, then regenerate.
|
||||
import base64
|
||||
import zlib
|
||||
CORE_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNrtPdl22ziy7/0VuOrM2M6IpPclifvcxHbfeCar48z0nChXByJBiW1uJig7Oonn26cKACmKIiVSluUlTr/IJFAo1F6FAvv7L4Q0LljEncBvPCPr+loTn4Q0oh6L4Tk8/A5P4JkX+N2g3XF8q205tOsHPHbM4XsYEQ9CBn83OkHgMuo3mskLi9m078bwzqYuZ+LxlXzb8Nm3uE1Nk4Ux9U3W7vR9y2XtfuQWgeZx5PjdIsiNRhWwvEfXt7bnCjliJnPC+Fqgf1HgG0HUyZD8/Pxcgyc48Awg8CgIzgz18H9X9W19r5HONAPPo76Vmc1jGsWaYJuGbNPiiJosiyCPWYgTvqgHCKbHzLOgH6fYknS4eB/1/QwE9dAHYUEkP0QMJIeR4NJnFnmLKxMemGcsJlmRaY7OVpgjAMcmvG8FhIax1oVZ/dCiMSPa+Tn5619HXzk+7M91iTYQr7nYHIxq+QBTgSTaBfE580GQCecpiHQqDPDIztYWMWIvFITNUCuDcQJWzFYraYyIH/s4mGjBNBhGGDHbdbq9WHeDLszts+ck7jEfIYcgILFNlti3MIhi8vHjx/bb9+/+73379OTlwVH75PO7/b+ct1r+Emk1nhwcnxy8OWr/6/3JP35/8/5f7b+/f9U+Pmw1yG+/4etXLz+9bh+9+2er0fIZaBvCZ2YvIEuSIYg1CBNJ8SF9n15Qx6UdF1AyAz92/D4wMXK6jk9d8pZeMJ8IweD6Usu3nZbfyLDwKv19NYvQvO9wFl3Aer47AII4nPwZdJY4AXyY66ZyZDvf4n7ExqSnQ82zbhT0hQAhVculK2Y8JpqPRPpeQOJn2hUQ8ccPwr45MVlt+YLdmgaEEhhqzL/Yl8Q/OG4Wc6FZAJeEg7gX+BtEN53IdJnpGEJyMoKih4NSmqpfX0eMEI+DsKZqV+fI747v8F5NLb4EUcbJ1L2kg0k6PhMX4qBv9qaqGFJFTBXifhD0XYv4QUy40xWCLLZiBiBVZhxEQpaDiNA4Zl6IFoU8WebsnKyR9dWV58QKUHUEuv9DNHvq6iAmYNX1P3ngIxKdiNEzhMBdxkKy1vKtwGdqP2bPC8BqnBD6t+iPqeanYEsePWMEjdhwHAFz79jUjDmBpS1UZ9wi2FRJ8lo7GNomk8Z1JhYbnQyWyl8Sz+Ec7NDzRLZQ3CLec0IY4VGw0WCXwigA2zOb0QFhiFg7pcm4uIc07qFATtlcXpQtICbYxVhGTI2SSTV0GSyu7XTBtmkh2DLaZZpyUPPS53+g/CnYifMT+KPlB+H3LWZNUNhfyWkPLCDp9B0QPo+aPcdnnKCjtxxu0gimE2qjkwWZQeutt/xfyWeY8hnCrri/hINhFhh5z4miIGqSy54D+gzBIKgCjSIH4OFcVFQAxpnZj5x4AA7XiZmudIbDG80hS/xHL47DZ4aRDNP7YhkdcDbkTzGCwxC1bsGI7hIxWGwaEE8YPOhHJuO66/BYLZaJEcQwnzELpBxDKgMZplstP/Haf+HCNS898SN89V2NuyL7YOTcVuP5EvkhY4eYsTJwxt6ehkJoOjo+IL8ZFrsw/L7r1vYMIEmceqHLNKmXfF6CdCBNJ/n7Zx9ss7AqEPFBxDK7N/DOLCciWkhw7wpr0Po/YRkDqAORnvFnH1ZLtiJNtlwVLXb5NA4aZTsREkGgaJwefTrVnurfPHfCLBsCIU7tolmJRwCD+gWtKTgwOQj81dehvQSLGY68q7gzAyejmZNuYhZjhxxpj/M8Z+8q4nMbCAiFYJFGw9B1TGGjNIHUDZn+ckw6UXDJEZM00Vs4CmBcQ5cOtEvHgj/5wteX62qYEESBu/j1bUYx4gcELliEbjETZSwcGcz6M8KggaR22SRUCo1ziJUWHidrsAvHYv5o4F5QepF76IGjzWGerzCMrZ4SZC5mX2JNEqwJWmL02gmDSBeS9et7AhGP5ShkvHhBhpTRBTEg35XeAOaJgH6ADiFv9kneopNSI0PGlZ7klJDklGLUJ1hg98tF6EmKZ4HDgFSg8twaBLqWU1HS2o6D9mUQnfFwNMdMRCcI4jSMzgt2c1zPRqVQvSjGvjEy7GsVRUvrYRCGZmphjHfawN0zFrUxugoij+b0TsW1o0rneCDWuDkZNGrr66ubzyDojJifuqShjoF9EoFk23Qpx8UbLnKv0ZxUbysN/oekS+qAhnrV/pNe0Oz7gqJdOse78Nsicp+povdPFjn2gBxiihYRBwL5pFIktF+SlGRJWq7glgICY+dYPjrp++Ql6LOHyPkWOaGdjhO//VgPN6AS0V4RjUvlSgs18FzjLI6x+ioiRy2U0SMIFNGoR7RD+KVnFtH5mRPuizo3MbEMTi4EDee5ZXbeB3OQ7FDlEdmdEsjt3jh+/9uELSdVKY20Wi9eLH34N2bZUeARVFLX6RDHE0H2B/hTvQEC6CyOIIlR745c5oEqnMKjFtiyxMLui0nL0qABfcqC61ZjRVpwtRMw3zBJcbPVaIJJTNgJQ59JY4nowQrJWgaxWw0RpMPe9LRMrsPSep413+U6VwfDZ8enyFaslRJR4IV8E4Bn9qWDXeJsGVdd0WETaO6WV+Rwj8agerjfLyblUiXED+GHMHW1Hd8CjYVdYWiM72Aj6C3wJ4KDN8hWfLqP6SJY+cD3wdTzl5hRvxIIn0gOtxpf5bpgXRhQ32X+skJBzF5rIi2STT7Ly4fI11TVJdmwgqSgfFn9KjAGpFCKQ2YJbDl5B96jAvBLCt41mdjyP/z7ptOYCbJVK2cWXqvdAdZidT9sSzuedQ/sGzP7gF2yMprWETO8SONeI3TDySTdGIHBPnANDaXLIJJBBnaB1Hj+xysYi6IIBbk1JN0wdhJF7WkTVZDfTmLISZXw2Y6kADCR/MzQQZgzsM1Y+B0kJxwQAbDZaFCyfaLJPCFZX9MwjsIwN605pjM1JXPzdxMqXhknQlmUNs/izXB/adpQudKTFFpKiWVkqC3K7pmo2gxrzKuCf27OjayRRvOy4I3+BA8LwZ2Ur/MUzxMLalPwdJhnqJqUfFYLk+eZ9OEul95mrbpl9zdBw2UQp+VrEoVKHknFSwarvFlPiXuzeVjpxisnY6UyXykfu9sVyusVBcfU4z4UxSaFPJny5iKCnRFqFAY+YyRJ3vqBxdQJ/zSa32A+/EqcgdlYqUBVJBYL8fjMN/EEK6mHSdrOJds8jEDSMDjbX9Xhv0QoMgUq7RAj7lPQK67yzYTs84skZBQJBBXbCx1fNAZQyE8c6hKL8l4nwOpk+Y67DuSnLhhZkpzMwZNevyNO5D5CDq8dvxfuDQIzH08kNU+B11LwOgLJVpnGh7R8HKMdTBlGEgEBY22xGOI+srazu8k22a61vmdumXvM3jU3V5m5aa9vm3TLtOyt3c7a3s567eCUSaWXweUYhc/YoNAEpzj7oaddrGlV0dO+f1edOn0P3OFkMhiJorqBeSaDmga5urqOla+gPBNFiNAuHv3HshEo1bMJkmVaU2XiVwIJPxAE3KjojJDAlZoAGh/evyWXII6JgAvzsMSTLjLNdUBuwQ6DH45Bz/VhoKCZZGLVJIQXaTEE/pC1hpUWiPi3GN7gI8SojX9jVSEAAmEZ4AWWL1Rf5m+bqPovjOwjTK9V7o5TQY36frwMs5NCQKvxQdJ4SNqPHz8SNR9khPpdmaUjDpeREzOJhAAHUZQLQQYCbOaxKTFEefxWYDdAG5BfYjqgZy7rUnOghYxFoHMhl41VNvwV2LagsKYJJSH/MXSY1fIPjvfR2hMEITtYNMygsFTx0nVViQ2irb5/7L8CPiGj3+JbwUSJGRAAGGf1TdHuIBj7HDuLsIWjqyI2kJmQXNLIR7Ormk/Sv3WBh1wLEYHVJJiWn2Y/PDINnGQkBWBuFImhrPKLyYZebZKBezqV0grvsIlJUCJ1OwMyIsmihw42KgR8pF8D9lHD62RYOcHxVPBN3bArC6KCk0vaIaK//xTd1dKou6pdKoKcfYJVFW72No1o1WBbSvuN2Vwsj6tzO4vGtElen55+EPUfddhFMiFghWLxK2yZxSBcyqHDRSCKQjUes6JuLImI/ynkSUupLH98BU4ay/WJAgqxVQ2ezYw4g9AHIy2hgQkaIJheT55rRVGQX2b2glTTzjtObCj7nRfb+aJRTfNwVBIp6QWR0lQIdlGOm2xQ+5A56Eb+8WZKhSyU7PHGXIt2KLRvqNexEgmpJqRAbaKxfgDhRciwmjBn5qijH8+xLJeBgwCdF0g28Wk/dlwnHqhHPEecCpWyfNFEOIzC9SaXWUaPQeRaNAz1T+KXJOuhw0PhR9PdoehPqc+MLZdimNv7dfD7dM4/S2jXxW2qPBa3ys2pfFGZaTd7QFKRMzPVoyZ0seR6VlS1AVgfsRmKFB7s08H8f9Hlils8eL8jhYaU9AupM4hgBXCChcWa6qKUPNgXqMjny69frlRzCfMkQE2fSVotLFmrkFdZN7G5dwDuUG3tVCQ2h4lC6ui2ju13wacQ0LQdZgmKJ8nOoRPrGXBvETmEdnwK79KadI+68Xv/d4AEUAtaEebrq5NmhCDuoZS6wQCPzokHaHEC8VsPAnVMmxbbiiAN3WgnQqHrSHsQpClsEs/xHQ8Cf2xGwIPwWZxYIZtl8t8k2ytNefhe9A9WLD4HmbxiRhKSZdbWVpKWifK2BkUmQ+1+rMMhAqHZxzh8WfYzyI4FId+Y4gPH8u+GjQMrIy0GCOm3/YS6splALnoFsgVvr4hUG/jZhBQd5B+z9+9qwlWuYyG/rC2lXWK1v09Whc3Ij2J4+2A4pqm2XbFVoThgmOS5TFV3enRco44roQt/QEXyxBCqnLbLfLDGKMKq31QIZKbnNOPARPWRuHhB5u3g08c3ear4QRt4Ffbjdux4DNkG622sevVyoVQYUcH3nyx7Z3jvTdxwSQ6X09bYoS/7Q/zDkiLine truncated
|
||||
CORE_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNrtPWt3m0iy3+dX9NVk13ZWgN+PJJ5zE9tz4908HWd39kS5Oi1oJMYIMI3s6CTe375V3Q1CCBDIsvyIM19k6K6urndVVzPffyGkccFC7vhe4xlZ19ea+CSgIe2zCJ7Dw+/wBJ71fa/rtzuOZ7Uth3Y9n0eOOXoPI6JhwODvRsf3XUa9RjN+YTGbDtwI3tnU5Uw8vpJvGx77FrWpabIgop7J2p2BZ7msPQjdPNA8Ch2vmwe50agClvfo+tb2XCGHzGROEF0L9C8KfMMPOymSn5+fa/AEB54BBB76/pmhHv7vqr6t7zWSmabf71PPSs3mEQ0jTbBNQ7ZpUUhNlkaQRyzACV/UAwTTY+aZP4gSbEkyXLwPB14KgnrogbAgkh9CBpLDiH/pMYu8xZUJ980zFpG0yDTHZyvMEYBjEz6wfEKDSOvCrEFg0YgR7fyc/PWv468cD/bnukQbitdcbA5GtTyAqUAS7YJ4nHkgyITzBEQyFQb0yc7WFjGifiAIm6JWCuMYrJitVtIYET/2cTDR/GkwjCBktut0e5Hu+l2YO2DPSdRjHkIOQEAimyyxb4EfRuTjx4/tt+/f/d/79unJy4Oj9snnd/t/OW+1vCXSajw5OD45eHPU/tf7k3/8/ub9v9p/f/+qfXzYapDffsPXr15+et0+evfPVqPlMdA2hM/Mnk+WJEMQaxAmkuBDBh69oI5LOy6gZPpe5HgDYGLodB2PuuQtvWAeEYLB9aWWZzstr5Fi4VXy+2oWoXnf4Sy8gPU8dwgEcTj50+8scQL4MNdN5Mh2vkWDkE1IT4eaZ93QHwgBQqoWS1fEeEQ0D4n0PYfEz7QrIOKPH4R9cyKy2vIEuzUNCCUw1Jh3sS+Jf3DczOdCMwcuCYZRz/c2iG46oeky0zGE5KQERQ+GhTRVv76OGSEe+UFN1a7Okd8dz+G9mlp8CaKMk6l7SYdlOj4TFyJ/YPamqhhSRUwV4n7gD1yLeH5EuNMVgiy2YvogVWbkh0KW/ZDQKGL9AC0KebLM2TlZI+urK8+J5aPqCHT/h2j21NVBTMCq639y30MkOiGjZwiBu4wFZK3lWb7H1H7MXt8Hq3FC6N/CP6aan5wt9ekZI2jERuMImHvHpmbECSxtoTrjFsGmSpLX2sHINpk0qjMx3+iksFT+kvQdzsEOPY9lC8Ut5D0ngBF9CjYa7FIQ+mB7ZjM6IAwhayc0mRT3gEY9FMgpm8uKsgXEBLsYyYipUTCphi6DxbWdLtg2LQBbRrtMUw5qXvr8D5Q/BTt2fgJ/tPwg/J7FrBKF/ZWc9sACks7AAeHrU7PneIwTdPSWw00awnRCbXSyIDNovfWW9yv5DFM+Q9gVDZZwMMwCI993wtAPm+Sy54A+QzAIqkDD0AF4OBcVFYBxZg5CJxqCw3Uipiud4fBGc8gS/9GLouCZYcTD9IFYRgecDflTjOAwRK2bM6K7RAwWmQbEEwb3B6HJuO46PFKLpWIEMcxjzAIpx5DKQIbpVsuLvfZfuHDNS0+8EF99V+OuyD4YObfVeL5EfsjYIWKsCJyxt6ehEJqOjg/Ib4bFLgxv4Lq1PQNIEqf9wGWa1Es+L0E6kKaT/P2zB7ZZWBWI+CBimd0b9M8sJyRaQHDvCmvQ+j9hGQOoA5Ge8ecAVou3Ik22XBUtdvE0DhplOyESQaBonB59OtWe6t/6bsksGwIhTu28WbFHAIP6Ba0pODA5CPzV15G9BIsZjL2ruDMDJ6OZk25iFmOHHGlP8jxj7yricxsICIVgoUaDwHVMYaM0gdQNmf5iTDqhf8kRkyTRWzgKYFwDlw61S8eCP/nC15frapgQhL67+PVtRjHiBwQuWIhuMRVlLBwZzPpTwqCBpHZZGSq5xjnASguP4jXYhWMxbzxwzym9yD30wNFmMM9WGCZWTwgyF7MvsSYx1gQtMXrtmEGkC8n69T2BiMcyFDJevCAjyuiCGJDvSm8A80RAP0SHkDX7JGvRSaGRIZNKTzJKSDJKMe4TLLD7xSL0JMEzx2FAKlB5bg0CXcupKGltR3770g/PeDCeY8ai4/tREkZnBbs5qWfjUqhe5GPfGBv2tYqiJfUwCENTtTDGO23g7hkL2xhd+WGfZvROxbXjSuf0QaxxczJo1NbXVzefQdAZMi9xSSMdA/skAsm26VKOizdc5F6jWVZvKwz+R6SL64CGetX+k17Q9Pucol0yp3/htUXkPlNF758sdOwhOcQULSQOBPJxpUhovyQpSZO0WMEtBQTGzrF8dDLwyEvQ5z4i51nkhHY6TvT2Yz3cgEpEe0U0LpUrKdTAc42zKMLqq4gctUBGjyBQRKN9oh3CLz21iM7PnGBf1LmJiWVwciFoOM8ts/MBmIN4hyqPSO+UQG73xvEG30q2HFelNNJqvXix9OHfmGWHfp+gkrpOhzh9EWR/gD/VGyCAzqIQkhj17shlfVCFU3jUAlsWW9h9MWlZGjSgT1Fw3WqsSAuudgLmGyYpbrYaTTCJMTth6DNpLBE9WCFeyyB2qyGCdNibnpTJdVhaz7Lmu1zn6mD07PgU2Yq1UiIKvJBvAvDUvnSwS5wt46orOmwCzd3yihzepxGoHu73i0m5VAnxQ/ghTF1tx7NAY2FXGBrjO9gIegv8ieDgDbIVn+5jughW3vc8MPX8JWbUrwTCJ5LDrcZXuS5YFwbUd5m3rFAQs9eaSIt4k8+y8iHyNVV1iTesICkoX1a/CowBKZTigFkCW07egfeoAPySgneNJ7a8D/++6TSmRLZq5czCa7U7wFqs7gdtacfT7oF9Y+YAsItXRtM6ZoYXadxrhG44mSQbIzDYA66hoXQZRDLIwC6QGs//eAVjkRehILdGpBvFTqKoPW2iCvLbcQxZVgmf7UgKABPJzxQdhDkD24yF32F8wgERAJuNBgXbJ5rME+L1NQ3jKAxzk5pjMlNTMjd/N6HilUkiFEVp8yzejPaXpA2VKz1xoaWQWEaK2qLsnoqqzaDGvCr4Z+bcyBpJNC8L3uhP8LAQ3EnxOk/xPDGnNgVPR3mGqknJZ7UweZ5KH+5y6W3Wqlt6fyUaLoM4LVuTyFXyUCpePFjlzXpC3JvNwwo3XjkZK5T5SvnY3a5QXq8oOKEe96EoVhbypMqbiwh2xqiRG/hMkCR+6/kWUyf802h+g/nwK3EGZmOlAlWRWCzA4zPPxBOsuB4maTuXbPMwBEnD4Gx/VYf/YqFIFai0Q4y4T0GvuMo3Y7LPL5KQUSQQVGwvcDzRGEAhP3GoSyzKex0fq5PFO+46kJ+6YGRJfDIHT3qDjjiR+wg5vHb8Xrg3CMw8PJHU+gq8loDXEUi6yjQ5pOXhGO1gyjASCwgYa4tFEPeRtZ3dTbbJdq31PXPL3GP2rrm5ysxNe33bpFumZW/tdtb2dtZrB6dMKr0MLicofMaGuSY4wdkL+trFmlYVPe37d9WpM+iDOywngxErquubZzKoaZCrq+tY+QrKUypChHbx6D+SjUCJnpVIlmlNlYlfCST8QBBwo6IzQgJXagJofHj/llyCOMYCLszDEo+7yDTXAbkFOwx+OAI910eBgmaS0qpJAC+SYgj8IWsNKy0Q8W8RvMFHiFEb/8aqgg8EwjLACyxfqL7M3zZR9V8Y6UeYXqvcHaeCGg28aBlmx4WAVuODpPGItB8/fiRqPsgI9boyS0ccLkMnYhIJAQ6iKBeCDATYzGJTYIiy+K3AboA2IL/EdEDPXNal5lALGAtB5wIuG6ts+Mu3bUFhTRNKQv5j6DCr5R0c76O1JwhCdrBomEFhqeKl66oSG0RbA+/YewV8Qka/xbeCiRIzIAAwzhqYot1BMPY5dhZhC0dXRWwgMwG5pKGHZlc1nyR/6wIPuRYiAqtJMC0vyX54aBo4yYgLwNzIE0NZ5ReTDb3aJAP3dCqlFd5hE5OgROJ2hmRMkkUPHWxUCPhYvwbso4bXSbGyxPFU8E3doCsLooKTS9ohor//FN3V0ri7ql0qgpy9xKoKN3ubRrRqsC2l/cZsLpbH1bmdRSPaJK9PTz+I+o867CKpELBCsfgVtsxiEC7l0OEiEEWhmoxZUTeWRMT/FPKkpUSWP74CJ43l+lgBhdiqBs9mSpxB6P2xllDfBA0QTK8nz7WiKMgvU3tBqmnnHScylP3Oiu180aimeTgqjpT0nEhpKgQ7L8eNN6h9SB10I/94M6FCGkr6eGOuRTsU2je037FiCakmpEBtorGBD+FFwLCaMGfmqKOfvmNZLgMHATovkGzi00HkuE40VI94hjgVKmXZoolwGLnrlZdZxo9B5Fo0CPRP4pck66HDA+FHk92h6E+pz0wsl2CY2ft18Pt0zj9LaNfFbao85rfKzal8UZlpN3tAUpEzM9WjSrpYMj0rqtoArA/ZDEWKPuzTwfx/0eWKWzx4vyOFhoT0C6kziGAFcIKFxZrqopQ82BeoyOfLr1+uVHMJ8yRATZ9JWi0sWauQV1k3sbl3AO5Qbe1UJDaHsULq6LaO7Xf+pwDQtB1mCYrHyc6hE+kpcG8ROYR2fArvkpp0j7rRe+93gARQc1oR5uur42YEP+qhlLr+EI/OSR/Q4gTitx4E6pg2LbYVQRq68U6EXNeR9CBIU9gkfcdz+hD4YzMCHoTP4sRy2SyT/ybZXmnKw/e8f7Bi/jlI+YopSYiXWVtbiVsmitsaFJkMtfuJDocQhGYf4/Bl2c8gOxaEfGOKDxzLvhs1DqyMtRggpN/2Y+rKZgK56BXIFry9IlJt4GcTUnSQf8zev6sJV5mOheyytpR2idX+PlkVNiM7iuHtg9GYptp2xVaF/IChzHOZqu706LjGHVdMF/6AiuSxIVQ5bZd5YI1RhFW/qRDIVM9pyoGJ6iNx8YLM2+Gnj2+yVPH8NvAqGETtyOkzZBust7Har5cLJcKICr7/ZLl/hvfexA2X+HA5aY0d+bI/xDLine truncated
|
||||
MATERIAL_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNq1lMtugzAQRfd8heV1TNRKlar8QXdVt02EXDMhDthGfqRBiH/vQAmPvKRU6g7PjH3PXI+pI0IoHEEEb6yjK1JjAENlwatvK7OdH2IYTY3IwWLks4+QIdflpeIZYJoqYWMlhTXObH0sjFqOB64OT/HLa/zM9lypig77m/5rE51WzaKl25uvq2CJB+fdFO/UR0swaWBxyjsPpZvRU7EDkZswFp11ZFHDWEgEx8qJVp/OoZof2Md1qViKYqyuSSfhgiJrWnKRo0OsQB/jvTN6TUnTTLTPt9NZZjNZNaNvt9CDvgTWXHUX9Kad50VBUAV0ClpIcGccFK9NcZ225QhEhCSMFZBxUbESwHaE9CEkxw/3rOyV/uwcVvnd9fswKSTKpKEA92+efgRN3oexIwfpAi+IhQyHyEmjye/A3nP5SMa57cof9LebVW693HIxexsTg+avg1kojfXLc6wUxaXmHrmvbvgLWNtQgmaE4h5bW8X6quUNmctfRdREPykCLqM=")))
|
||||
VOYAGE_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNrtW+tz2zYS/56/AqPJXJKeSUqqHzndqBNbdm1NGzsnObneVDkNREISLBKgAVCWYuv+9lvwpQepV2K1TpoPsUhggV38drEPgLl/hlBhSISknBUqqGyW9nTLDe9IeL2HZ3jrBNR1DMwcQxGp0nbo8bDdp4zMNEEj9XBPNxWCTsBUYJTLxf2KHQhBmCrEZJO9ZApBJA+ETdq2i6VmWnCxgPEpAWFDKjjz9Og5Pm+PP5xdtq/eXTf1KOM3bwSMXntZFlIRX8/8+3Ss3Sf2gAcqZYNmZob+29tbg4uORZlU2HXbN3iI57iHVLq1PUWvUC4VZggm6fNkyiWd2Ruytg3wkTYgoLggy0QRActyZtgLIf5ABO2OEfZ9l9pYgRx7iDJFeiJ8QVphcg/dBtilCuiYg2wOEuPeLL8IEu550K9nBdmQ7RLM0DCa3zhBxvU+MozAd7AihmTYl32uJDQxbiiBmewSYfiC92A58nNgkHi4FIMQoLZejEYrcJXMQuJj1dfCK20+ypKBIF0qiCGIz4VaL9KXMuxi6krc/SKGWCjaxfYG3IZ8DDo0b7BY1KMDQlMWql/Tg2mEVDuVRlJF1sgRm2DoRlIT3EooX280UIji7TsuBtLHNslKJjjXfqJgLsqjRZ73AnFHvIbQ/xC5MGzar43BWEeUsbplhEv1h1aBOu28wTa3uUlGxM7SnHJ7QESXujnjfe6ZI8/NdkiRM5Npc9alvRxqznC432+IrUz4Be0oSuasHqGP+Qp+Ntsbq7sgiS2Ikoa08azDKzjhYuYUN2cWabixqdezOliSxWAzw3FF2JEeOPrClwSNlZ66HkUSdE4VeNaBXOF+70DByLhFfaV8WbGsHlX9oAOq8PRjOHr6IAj8gklaDr9jLseONXxtlktmOSVpR+9tl7Jg1B4d7ptgQ2bvU4vBLzJGn7poPand97iD/j5KSVtsu927EpxGwDYBxkzXhByiwPAg/ERqRCY8RraKzITIVNxzoR2cTYfLZd4m3xgxbPbPtkSw40AwBRtU+JYkHsREv+LiMHvayiY94tDA251RNiPZUPO4eY2aernLwY/XgTQqqNVqMTSFHAeKL7b5lk6Psq38DkvfUBz+EZbtBjcQiDBQBA5VaT8Rgov07QbcD/zAsv1AoVgyI47UZtgbkZro4QEpEZAW26X8Jir/9LcSAlaEzIijQ4CpRmoqxI7CcB4Ca+Jx7pBdS5cAsrFs6YDtAklkQYv7dyfVyvotOgcaVgqkWJXA4Dh92c67RjVElEpn5xyQcW7qE2UhkL/T4dgYFo8Oi8ZGAfyzfI2d1BOUOLMVC/hynzCHMJtmMqsC4+1ol7cV9Yj2czBV6cBb6aggLpCAI5/6RGflEOgAHmRIlEm+WswbOBSioI9ahecXV2/PLDPE0AoxaRWSIalRxYldi0XRAMGawTsIL3EUQz3Ru3+fWhleFUsb5GyD4MBgybhFhhUr4Tw/JCMzEEIKnDSl1BgScJgzpimaR4dm8Y3s4/LBYaW8Xzp0St2j/eL+QefgqHy073RL+Mf9IiHksHNQKsJcztGP9uEhPrTtzmsHF/9R7pRKR6Rz5JTLZaQ5dmXqK/U6GWTraBi4Gh1JtN6h/ryon1/s1Rr163rt+Fft1UcUcmruEFRKB7uQ4xuQKRn+oKfryy4XHlZo3uUnYFizthM5/hDlrT0tFJ8rNo82q5y9cn+PQAy7jyaTjauN/80p60t229ZF2oJFrXHCKbLgSOORW3nhsEB4Ii54bZa0I7f8tRwjNbWyai4PHHTMsDuGOnvN8ZBxgkINV8K/yeY9Dd/MuCz8hYyrt8DXoLwd7Z4FOi56mNFPockllAskfS6VGQi3mhRDYbOtRTUzxIoPCKs+v29eXR432tdXv5xdThZIkqMHMy6goRRuhNvhnd6w1ZnSOy6xrSnhwlTS9kyHStxxiVNdmd7lbpAofhjh4epfeIustMsTDQ6KjjNQ3Vt9ahkH5BBQZCgUmVzl+X2t3qj9etZuXhyXKsXK0QSZafRWuLeCLu6JKrdHLXivtfNHNQ4lOOharKu9rhv1D/9pfzhrNOtXl9VWoWgeQKXeKqzIQqwhFhY0WxGFKeGnkteYmx88v59jOUGhAU4DfBLQk1i+pyP7tFjLCeu0x8A7GQFUUSPI/6LmpdA/KtjvAtlHUOCdU3URdNA7bA+A6yoPR+w+18nV/flFLXEkkEk9oBhul/coQ72+LbQTMoIp7fvmWePy+O2ZJjcMH7YfbBjHkMqhbEOri6e1YodorQApO19kq8vmSCw5HudrYLZnt2pYwoF20e8hKvEsJ43jy9qFRqUKzQ7Yj8t9ePuoa+MVlB7sD032T6T6hGmbeRQMEybT+TbCYmZYbCLatsCcNx0IO7ZLW2y7wVMNFLYLMbLDvc8/SuWQ/904g0q5ZBb/gKPUR05pzgn4VTBG1Dy5eote1saQMzBy+turNYkN5CSmHRE7o0r6ZHhQITDDdwPY+RUPD8hxrydIDzic8NTtnkblyc9hyVKdOYdKei5BuqpWS9J+srMzoSSVAWYbnQeldFsYWHwdaDthTfEkT+3nwSMq8MGsPK5IOxVxAcCZTNwhXRy46hFj0fsQMVQ7RRoxdEdVHzFyF8dXh/b0GfEfH5harP72+PysfVo/P2teV5+/jOcDvrSrAwpUML4+cN/YV03P3VH5JyjThxYLXBckvblFhkAvzFMibUF9MGczWjayLEQ8X41fJOelr7RgYSQxPoUrmRVykhsWfM1ki5CGUP7Ck/Um5w/VF/f3FMrhETJ1vXAaSixRcTJ5sQUmD8gOYE7nDTK65VdRLEijQX3GBCoos9gtsIgnPGs0rhoVyDIDSIgZV+GlifAg4ZwzNz01jIGcDZUSkXrwEno+AKAjMLP7SEfi9EoKjKp+ffH+JDbBNzNXVP9qAAr1K+tDdF5SOzXeUZ+4wFVfzLSY7aBsl+YpIRQaFOSWDwkI8fmU+cPDElgeYOG6onPxWFpQdjqBrR2aNQjAT3pxaWmOsefG8OmFObTbhXXdBpSoDUdnsL3kyO5jBkLpzFLvUJjYiBFF2BUEO2MU+LpX7/gZkItzIEfXCoGEbFz7CJi6RqG+JbW6HrJIQkALUIeC13yjrQ2sLqHCjrMxEtGskcienq0PMeXldNCrCoo8e5xhxeYCS8nPSfR8Yf7EBdX+J0zapobdDGybSNmFrTmOJ3ZWe8BtE534EhTCs/75/DjEoGaplItmqfSk7o719ejCEpeHCOaPFohRtDuNuDUtyZLjkrmtnPZq7Rj65CXZ0PNHOMlshhr7pBpmR2mXH1VYhpa/Gg1rsRy50lfxJ8m0kY09i+2soI8+ui6/m/k0Tocb4HobLHwYF39At8Toln5QF/d3qQuOOpvb6aGhMyZ5fdqawyI759g5PlmiLPMlR5wLhvVYfqf133m1GYb5g7WGtNWyNFGG5uNCy+TZsrcNMuH8D0a+Y7jVvUf2O4fvAG5nhLmXzWmogCxDkNyboAUvMH8TtPddGSuUkX+gHov857thztxxVD5GEj19p7dLiXfgYnYL8JPe0Ltcet6VbZpgMkVGKm/Z0xu51Uv+pqGbP2ldD1qUkv91AVty+7kBcH1bPBps2c+aZ91rtjd1ZXkDZ9zG01XHqvL56UTOLZOVb6Hs+BpWvrtI/sQV/jVkBE8awr9GZvGkVfAtZyhPGfjvmc43qdalV98b76y9x1X4nJ3tNgENVfCIWC69vtm9l/oaUFtyHcHF7H/Sj7+e0YwGlPWk4HxgxY1v2u2rxknyMV+7raWYPJs8+z8FFeia")))
|
||||
NEXT_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNrNVE1v2zAMvedXCFqAtlscb70tQLGtXTEUGFYgHXqJM0OxmUSJLCmS7DYr8t9HubaXOOnHYYfdLPLp6fGR9EOHEFqAsVxJOiCn/Q89H1FmYvH4gN94Wq1WAUYwQJdczqxRahlWwc9xfD08j28vhzdX1z/imOKVTcmxUNscOp8Ibudxyux8ophJ4wUzTRoBcA9J7pSP1Q+GC1Yw2qsR1oH2jKMqgKFkDslS5a4BkYZwW3nIpXVMiLgkHOxgEFVnpUoBs87k0NtF+Ez81yZ6ekq3AJvme/OEDpPL/Wcly/xz9BYMn66JmwPhWZY7NhFADAhgFkj9aEtQorKMydRfr5C1vLOIdh8uroYX3y/jn1++vSk2EY3k1CjpQKYNqnvsayKBJhEyrHJu4PioH2qWLNkM+gur5NFJv0JH9CSSGStAbt3PCkmCcxKsyByEHkDBRM4cUn6Fe23AljBt1AISVxNhcqpMAjcuxa4hqQPrUEG3rS+i5MzHW8X5UuorO3qewf/DTp3nXKRlo3BgHE8IVqqMe6Y5iTLgPSUgfVsjqaXOSDVwJAiw7t8gA6GS5ZQ3ebfWUI52dRZcujqF1VefE69m8CjhFWU2y4CN88sXp6CFWtf3xp365uP63imznAp1d2CHK5O3t7da9tHza1iuEdM6LqXv+4wWOGzcXgJTE8MkWnIo5zd4JtFob3jYfxvSFmDTa7M5NnuCSUmx3ilkKxcWB8hr5w653z69PH4v/SabCcM1uHcHhJZel6sRGJhx68w6SAykIB1nwu6qH/f+V/9p+KsYvQ8+jt9FUb/1cRwML5rDyafu/nuvacD+2Hc2nT/snNGl")))
|
||||
ALL_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNq1T7EOgjAU3PmKpjNIdHQyJg4ukmDiYkwDUkxt4UkLEkP4dwsSUojootu7u/fu7lUWQvhOpWKQ4iVazOZ2w4AMlYaVnjXKsszRjCYwZ+lFSQDuduSKEM9fk8PG32+9HSFYn9StRwmSxwJKw0gHsfjhBGnkqAQ47RWtXaGNPHYY9YpZwQ2EIC8X47bbSYOkccRhwUTUhuRU5dhYqvu5rfglZ1xxFHMGSbE91G4SYiYmZUmzgkk6/LPTplsjdPrXD3EhxIcf3sk//sEy1QbVVm09Ad15mhY=")))
|
||||
STARTER_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNp1UMGKwkAMvc9XlDmKtejR0yJ42MsKCl5EQnUzZbbTiZ0ZqiL9d1OpWqt7y3t5eS/JRUSRrNB5TVZOo8loPGwYcjvP8MI1o7IsY2aYkLm2mXdEedKSXwCL5QzW8+Xqe/EDIHmkvnkcyeXK0LFjFNAHIGvOD4rJP7plbVocPTrd7KSoLHwab2V7Kg7aIOxQkUPgi7RqZCo1Hod9rQ14Ci+ZnbQirdDGDjPtgzvHe4e/aINmI/ki3/ZslTaBH/m2W3N3mn3iuaMzy/s2n01Gg0T2BLX4Dz3re7UVd1SLWlwByepyJQ==")))
|
||||
ANDROID_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNrtWG1v2zYQ/p5fQWgF6rSW3CXYMDhJ13YNkm5rkrrugKEKDFqiZCYUyZBUbC/wfvuO1LvtpmmBAcUwIYAp3vHuuXcqdzsIebdEaSq4N0R7wfd9uyPUVMPrHazhDfNYCRrDhhdRFTES0UG592Iv+CF45vULxjSSfsRoh7Hce7Ef7AV7HvCtnIYr0daQc2p8Q7Spt2CTLEiUG6Fae7DLcUasghJABcTPcDSjnJRQHKvBqeWMSYJzZrySsKo4PG2ItCA+NkeiGYmuRW7aYipNCvAJRfxU4ZgRPwKFbXUNxvaZnDu7dMeIAh1s+5HIMuC1MINBIXiOLOU1meap1zqxqterLdg0vn0YMAC0CaXy6XuQ4rQjsBVcpltSCvc0cLPrmCrkS/T3wFlSnhhc2VgOQp5QHqMA+WYpCUoQ0FOyQKEXPBlMc8ri7qngySJjoYd8G3QUSXS32i4YheFByNdxzWeEu6Rgc7zUn3VbxyMuqBOralJZveEgic3Myu9C+ho9WBma4OjhSgq7P6GqXF3utBRXObFRTxIriLOBYu9WlF5CIWQ+zXBK1lB5NnoWkzaK8rTj9rquhl0R+qACsP/sIBUiZWC1pPpg8dOP3gb0phq/kXLfWsbaQNx8kuUMA0Iftv2vquxIcE4iQ2JX3i8L6WMbp7WEXouJd3iImvAFbSp6/vzL8vDb6QDgeJnDAdz44f+G8C80hAyCyv2YSCaW/5UBe28WX+RTRvUMzamZIT3DisTorXUCimBJuKGY3ZfZmhhIvFwgSSXYRlnIh5C4j+5O3oxPP7yajM9/Oz4b/nxCzWk+RRc4urZ9D80VNaSlAlENZXSTU9hZhV7Iv0MvczOzxAgD55QkYC40hUxSBu31ALil0BScsEQ4iojWVgR0DFhhtewjLgySSogEwZ8s9JZqC/4g5FBmFNr1cpJroo4e9eTSzATfR4eHjy/+HJ+enz0OOc2kUAZdacHrF6HrZa4Yo9PAYoeQhMATMQxgzsSIQM1DB+t1WYLT8fiiop2CFxlRu8OQW8dCdoBdBWlS8vc0YUnfOqePEtkHF8SkjzKd9tGM4Bh6HNhK5qCkkmIfhakmaJSD/zJyrJRQvdArg4Abx8JVtlYIiwT8EIferjUD3FLKE5JwotDRmq2B60yTgtpr7N1tbEmJ6dnK7CArTm+KG5X2ht7MGKmHgwGMwiCFvMynAQR+AC3uKbLi+o00+5RuOLoLPZszQtG/nGWhZzPxFYGcVsgdFjog/JYqwT+CN1oJGnqXa0LrB4RCukhTSMNSstJxg1sel/CeuuzwVruNDFdPhW8C+9MrzewjGxGo6qP9Z7sI26zXUnBNhl39iphccZd2ARM47lV8pQ6bseBD6+HQsy8QtqB4YyKl3EXRMtLElQLVlMNo5hHpWe4+grzfRUI5ohNWrgsZrroG7yCB/Dfng5ubG7/ojFXFNNokURnV9qNIh14fZk9NyPUM2DaS8g/M8jol39qjPEWV3LI2a/QS7lLGIYYNsrCBQMfuBxSWogux792gP15Q02R63W5ajSbC3JpZ9gyohLLdtSxDiRIZVDAnIS/6QMidelfx56OTycno5evfjycXo/Nfj38ZT4oseHdz40SNSkkfALVts0fQDjudxva3LxV2AU1lLlRshXUzN+TNpcmHuxTXgpEjyTDlSBbd3Ulynb5I3bE42apkVPdUuCY8ZGrW36gA7TphYt76UHWD2F7/XJ9oD9Pyg7Y16pqP2vtvl/VF/fNXhk9M8momFrOmC+MeMNtgdIiXayMyoWzj86EkTRWU4Yxso9n/J3C2dFdGiJ+3Rl513lcbU5kbsjDbTNosYL893ruWPDTwO6udfwDDGeSz")))
|
||||
ANDROID_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNrtWG1v2zYQ/p5fQWgF6rSW3CXYMDhJ13YNkm5rkrrugKEKDFqiZCYUyZBUbC/wfvuO1LvtpmmBAcUwIYAp3vHuuXcqdzsIebdEaSq4N0R7wfd9uyPUVMPrHazhDfNYCRrDhhdRFTES0UG592Iv+CF45vULxjSSfsRoh7Hce7Ef7AV7HvCtnIYr0daQc2p8Q7Spt2CTLEiUG6Fae7DLcUasghJABcTPcDSjnJRQHKvBqeWMSYJzZrySsKo4PG2ItCA+NkeiGYmuRW7aYipNCvAJRfxU4ZgRPwKFbXUNxvaZnDu7dMeIAh1s+5HIMuC1MINBIXiOLOU1meap1zqxqterLdg0vn0YMAC0CaXy6XuQ4rQjsBVcpltSCvc0cLPrmCrkS/T3wFlSnhhc2VgOQp5QHqMA+WYpCUoQ0FOyQKEXPBlMc8ri7qngySJjoYd8G3QUSXS32i4YheFByNdxzWeEu6Rgc7zUn3VbxyMuqBOralJZveEgic3Myu9C+ho9WBma4OjhSgq7P6GqXF3utBRXObFRTxIriLOBYu9WlF5CIWQ+zXBK1lB5NnoWkzaK8rTj9rquhl0R+qACsP/sIBUiZWC1pPpg8dOP3gb0phq/kXLfWsbaQNx8kuUMA0Iftv2vquxIcE4iQ2JX3i8L6WMbp7WEXouJd3iImvAFbSp6/vzL8vDb6QDgeJnDAdz44f+G8C80hAyCyv2YSCaW/5UBe28WX+RTRvUMzamZIT3DisTorXUCimBJuKGY3ZfZmhhIvFwgSSXYRlnIh5C4j+5O3oxPP7yajM9/Oz4b/nxCzWk+RRc4urZ9D80VNaSlAlENZXSTU9hZhV7Iv0MvczOzxAgD55QkYC40hUxSBu31ALil0BScsEQ4iojWVgR0DFhhtewjLgySSogEwZ8s9JZqC/4g5FBmFNr1cpJroo4e9eTSzATfhzo5PHx88ef49PzscchpJoUy6EoLXr8IXS9zxRidBhY9BCUEnohhgHMmRgSqHnpYr8sSnI7HFxXtFPzIiNodhty6FvIDLCtIk5K/pwlL+tY9fZTIPjghJn2U6bSPZgTH0OXAWjIHJZUU+yhMNUGjHDyYkWOlhOqFXhkG3LgWLrO1Qlgk4Ik49HatGeCYUp6QhBOFjtZsDVxvmhTUXmPvbmNLSkzP1mYHWXF6U9yotDf0ZsZIPRwMYBgGKWRmPg0g9ANock+RFddvpNmndMPRXejZrBGK/uUsCz2bi68IZLVC7rDQAeG3VAn+EbzRStHQu1wTWj8gFBJGmkIalpKVjhvc8riE99Rlh7fabWS4iip8E9ifXmlmH9mIQF0f7T/bRdjmvZaCazLs6lfE5Iq7tAuYwHGv4it12JwFH1oPh559gbAFxRsTKeUuipaRJq4YqKYchjOPSM9y9xFk/i4SyhGdsHJdyHD1NXgHCeS/OR/c3Nz4RW+saqbRJonKqLafRTr0+jB9akKuZ8C2kZR/YJbXKfnWHuUpquSW1Vmjl3CbMg4xbJCFDQQ6dj+gsBRdiH3vRv3xgpom0+uG02o1EebWzLJrQCWUDa9lGUqUyKCCOQl50QdC7tS7ij8fnUxORi9f/348uRid/3r8y3hSZMG7mxsnalRK+gCobaM9gobY6TW2w32psAtoKnOhYiusm7khb65NPtymuBaMHEmGKUey6O9Okuv1ReqOxclWJaO6q8JF4SFzs/5KBWjXCRPz1qeqG8X2Auj6RHuclp+0rWHXfNbef7+sr+qfvzR8YpZXU7GYNl0Y94DZBqNDvFwbkgllGx8QJWmqoAxnZBvN/keBs6W7NEL8vDXyqvO+2pjL3JCF2WbSZgH77QHfteShgd9Z7fwDUg7law==")))
|
||||
POLICY_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNqdVU1v2zAMvedXCL442Rq3aYEegubQQwYMWLGeBgxVESg2nWqVJU9S1gZB/vtIWXbsdCuw+WKJH4+PlEjtR4wlv8A6aXQyZ5fZ7IwkP8za4XaPa9ILJQvhYVoL78HqowqVhcmfwaLkIUpYpwt6WYkNoDrJZbU5r3f+yej5VTa7TDqrQ1w9nrWgFpzZ2hxWuRKOwiWuEkolnYHzULtB0CR/gvzZbH1ndELEbnWPdxRqUQVyn7XzGIDd777f3n3pQTTQpqqELsiwljWT0Xg6LaQTa4WFkfU0VnEaeESgxeI6u8iukh7c4Zj2/xD9Fs+CxbNgpVTg3iMcKn7FOL+5Yeny66eUa1nVxnq2E5XqNm7nuC6tqQj5Sck1i4p73HLNNVhrrGML9vDIdYy+ys1We5RdkEVpbMBcESesUnAdp1k6yexGmfU4/ZCRPp3MuSaesmRC78a1sD7DklrvXmT0YARGCsLpQDOSuNadvtxoL/UWKD7tvd31tATHTA163EFMmHCs7NnQhxUVmAUZZU6UsFJGFONy0qJGsmnbAOcp0XLe9nBPIC383EoLBRUslUV6xlIvvQJa5HiAG2N3tC7A5VbWHq9OUGE+oH36OESrpHNSbwisDKUpiUAXA7mVTJtQLMrlxBvVEeCEJH3NuWaixjJhzjzZdzkd5uwuBi4lqMLhtYxAB55MhljDK/FxwWZt9eA1h7q5bhl1xZIi0jFAj877NMiNroODxhCJwJFCDLAMPyzkv2Evh4B4ua3UnszaZivYfpDdYdh90Q2r3ASKkRsYnnCul03vlOhczDvadI6hTwZuR1dkwNh0kCl2aQav0o9nKADl4CTWLY6l9pKyMLeDK7Y913+ZQu3wHbW7MJeSF2OfS2Ve/vAO9Md/fCl6c/jta/E2wugw+g3AlOQ8")))
|
||||
REPOSITORY_POLICY_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNqdVltv2zYUfs+vIPwiGYjotgGGIa0fis7YAnRO0HgtgigwaOnIZkKRGkkl9gL/951DXSK7aDNML+K56ON3bqSeTxgbPYJ10ujROXvH356S5t6sHIrPuCa7UDIXHpLKKJntegOacpM9gEXNbathvS3YZSnWgOZRJsv1pNr5jdHnZ/ztu1HvtW9Xd6cdqAVnapvBMlPCEY+RK4VSo97BeajcwaajbAPZg6l973RExNZ6wLtValEGchfaedyA+Q2wm49/fmaVsA7DOj10z0xZCp3TF00kZywpWSUrJluAJMmlEyuFqZJV0uY1CdzY1Y6gp9Nf+Bt+Nhog719S8X/If22rE9iLqsISCU/bFhaArazQ2eY/RJKmHz5EVzeLPy7nUaplWRnr2b0zOtWFNRim8BslV6y1XKHYu7l6VVmTgXO9alsqDh4Z8JmCErRfEBvh2GzR++xEqVLEJ2jHpgMYHlK2xHpWtY9vo7X00SmLlEsKqcDROvknuhvzHDKTQxzVvkh+jcbcYfg+jtL0TTR+Ab4ltjElbMwKYxmtsGasscsiKO6QF66Mb9TnqaZUWSEdsOsdtlw52xL23DBvBbZ9zpp5YA0n3I9gaL/n6PPFp9n8ekZE55cLFGj16fLq5mL+e7P8bXb5bT77ck0Sx/jkWhsLnSS8t3JVe4q1IRKeiEMuvbGZ0YVck3P5qJ+6N8/KPNqn2tVFIbctE446ArUupJD7bfMWOLphsStV8xbtgooeFliMCqyXSAJRsWFkAc43sJUp+bb5oMJk4JT332XCrg33prXuEOYeMt9r0Fia/CCuENuqlirnaytyFfIwlPmDD1V34L3UazdwO1IFT6RLdaZC9nVuC4o1JpFLt3QYutQP8Zi1vpwKyJV5AotK/LCP+fyF7HcdUUQfX6aOoFpcZuHvWuJhxgQLsTDMBPUKjjLttqeWGVISblkZJ7e493SK+ceaTp4sjjTYSSkeQSetxO+FjQaUsB0wBTX8FC6MlPM7BZOmfUL9BiizBadJd9h5eh0HCAsiX3rY+hg0Dhrqp92ojV/bPgwZjRPmsRkMPHEaU9OgfaJbp65tf57rv7C9Kzo+DufvPXOgsMteyzUFgw38WnSHoRzxRa7x0dyMB6SVdD4mLXeigKUyuAteDjHt1KUN1A+wQ+HDJA0QSeaE4xoQPNmwRpSN7vDP2bMCHYrmxvujA2qCo17TGdx0aDiu3mPWh7dFmzUahHrVKSkPzZWQ6h/cWN3lfdJJ4Q4bPRn7UGBMw78IsLI4+Hlo/zIGt/jxn8b36Cf7k38B2PmvaA==")))
|
||||
MARKETING_TEMPLATE = json.loads(zlib.decompress(base64.b64decode("eNrtW89v4ygUvuevQD7MSKsSp5lbD6OR9rRa7W1uk8oiGMc0GFzATbNV/vd9+EeaOHaqVpt6knBpbfh4wHufee9z3ZcRQsET04YrGdyh6fj2xrUoPTdw+wLXcJcVkluloSFYqmQ6DeuGH5Pxt/HtbQCoTTlspfQyEWq1MzbXmKaMLl+boPFBleZ/1fdo27M7XShVzKJ5wUUcERlHlhm7Y6QGS5Ixt7AShwGHS9xNC+VMvW4zmE7biJzQJVmwKCMSfpV7lXnWRjnbEVUZoGK3g+DXzMGQLiQSXNpZcINeW8o1NU3PKBdkvdJ8kVrEpbFECERTrTJeZAjjFbcpjllu9m24Ge8a2H6XcKZSVRg2C+7bK6VKWvZs97xcdy1gpmIe7DXft4YnXFhw14G/oWuuiYSQdvU55kix7pi07AsTRmyh2WwWjv8IWytuMKmyCX+uIQeI+1bLZtR393q9uTnOs1jRJdORoUT284tnQAtcYlquKnuiBpcRvWSWywVesbnhlrXhmi24sdo5KVikVI+5Ch8hnJirQ+hjwTUzXUFs8d0Hsw6mYVQza96IZo3qjOdVeYvAefaGqwDS7SfCRaRklHAZA9/L4zAhwrBrdmiZaKj78VgQwe2637EOhRvUifKVP0E+HELwPjj2WPhqxIlCl3G5u4pgcpjhq87IKiUc4qGj8NmCTAFFi15HObHp7vLD5gLXiPGDUbLXTF37lJluGXONcI6aTvTlC2I0VejryyywCgqcWXAHl1AYMVNd5hRKpLvbyWSz2XxF39H7VuHZ/EE2Q7orNBwzb2fFEtZ53Psj6YRBHO2OrUMaxOyJCZV74XRNwmkb9YusjU+6u3OoZU/pAF97DuVyXyv6WnEo9vna7nOdXr8trMqrvDBpv88rKK52XkI/7dVhyoiw6fYJDvv6lXaVTvBtMpn0QOJ5bzI8Fu4jGQb1HojoaPWB+hM4ahP8KL2OlXjg1t+Ym506ASriBbDG64Sr0glN1C9TJ5xyd2ehE07oAK8ThnK51wleJwzFPq8TPtfpXid4nTAYNzt1gmaCEcMw7DvmMbHMKwb/SVZNCv+JzjCuON/vbwbwlhcuZxEfr3K8yjkLqnpJ9BtHyOsnr58ugshVbR/VM/ezuMLhBvcODhzy/8NhuSgd0qmCc63igtrquPby93r+YJYRLv2LvsES1XuKmGFfvJ2KKEXuXrpFNIZckKt+llQ4TGNc4tq+2Q4P/lbJzxWTdv1zpUIqCgP7xvAsqdy0RzH5xLWSGaDdSHcIdhhOlM5ICViCMXjA/2UdqH2StngXvklasNCw0DJJpDXl2LCGh83UxJ3R4zXJxIEFHsEuIwaBKuUb5T+chfEh62tks9Z/qkigP/+63PR6Mvr6Oub/cvR+XeL+e3q0Gf0Hgnpc+w==")))
|
||||
|
||||
|
||||
@@ -69,7 +70,7 @@ def load_manifest(path, setup_version):
|
||||
if not isinstance(data, dict):
|
||||
raise ManifestError("Manifest must be a mapping")
|
||||
profile = data.get("profile")
|
||||
if profile not in ("qbit", "core", "frontend-material", "frontend-core", "cli", "voyage", "frontend-next", "all", "app-starter", "android", "mcp-policy", "marketing"):
|
||||
if profile not in ("qbit", "core", "frontend-material", "frontend-core", "cli", "voyage", "frontend-next", "all", "app-starter", "android", "mcp-policy", "repository-policy", "marketing"):
|
||||
raise ManifestError("Unsupported profile")
|
||||
allowed = ("profile", "orb_version", "default_branch", "docs_only", "publish")
|
||||
if profile in ("qbit", "core"):
|
||||
@@ -78,7 +79,7 @@ def load_manifest(path, setup_version):
|
||||
allowed += ("release_completion", "release_on_main", "release_tag_policy", "test_branch_policy")
|
||||
elif profile == "cli":
|
||||
allowed += ("publish_context",)
|
||||
elif profile in ("voyage", "frontend-next", "app-starter", "android", "mcp-policy", "marketing"):
|
||||
elif profile in ("voyage", "frontend-next", "app-starter", "android", "mcp-policy", "repository-policy", "marketing"):
|
||||
allowed = ("profile", "orb_version", "default_branch")
|
||||
elif profile == "all":
|
||||
allowed = ("profile", "orb_version", "default_branch", "publish", "publish_context")
|
||||
@@ -88,8 +89,8 @@ def load_manifest(path, setup_version):
|
||||
allowed += ("java_version",)
|
||||
mapping(data, allowed, "manifest")
|
||||
version = data.get("orb_version")
|
||||
if not isinstance(version, str) or not re.fullmatch(r"(?:\d+\.\d+\.\d+|dev:[A-Za-z0-9_-]+)", version):
|
||||
raise ManifestError("orb_version must be an explicit stable or development version")
|
||||
if not isinstance(version, str) or not re.fullmatch(r"(?:\d+\.\d+(?:\.\d+)?|dev:[A-Za-z0-9_-]+)", version):
|
||||
raise ManifestError("orb_version must be a quoted minor track, exact stable pin, or development version")
|
||||
if version != setup_version:
|
||||
raise ManifestError("Manifest orb_version must match the setup job orb_version")
|
||||
default_branch = data.get("default_branch")
|
||||
@@ -110,6 +111,8 @@ def load_manifest(path, setup_version):
|
||||
publish_keys = ("release",)
|
||||
if profile in ("qbit", "core"):
|
||||
publish_keys += ("asciidoc",)
|
||||
if profile == "qbit":
|
||||
publish_keys += ("hotfix", "feature")
|
||||
if profile not in ("qbit", "cli", "all", "app-starter", "voyage", "frontend-next"):
|
||||
publish_keys += ("feature", "hotfix")
|
||||
publishing = mapping(data.get("publish", {}), publish_keys, "publish")
|
||||
@@ -126,7 +129,8 @@ def load_manifest(path, setup_version):
|
||||
if tag_policy not in ("stable", "v-prefix", "none"):
|
||||
raise ManifestError("Unsupported QBit release tag policy")
|
||||
test_policy = data.get("test_branch_policy", "all")
|
||||
if test_policy not in ("all", "non-release", "non-release-no-integration", "feature", "non-release-no-main"):
|
||||
if test_policy not in ("all", "non-release", "non-release-no-integration", "feature", "non-release-no-main",
|
||||
"non-release-no-hotfix", "non-release-no-hotfix-integration", "non-release-no-main-hotfix", "feature-no-hotfix"):
|
||||
raise ManifestError("Unsupported QBit test branch policy")
|
||||
return {"profile": profile, "orb_version": version, "default_branch": default_branch,
|
||||
"java_version": java, "node_version": node, "browsers": boolean(data, "browsers", True),
|
||||
@@ -136,7 +140,8 @@ def load_manifest(path, setup_version):
|
||||
"release_tag_policy": tag_policy, "test_branch_policy": test_policy,
|
||||
"docs_only": boolean(data, "docs_only", False),
|
||||
"publish_context": publish_context,
|
||||
"publish": {key: boolean(publishing, key, key != "asciidoc" and profile not in ("cli", "all"))
|
||||
"publish": {key: boolean(publishing, key, key != "asciidoc" and profile not in ("cli", "all", "repository-policy")
|
||||
and not (profile == "qbit" and key in ("hotfix", "feature")))
|
||||
for key in publish_keys}}
|
||||
|
||||
|
||||
@@ -171,9 +176,28 @@ def proven_docs_only(config, branch, tag):
|
||||
return base.decode().strip() if eligible else False
|
||||
|
||||
|
||||
def proven_ci_maintenance(branch, tag):
|
||||
"""Keep application tests, but do not publish for isolated CI declarations."""
|
||||
if tag or not (branch == "main" or branch == "develop" or branch.startswith(("release/", "hotfix/"))):
|
||||
return False
|
||||
subject = git("log", "-1", "--format=%s")
|
||||
if not subject or not re.match(rb"ci(?:\([^\r\n()]+\))?: ", subject):
|
||||
return False
|
||||
parents = git("rev-list", "--parents", "-n", "1", "HEAD")
|
||||
if not parents or not re.fullmatch(rb"[0-9a-f]{40} [0-9a-f]{40}\n", parents):
|
||||
return False
|
||||
base = parents.split()[1].decode("ascii")
|
||||
changed = git("diff", "--no-ext-diff", "--no-renames", "--name-only", "-z", base, "HEAD")
|
||||
if not changed:
|
||||
return False
|
||||
allowed = {b".qqq-ci.yml", b".circleci/config.yml", b".github/workflows/codeql.yml",
|
||||
b".github/workflows/release-rc.yml", b".github/workflows/schema-drift.yml"}
|
||||
return all(path in allowed for path in changed.rstrip(b"\0").split(b"\0"))
|
||||
|
||||
|
||||
def event_kind(branch, tag, release_on_main=False, release_tag_policy="stable"):
|
||||
if tag:
|
||||
if release_tag_policy == "none" or not re.fullmatch(r"v.*" if release_tag_policy == "v-prefix" else r"v\d+\.\d+\.\d+", tag):
|
||||
if release_tag_policy == "none" or not re.fullmatch(r"v.*" if release_tag_policy == "v-prefix" else r"v\d+\.\d+\.\d+", tag, flags=re.ASCII):
|
||||
return "test"
|
||||
tagged = git("rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}")
|
||||
if not tagged and git("fetch", "--no-tags", "origin", f"refs/tags/{tag}:refs/tags/{tag}") is not None:
|
||||
@@ -191,21 +215,36 @@ def event_kind(branch, tag, release_on_main=False, release_tag_policy="stable"):
|
||||
def qbit_template(config, branch, tag):
|
||||
"""Profile-owned template; add future profiles as reviewed renderers, not raw YAML."""
|
||||
kind = event_kind(branch, tag, config.get("release_on_main", False), config.get("release_tag_policy", "stable"))
|
||||
if tag.startswith("publish") and config["publish"].get("feature", False):
|
||||
tagged = git("rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}")
|
||||
if not tagged and git("fetch", "--no-tags", "origin", f"refs/tags/{tag}:refs/tags/{tag}") is not None:
|
||||
tagged = git("rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}")
|
||||
if tagged and tagged == git("rev-parse", "HEAD"):
|
||||
kind = "feature"
|
||||
if not tag and re.fullmatch(r"hotfix/.+", branch) and config["publish"].get("hotfix", False):
|
||||
kind = "hotfix"
|
||||
if kind != "test" and not config["publish"][kind]:
|
||||
kind = "test"
|
||||
maintenance = proven_ci_maintenance(branch, tag)
|
||||
if maintenance:
|
||||
kind = "test"
|
||||
workflow = "test_only" if kind == "test" else "publish_" + kind
|
||||
result = {"version": 2.1, "orbs": {"qqq-orb": "kingsrook/qqq-orb@" + config["orb_version"]}}
|
||||
filters = ({"branches": {"ignore": "/.*/"}, "tags": {"only": "/.*/"}} if tag else
|
||||
{"branches": {"ignore": "automation/qbit-rc-pending"}, "tags": {"ignore": "/.*/"}})
|
||||
if kind == "test" and not tag:
|
||||
if kind == "test" and not tag and not maintenance:
|
||||
ignores = {"all": None, "non-release": "/(develop|release/.*)/",
|
||||
"non-release-no-integration": "/(develop|release/.*|integration.*)/",
|
||||
"feature": "/(develop|main|release/.*|integration.*)/",
|
||||
"non-release-no-main": "/(develop|main|release/.*)/"}
|
||||
"non-release-no-main": "/(develop|main|release/.*)/",
|
||||
"non-release-no-hotfix": "/(develop|release/.*|hotfix/.*)/",
|
||||
"non-release-no-hotfix-integration": "/(develop|release/.*|hotfix/.*|integration.*)/",
|
||||
"non-release-no-main-hotfix": "/(develop|main|release/.*|hotfix/.*)/",
|
||||
"feature-no-hotfix": "/(develop|main|release/.*|hotfix/.*|integration.*)/"}
|
||||
ignored = ignores[config.get("test_branch_policy", "all")]
|
||||
if ignored:
|
||||
filters["branches"]["ignore"] = ["automation/qbit-rc-pending", ignored]
|
||||
docs_base = proven_docs_only(config, branch, tag) if kind == "test" else False
|
||||
docs_base = proven_docs_only(config, branch, tag) if kind == "test" and not maintenance else False
|
||||
if docs_base:
|
||||
# Keep the existing workflow/result identity even when Maven is not needed.
|
||||
result["jobs"] = {"docs_only": {"resource_class": "small", "docker": [{"image": "cimg/base:2026.03"}],
|
||||
@@ -304,7 +343,10 @@ def frontend_template(config, branch, tag):
|
||||
kind = "release"
|
||||
if kind != "test" and not config["publish"][kind]:
|
||||
kind = "test"
|
||||
if kind == "test" and proven_docs_only(config, branch, tag):
|
||||
maintenance = proven_ci_maintenance(branch, tag)
|
||||
if maintenance:
|
||||
kind = "test"
|
||||
if kind == "test" and not maintenance and proven_docs_only(config, branch, tag):
|
||||
result = qbit_template(config, branch, tag)
|
||||
result["workflows"]["test_only"]["jobs"][0]["docs_only"]["name"] = (
|
||||
"qqq-orb/mvn_frontend_test_only" if material else "qqq-orb/node_test_only")
|
||||
@@ -407,7 +449,13 @@ def voyage_template(config, branch, tag):
|
||||
return json.loads(json.dumps(VOYAGE_TEMPLATE).replace("__ORB_VERSION__", config["orb_version"]))
|
||||
|
||||
|
||||
def repository_policy_template(config, branch, tag):
|
||||
del config, branch, tag
|
||||
return copy.deepcopy(REPOSITORY_POLICY_TEMPLATE)
|
||||
|
||||
|
||||
PROFILE_TEMPLATES = {"qbit": qbit_template, "core": core_template,
|
||||
"repository-policy": repository_policy_template,
|
||||
"frontend-material": frontend_template, "frontend-core": frontend_template,
|
||||
"cli": cli_template, "voyage": voyage_template, "frontend-next": next_template, "all": all_template, "app-starter": starter_template, "android": android_template, "mcp-policy": policy_template, "marketing": marketing_template}
|
||||
|
||||
@@ -441,6 +489,12 @@ def main():
|
||||
if expected and (not re.fullmatch(r"[0-9a-fA-F]{40}", expected) or git("rev-parse", "HEAD") != (expected + "\n").encode()):
|
||||
raise ManifestError("Checkout does not match the pipeline commit")
|
||||
output = PROFILE_TEMPLATES[config["profile"]](config, os.environ.get("CIRCLE_BRANCH", ""), os.environ.get("CIRCLE_TAG", ""))
|
||||
if config["profile"] in ("qbit", "core", "frontend-core", "frontend-material"):
|
||||
# One whole-workflow check follows the same commit from feature to release.
|
||||
# Keep every selected job, dependency and filter; add no runner or duplicate build.
|
||||
if len(output["workflows"]) != 1:
|
||||
raise ManifestError("Stable verification requires exactly one selected workflow")
|
||||
output["workflows"] = {"verify": next(iter(output["workflows"].values()))}
|
||||
parameters = continuation_parameters(config["profile"])
|
||||
if config["profile"] == "core":
|
||||
output["parameters"] = copy.deepcopy(CORE_TEMPLATE["parameters"])
|
||||
|
||||
@@ -116,7 +116,7 @@ def reconcile(api, repo):
|
||||
validate_run(run, intent, repo, merge, run_id)
|
||||
checks = api.request(prefix + 'commits/' + merge + '/check-runs?filter=latest&per_page=100')
|
||||
publishers = [c for c in checks.get('check_runs', [])
|
||||
if c.get('name') == 'publish_release_candidate' and c.get('app', {}).get('id') == 18001]
|
||||
if c.get('name') in ('publish_release_candidate', 'verify') and c.get('app', {}).get('id') == 18001]
|
||||
if not publishers:
|
||||
return 'publisher-running'
|
||||
publisher = max(publishers, key=lambda c: c['id'])
|
||||
|
||||
@@ -87,7 +87,7 @@
|
||||
{
|
||||
"run": {
|
||||
"name": "Publish with shared Maven credentials",
|
||||
"command": "set -euo pipefail\n: \"${GITHUB_TOKEN:?GitHub Packages write credential is required}\"\n# Authenticate before compiling; repository access is necessary, not proof of package write access.\nregistry_user=$(python3 <<'PYTHON'\nimport json\nimport os\nimport urllib.request\n\nclass NoRedirect(urllib.request.HTTPRedirectHandler):\n def redirect_request(self, req, fp, code, msg, headers, newurl):\n raise RuntimeError(\"GitHub authentication redirect refused\")\n\ntry:\n opener = urllib.request.build_opener(NoRedirect)\n def get(path):\n request = urllib.request.Request(\"https://api.github.com/\" + path,\n headers={\"Authorization\": \"Bearer \" + os.environ[\"GITHUB_TOKEN\"],\n \"Accept\": \"application/vnd.github+json\"})\n with opener.open(request, timeout=30) as response:\n return json.load(response)\n user = get(\"user\").get(\"login\")\n if not isinstance(user, str) or not user or not get(\"repos/QRun-IO/qqq-maven-registry\").get(\"permissions\", {}).get(\"push\"):\n raise ValueError(\"Missing registry access\")\n print(user)\nexcept Exception:\n raise SystemExit(\"GitHub Packages credential cannot access the Maven registry\") from None\nPYTHON\n)\nexport ORG_GRADLE_PROJECT_githubQqqMavenRegistryUsername=\"$registry_user\"\nexport ORG_GRADLE_PROJECT_githubQqqMavenRegistryPassword=\"$GITHUB_TOKEN\"\n./gradlew --console=plain publishMavenPublicationToGithubQqqMavenRegistryRepository\n"
|
||||
"command": "set -euo pipefail\n: \"${GITHUB_TOKEN:?GitHub Packages write credential is required}\"\n# Authenticate before compiling; repository access is necessary, not proof of package write access.\nregistry_user=$(python3 \\<<'PYTHON'\nimport json\nimport os\nimport urllib.request\n\nclass NoRedirect(urllib.request.HTTPRedirectHandler):\n def redirect_request(self, req, fp, code, msg, headers, newurl):\n raise RuntimeError(\"GitHub authentication redirect refused\")\n\ntry:\n opener = urllib.request.build_opener(NoRedirect)\n def get(path):\n request = urllib.request.Request(\"https://api.github.com/\" + path,\n headers={\"Authorization\": \"Bearer \" + os.environ[\"GITHUB_TOKEN\"],\n \"Accept\": \"application/vnd.github+json\"})\n with opener.open(request, timeout=30) as response:\n return json.load(response)\n user = get(\"user\").get(\"login\")\n if not isinstance(user, str) or not user or not get(\"repos/QRun-IO/qqq-maven-registry\").get(\"permissions\", {}).get(\"push\"):\n raise ValueError(\"Missing registry access\")\n print(user)\nexcept Exception:\n raise SystemExit(\"GitHub Packages credential cannot access the Maven registry\") from None\nPYTHON\n)\nexport ORG_GRADLE_PROJECT_githubQqqMavenRegistryUsername=\"$registry_user\"\nexport ORG_GRADLE_PROJECT_githubQqqMavenRegistryPassword=\"$GITHUB_TOKEN\"\n./gradlew --console=plain publishMavenPublicationToGithubQqqMavenRegistryRepository\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -198,20 +198,10 @@
|
||||
"caller": {
|
||||
"name": "Database migrations",
|
||||
"on": {
|
||||
"pull_request": {
|
||||
"paths": [
|
||||
"db/**",
|
||||
"pom.xml",
|
||||
".github/workflows/db-migrations.yml"
|
||||
]
|
||||
},
|
||||
"pull_request": {},
|
||||
"push": {
|
||||
"branches": [
|
||||
"main"
|
||||
],
|
||||
"paths": [
|
||||
"db/**",
|
||||
"pom.xml"
|
||||
]
|
||||
}
|
||||
},
|
||||
@@ -222,6 +212,10 @@
|
||||
"validate": {
|
||||
"uses": "QRun-IO/qqq-orb/.github/workflows/business-db.yml@WORKER_SHA"
|
||||
}
|
||||
},
|
||||
"concurrency": {
|
||||
"group": "business-db-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}",
|
||||
"cancel-in-progress": "${{ github.event_name == 'pull_request' }}"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -262,11 +256,6 @@
|
||||
"main"
|
||||
]
|
||||
},
|
||||
"schedule": [
|
||||
{
|
||||
"cron": "17 6 * * *"
|
||||
}
|
||||
],
|
||||
"workflow_dispatch": null
|
||||
},
|
||||
"permissions": {
|
||||
@@ -274,7 +263,7 @@
|
||||
},
|
||||
"concurrency": {
|
||||
"group": "e2e-${{ github.ref }}",
|
||||
"cancel-in-progress": true
|
||||
"cancel-in-progress": "${{ github.event_name == 'pull_request' }}"
|
||||
},
|
||||
"jobs": {
|
||||
"validate": {
|
||||
@@ -438,5 +427,45 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"qbit-codeql": {
|
||||
"repository": null,
|
||||
"profile": "qbit",
|
||||
"path": ".github/workflows/codeql.yml",
|
||||
"caller": {
|
||||
"name": "CodeQL",
|
||||
"on": {
|
||||
"push": {
|
||||
"branches": [
|
||||
"main",
|
||||
"develop",
|
||||
"release/**",
|
||||
"hotfix/**"
|
||||
]
|
||||
},
|
||||
"pull_request": {},
|
||||
"schedule": [
|
||||
{
|
||||
"cron": "41 16 * * 0"
|
||||
}
|
||||
],
|
||||
"workflow_dispatch": {}
|
||||
},
|
||||
"permissions": {
|
||||
"contents": "read",
|
||||
"actions": "read",
|
||||
"security-events": "write",
|
||||
"packages": "read"
|
||||
},
|
||||
"concurrency": {
|
||||
"group": "codeql-${{ github.event.pull_request.number || github.ref }}",
|
||||
"cancel-in-progress": "${{ github.event_name == 'pull_request' }}"
|
||||
},
|
||||
"jobs": {
|
||||
"analyze": {
|
||||
"uses": "QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@WORKER_SHA"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -649,7 +649,9 @@
|
||||
"qqq-orb/mvn_publish": {
|
||||
"requires": [
|
||||
"sample_bootstrap_source",
|
||||
"snapshot_esb_broker_conformance"
|
||||
"snapshot_esb_broker_conformance",
|
||||
"snapshot_sample_coverage_gate",
|
||||
"qqq-orb/static_analysis"
|
||||
],
|
||||
"pre-steps": [
|
||||
"configure-package-install"
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"version": 2.1,
|
||||
"jobs": {
|
||||
"validate-policy": {
|
||||
"docker": [
|
||||
{
|
||||
"image": "cimg/python:3.12"
|
||||
}
|
||||
],
|
||||
"resource_class": "small",
|
||||
"steps": [
|
||||
"checkout",
|
||||
{
|
||||
"run": {
|
||||
"name": "Install the YAML parser",
|
||||
"command": "python3 -m pip install --disable-pip-version-check PyYAML==6.0.3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"run": {
|
||||
"name": "Validate the application-free branch",
|
||||
"command": "python3 \\<<'PYTHON'\nimport json\nfrom pathlib import Path\nimport subprocess\nimport xml.etree.ElementTree as ET\nimport yaml\n\npaths = subprocess.check_output(['git', 'ls-files', '-z']).decode('utf-8').split('\\0')\npaths = [Path(name) for name in paths if name]\nif not paths:\n raise SystemExit('No tracked policy files')\nnames = {'LICENSE', 'NOTICE', 'COPYING', 'CODEOWNERS', '.gitignore', '.gitattributes',\n '.editorconfig', 'mvnw', 'mvnw.cmd'}\nsuffixes = {'.md', '.rst', '.txt', '.adoc', '.yml', '.yaml', '.json', '.properties'}\nmanifests = {'pom.xml', 'package.json', 'cargo.toml', 'pyproject.toml', 'go.mod',\n 'build.gradle', 'build.gradle.kts', 'settings.gradle', 'settings.gradle.kts'}\nfor path in paths:\n if path.is_symlink() or path.name.lower() in manifests:\n raise SystemExit(f'Application or symlink requires a build profile: {path}')\n if path.as_posix() == '.mvn/wrapper/maven-wrapper.jar':\n continue\n if path.as_posix() == 'checkstyle/config.xml':\n ET.fromstring(path.read_text(encoding='utf-8'))\n continue\n if path.name not in names and path.suffix.lower() not in suffixes:\n raise SystemExit(f'Unsupported policy file; select a build profile: {path}')\n text = path.read_text(encoding='utf-8')\n if path.suffix.lower() in ('.yml', '.yaml'):\n list(yaml.safe_load_all(text))\n elif path.suffix.lower() == '.json':\n json.loads(text)\nprint(f'Validated {len(paths)} tracked policy/documentation files; no application build or publication')\nPYTHON\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"workflows": {
|
||||
"verify": {
|
||||
"jobs": [
|
||||
"validate-policy"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,7 @@ PROFILE_PARAMETERS = {
|
||||
"app-starter": {},
|
||||
"android": {},
|
||||
"mcp-policy": {},
|
||||
"repository-policy": {},
|
||||
"marketing": {},
|
||||
}
|
||||
|
||||
@@ -58,7 +59,7 @@ def load_yaml(raw):
|
||||
raise StandardError("Invalid YAML declaration") from error
|
||||
|
||||
|
||||
def canonical_caller(profile, version, explicit_manifest=False, release_tag_policy=None):
|
||||
def canonical_caller(profile, version, explicit_manifest=False, release_tag_policy=None, publish_feature=False):
|
||||
if not isinstance(profile, str) or profile not in PROFILE_PARAMETERS:
|
||||
raise StandardError("Unsupported CI profile")
|
||||
options = {
|
||||
@@ -72,6 +73,9 @@ def canonical_caller(profile, version, explicit_manifest=False, release_tag_poli
|
||||
if release_tag_policy not in tag_filters:
|
||||
raise StandardError("Unsupported QBit release tag policy")
|
||||
options["filters"]["tags"] = tag_filters[release_tag_policy]
|
||||
if publish_feature:
|
||||
options["filters"]["tags"] = {"only": ([tag_filters[release_tag_policy]["only"]]
|
||||
if release_tag_policy != "none" else []) + ["/publish.*/"]}
|
||||
if explicit_manifest:
|
||||
options["manifest"] = ".qqq-ci.yml"
|
||||
parameters = PROFILE_PARAMETERS[profile]
|
||||
@@ -105,12 +109,15 @@ def validate(config_raw, manifest_raw, setup_version):
|
||||
if not isinstance(manifest, dict):
|
||||
raise StandardError("Manifest must be a mapping")
|
||||
version = manifest.get("orb_version")
|
||||
if not isinstance(version, str) or not re.fullmatch(r"(?:\d+\.\d+\.\d+|dev:[A-Za-z0-9_-]+)", version):
|
||||
raise StandardError("Manifest must pin an explicit orb version")
|
||||
if not isinstance(version, str) or not re.fullmatch(r"(?:\d+\.\d+(?:\.\d+)?|dev:[A-Za-z0-9_-]+)", version):
|
||||
raise StandardError("Manifest must select a quoted minor track, exact stable pin, or development version")
|
||||
if version != setup_version:
|
||||
raise StandardError("Manifest and running setup orb versions differ")
|
||||
expected = canonical_caller(manifest.get("profile"), version, release_tag_policy=manifest.get("release_tag_policy"))
|
||||
explicit = canonical_caller(manifest.get("profile"), version, explicit_manifest=True, release_tag_policy=manifest.get("release_tag_policy"))
|
||||
publishing = manifest.get("publish", {})
|
||||
if not isinstance(publishing, dict) or type(publishing.get("feature", False)) is not bool:
|
||||
raise StandardError("Feature publication must be an explicit boolean")
|
||||
expected = canonical_caller(manifest.get("profile"), version, release_tag_policy=manifest.get("release_tag_policy"), publish_feature=publishing.get("feature", False))
|
||||
explicit = canonical_caller(manifest.get("profile"), version, explicit_manifest=True, release_tag_policy=manifest.get("release_tag_policy"), publish_feature=publishing.get("feature", False))
|
||||
if not exact(config, expected) and not exact(config, explicit):
|
||||
raise StandardError("Consumer config must be the canonical setup caller with only its profile's declared and forwarded parameters")
|
||||
|
||||
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
version: 2.1
|
||||
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
workflows:
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|main|release/.*|hotfix/.*|integration.*)/
|
||||
tags:
|
||||
only: []
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- develop
|
||||
|
||||
publish_feature:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: feature
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- feature/.*
|
||||
tags:
|
||||
only: [/publish.*/]
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /release\/.*/
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [/^v\d+\.\d+\.\d+$/]
|
||||
|
||||
publish_hotfix_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: hotfix
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /hotfix\/.*/
|
||||
+1835
File diff suppressed because it is too large.
Load diff
+233
-1
@@ -1,4 +1,6 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import io
|
||||
import pathlib
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
@@ -25,7 +27,197 @@ class Snapshot:
|
||||
return self.required
|
||||
|
||||
|
||||
class InventoryTests(unittest.TestCase):
|
||||
def repo(self, name, archived=False):
|
||||
return {'name': name, 'full_name': 'QRun-IO/' + name,
|
||||
'archived': archived, 'default_branch': 'main'}
|
||||
|
||||
def test_all_pages_include_new_private_and_archived_repositories(self):
|
||||
new = dict(self.repo('new-product'), private=True)
|
||||
pages = [[self.repo('qbit-existing')], [new, self.repo('retired', True)]]
|
||||
with patch.object(audit, 'command', return_value=json.dumps(pages)) as command:
|
||||
entries = audit.discover_inventory('QRun-IO')
|
||||
self.assertEqual([e['name'] for e in entries], ['new-product', 'qbit-existing', 'retired'])
|
||||
self.assertTrue(entries[-1]['isArchived'])
|
||||
self.assertEqual(command.call_args.args[0], ['gh', 'api', '--method', 'GET',
|
||||
'--paginate', '--slurp', 'orgs/QRun-IO/repos?type=all&per_page=100'])
|
||||
|
||||
def test_empty_duplicate_foreign_and_malformed_inventories_fail_closed(self):
|
||||
repo = self.repo('qbit-example')
|
||||
for pages in ([], [[]], {}, [repo], [[repo], [repo]],
|
||||
[[dict(repo, full_name='Other/qbit-example')]],
|
||||
[[dict(repo, archived='false')]], [[dict(repo, default_branch=None)]]):
|
||||
with self.subTest(pages=pages), patch.object(audit, 'command', return_value=json.dumps(pages)):
|
||||
with self.assertRaises(audit.ReadError):
|
||||
audit.discover_inventory('QRun-IO')
|
||||
with patch.object(audit, 'command') as command:
|
||||
with self.assertRaises(audit.ReadError):
|
||||
audit.discover_inventory('QRun-IO/../../other')
|
||||
command.assert_not_called()
|
||||
|
||||
def test_live_cli_count_is_dynamic_but_explicit_expectation_is_enforced(self):
|
||||
for expected, status in ((None, 0), ('1', 0), ('68', 1)):
|
||||
args = ['audit', '--organization', 'QRun-IO', '--live', '--orb-version', '0.8']
|
||||
if expected:
|
||||
args += ['--expected-active', expected]
|
||||
report = {'active_count': 1, 'unique_repository_count': 1,
|
||||
'inventory_count': 1, 'results': [{'status': 'pass'}]}
|
||||
output = io.StringIO()
|
||||
with patch.object(audit.sys, 'argv', args), patch.object(audit.sys, 'stdout', output), \
|
||||
patch.object(audit, 'discover_inventory', return_value=[ENTRY]), \
|
||||
patch.object(audit, 'run', return_value=report):
|
||||
self.assertEqual(audit.main(), status)
|
||||
self.assertEqual(json.loads(output.getvalue())['inventory_source'], 'organization-api')
|
||||
|
||||
def test_discovery_api_failure_never_falls_back_to_old_inventory(self):
|
||||
with patch.object(audit, 'command', side_effect=audit.ReadError('unavailable')):
|
||||
with self.assertRaises(audit.ReadError):
|
||||
audit.discover_inventory('QRun-IO')
|
||||
|
||||
|
||||
class AuditTests(unittest.TestCase):
|
||||
def test_all_scanner_and_release_requirements_follow_branch_profile(self):
|
||||
entry = dict(ENTRY, name='qqq-all', nameWithOwner='QRun-IO/qqq-all')
|
||||
scanner = '.github/workflows/codeql.yml'
|
||||
release = audit.auxiliary.BRIDGES['all-release']['path']
|
||||
for profile in ('all', 'repository-policy'):
|
||||
files = {'.qqq-ci.yml': audit.generator.yaml.safe_dump({'profile': profile, 'orb_version': VERSION, 'default_branch': 'develop'}),
|
||||
'.circleci/config.yml': audit.generator.yaml.safe_dump(audit.standard.canonical_caller(profile, VERSION))}
|
||||
policy = dict(POLICY, profile=profile, codeql_worker_sha=SHA)
|
||||
report = audit.audit(entry, Snapshot(files), VERSION, SHA, policy)
|
||||
missing = {f.get('path') for f in report['findings'] if f['code'] == 'missing_native_maintenance_bridge'}
|
||||
self.assertIn(scanner, missing)
|
||||
self.assertEqual(profile == 'all', release in missing)
|
||||
files[scanner] = audit.generator.yaml.safe_dump(audit.auxiliary.canonical('qbit-codeql', SHA), sort_keys=False)
|
||||
if profile == 'all':
|
||||
files[release] = audit.generator.yaml.safe_dump(audit.auxiliary.canonical('all-release', SHA), sort_keys=False)
|
||||
self.assertEqual([], audit.audit(entry, Snapshot(files), VERSION, SHA, policy)['findings'])
|
||||
files[scanner] = files[scanner].replace(SHA, 'b' * 40)
|
||||
self.assertIn('native_maintenance_bridge_drift', [f['code'] for f in audit.audit(entry, Snapshot(files), VERSION, SHA, policy)['findings']])
|
||||
files[release] = 'jobs: {}'
|
||||
self.assertTrue(any(f['code'] == 'native_maintenance_bridge_drift' and f.get('path') == release for f in audit.audit(entry, Snapshot(files), VERSION, SHA, policy)['findings']))
|
||||
|
||||
def test_schema_drift_bridge_tracks_branch_source_without_exempting_default(self):
|
||||
entry = dict(ENTRY, name='qbit-authentik', nameWithOwner='QRun-IO/qbit-authentik')
|
||||
path = '.github/workflows/schema-drift.yml'
|
||||
cases = [
|
||||
('main', {}, False),
|
||||
('main', {'src/test/java/example/SchemaDriftTest.java': 'class SchemaDriftTest {}'}, True),
|
||||
('develop', {}, True),
|
||||
]
|
||||
for branch, files, required in cases:
|
||||
with self.subTest(branch=branch, files=files):
|
||||
snapshot = Snapshot(files)
|
||||
snapshot.branch, snapshot.default_branch = branch, 'develop'
|
||||
report = audit.audit(entry, snapshot, VERSION, SHA, POLICY)
|
||||
missing = any(f['code'] == 'missing_native_maintenance_bridge' and f.get('path') == path for f in report['findings'])
|
||||
self.assertEqual(required, missing)
|
||||
snapshot = Snapshot({path: 'jobs: {}'})
|
||||
snapshot.branch, snapshot.default_branch = 'main', 'develop'
|
||||
report = audit.audit(entry, snapshot, VERSION, SHA, POLICY)
|
||||
self.assertIn('native_maintenance_bridge_drift', [f['code'] for f in report['findings']])
|
||||
|
||||
def test_placeholder_can_retain_canonical_release_and_scanner_bridges(self):
|
||||
entry = dict(ENTRY, name='qbit-help', nameWithOwner='QRun-IO/qbit-help')
|
||||
files = {'.qqq-ci.yml': audit.generator.yaml.safe_dump({'profile': 'repository-policy', 'orb_version': VERSION, 'default_branch': 'develop'}),
|
||||
'.circleci/config.yml': audit.generator.yaml.safe_dump(audit.standard.canonical_caller('repository-policy', VERSION)),
|
||||
audit.rc_worker.CALLER_PATH: audit.rc_worker.generate(SHA),
|
||||
'.github/workflows/codeql.yml': audit.generator.yaml.safe_dump(audit.auxiliary.canonical('qbit-codeql', SHA), sort_keys=False)}
|
||||
snapshot = Snapshot(files)
|
||||
snapshot.branch = 'main'
|
||||
snapshot.default_branch = 'develop'
|
||||
policy = dict(POLICY, branch_profiles={'main': 'repository-policy'}, rc_worker_sha=SHA, codeql_worker_sha=SHA)
|
||||
self.assertEqual([], audit.audit(entry, snapshot, VERSION, SHA, policy)['findings'])
|
||||
files['.github/workflows/codeql.yml'] = files['.github/workflows/codeql.yml'].replace(SHA, 'b' * 40)
|
||||
self.assertTrue(any(f['code'] == 'native_maintenance_bridge_drift' for f in audit.audit(entry, snapshot, VERSION, SHA, policy)['findings']))
|
||||
|
||||
def test_placeholder_profile_is_scoped_to_selected_branch(self):
|
||||
entry = dict(ENTRY, name='qbit-help', nameWithOwner='QRun-IO/qbit-help')
|
||||
files = {'.qqq-ci.yml': audit.generator.yaml.safe_dump({'profile': 'repository-policy', 'orb_version': VERSION, 'default_branch': 'develop'}),
|
||||
'.circleci/config.yml': audit.generator.yaml.safe_dump(audit.standard.canonical_caller('repository-policy', VERSION))}
|
||||
snapshot = Snapshot(files)
|
||||
snapshot.branch = 'main'
|
||||
snapshot.default_branch = 'develop'
|
||||
policy = dict(POLICY, branch_profiles={'main': 'repository-policy'})
|
||||
self.assertEqual([], audit.audit(entry, snapshot, VERSION, SHA, policy)['findings'])
|
||||
snapshot.branch = 'develop'
|
||||
report = audit.audit(entry, snapshot, VERSION, SHA, policy)
|
||||
self.assertEqual('qbit', report['profile'])
|
||||
self.assertTrue(any(f['code'] == 'profile_mismatch' for f in report['findings']))
|
||||
|
||||
def test_qbit_scanner_policy_requires_exact_canonical_worker(self):
|
||||
entry = dict(ENTRY, name='qbit-help', nameWithOwner='QRun-IO/qbit-help')
|
||||
manifest = {'profile': 'qbit', 'orb_version': VERSION, 'default_branch': 'develop',
|
||||
'publish': {'snapshot': False, 'release_candidate': False, 'release': False}}
|
||||
files = {'.qqq-ci.yml': audit.generator.yaml.safe_dump(manifest),
|
||||
'.circleci/config.yml': audit.generator.yaml.safe_dump(audit.standard.canonical_caller('qbit', VERSION))}
|
||||
policy = dict(POLICY, codeql_worker_sha='b' * 40)
|
||||
report = audit.audit(entry, Snapshot(files), VERSION, SHA, policy)
|
||||
self.assertTrue(any(f['code'] == 'missing_native_maintenance_bridge' for f in report['findings']))
|
||||
files['.github/workflows/codeql.yml'] = audit.generator.yaml.safe_dump(audit.auxiliary.canonical('qbit-codeql', 'b' * 40), sort_keys=False)
|
||||
self.assertEqual([], audit.audit(entry, Snapshot(files), VERSION, SHA, policy)['findings'])
|
||||
files['.github/workflows/codeql.yml'] = files['.github/workflows/codeql.yml'].replace('b' * 40, 'a' * 40)
|
||||
report = audit.audit(entry, Snapshot(files), VERSION, SHA, policy)
|
||||
self.assertTrue(any(f['code'] == 'native_maintenance_bridge_drift' for f in report['findings']))
|
||||
|
||||
def test_live_snapshot_reads_requested_branch_and_keeps_default_identity(self):
|
||||
entry = dict(ENTRY, sourceBranch='release/1.2')
|
||||
replies = [{'default_branch': 'develop', 'archived': False}, {'sha': SHA},
|
||||
{'truncated': False, 'tree': []}]
|
||||
with patch.object(audit, 'gh', side_effect=replies) as gh:
|
||||
snapshot = audit.Snapshot(entry, live=True)
|
||||
self.assertEqual('develop', snapshot.default_branch)
|
||||
self.assertEqual('release/1.2', snapshot.branch)
|
||||
self.assertEqual('repos/QRun-IO/qctl/commits/release%2F1.2', gh.call_args_list[1].args[0])
|
||||
|
||||
def test_other_source_branch_does_not_change_manifest_default_branch(self):
|
||||
entry = dict(ENTRY, name='qbit-help', nameWithOwner='QRun-IO/qbit-help')
|
||||
manifest = {'profile': 'qbit', 'orb_version': VERSION, 'default_branch': 'develop',
|
||||
'publish': {'snapshot': False, 'release_candidate': False, 'release': False}}
|
||||
files = {'.qqq-ci.yml': audit.generator.yaml.safe_dump(manifest),
|
||||
'.circleci/config.yml': audit.generator.yaml.safe_dump(audit.standard.canonical_caller('qbit', VERSION))}
|
||||
snapshot = Snapshot(files)
|
||||
snapshot.branch = 'main'
|
||||
snapshot.default_branch = 'develop'
|
||||
report = audit.audit(entry, snapshot, VERSION, SHA, POLICY)
|
||||
self.assertEqual([], report['findings'])
|
||||
self.assertEqual('main', report['source_branch'])
|
||||
self.assertEqual('develop', report['default_branch'])
|
||||
|
||||
def test_selected_branch_read_failure_is_unknown_not_default_fallback(self):
|
||||
captured = []
|
||||
def factory(entry, live):
|
||||
captured.append(entry)
|
||||
raise audit.ReadError('branch unavailable')
|
||||
report = audit.run([ENTRY], VERSION, SHA, {}, live=True,
|
||||
snapshot_factory=factory, source_branch='main')
|
||||
self.assertEqual('main', captured[0]['sourceBranch'])
|
||||
self.assertEqual('unknown', report['results'][0]['status'])
|
||||
self.assertEqual('main', report['results'][0]['source_branch'])
|
||||
self.assertNotIn('sourceBranch', ENTRY)
|
||||
|
||||
def test_omitted_source_branch_still_reads_repository_default(self):
|
||||
replies = [{'default_branch': 'develop', 'archived': False}, {'sha': SHA},
|
||||
{'truncated': False, 'tree': []}]
|
||||
with patch.object(audit, 'gh', side_effect=replies) as gh:
|
||||
snapshot = audit.Snapshot(ENTRY, live=True)
|
||||
self.assertEqual('develop', snapshot.branch)
|
||||
self.assertEqual('develop', snapshot.default_branch)
|
||||
self.assertEqual('repos/QRun-IO/qctl/commits/develop', gh.call_args_list[1].args[0])
|
||||
|
||||
def test_explicit_source_branch_cannot_relabel_offline_snapshot(self):
|
||||
with self.assertRaises(ValueError):
|
||||
audit.run([ENTRY], VERSION, SHA, {}, source_branch='main')
|
||||
with self.assertRaises(audit.ReadError):
|
||||
audit.Snapshot(dict(ENTRY, sourceBranch='main'), live=False)
|
||||
|
||||
def test_invalid_source_branch_is_rejected_before_api_calls(self):
|
||||
for branch in ('', 'main?ref=develop', 'main\n', 123):
|
||||
with self.subTest(branch=branch), patch.object(audit, 'gh') as gh:
|
||||
with self.assertRaises(audit.ReadError):
|
||||
audit.Snapshot(dict(ENTRY, sourceBranch=branch), live=True)
|
||||
gh.assert_not_called()
|
||||
|
||||
def native(self, files=None, checks=None, policy=POLICY):
|
||||
files = files if files is not None else {'.github/workflows/build.yml': audit.qctl.generate(SHA)}
|
||||
return audit.audit(ENTRY, Snapshot(files, checks), VERSION, SHA, policy)
|
||||
@@ -74,6 +266,25 @@ class AuditTests(unittest.TestCase):
|
||||
report = audit.audit(entry, Snapshot({'.qqq-ci.yml': audit.generator.yaml.safe_dump(manifest)}), VERSION, SHA, POLICY)
|
||||
self.assertIn('missing_rc_bridge', [f['code'] for f in report['findings']])
|
||||
|
||||
def test_rc_pin_policy_does_not_force_unchanged_auxiliary_worker_updates(self):
|
||||
entry = dict(ENTRY, name='qbit-authentik', nameWithOwner='QRun-IO/qbit-authentik')
|
||||
rc_sha = 'b' * 40
|
||||
definition = audit.auxiliary.BRIDGES['qbit-schema-drift']
|
||||
manifest = {'profile': 'qbit', 'orb_version': VERSION, 'default_branch': 'develop', 'release_completion': True}
|
||||
files = {audit.rc_worker.CALLER_PATH: audit.rc_worker.generate(rc_sha),
|
||||
definition['path']: audit.generator.yaml.safe_dump(audit.auxiliary.canonical('qbit-schema-drift', SHA), sort_keys=False),
|
||||
'.qqq-ci.yml': audit.generator.yaml.safe_dump(manifest),
|
||||
'.circleci/config.yml': audit.generator.yaml.safe_dump(audit.standard.canonical_caller('qbit', VERSION))}
|
||||
policy = dict(POLICY, rc_worker_sha=rc_sha)
|
||||
self.assertEqual('pass', audit.audit(entry, Snapshot(files), VERSION, SHA, policy)['status'])
|
||||
for pin in (SHA, 'main', ''):
|
||||
with self.subTest(pin=pin):
|
||||
report = audit.audit(entry, Snapshot(files), VERSION, SHA, dict(POLICY, rc_worker_sha=pin))
|
||||
self.assertTrue(report['findings'])
|
||||
files[definition['path']] = files[definition['path']].replace(SHA, rc_sha)
|
||||
report = audit.audit(entry, Snapshot(files), VERSION, SHA, policy)
|
||||
self.assertIn('native_maintenance_bridge_drift', [f['code'] for f in report['findings']])
|
||||
|
||||
def test_qbit_rc_bridge_missing_drift_pin_and_duplicate(self):
|
||||
entry = dict(ENTRY, name='qbit-test', nameWithOwner='QRun-IO/qbit-test')
|
||||
path = audit.rc_worker.CALLER_PATH
|
||||
@@ -226,10 +437,31 @@ class AuditTests(unittest.TestCase):
|
||||
audit.audit(other, Snapshot({}, checks=[]), VERSION, SHA, policy)
|
||||
|
||||
def test_invalid_managed_branches_cannot_bypass_policy(self):
|
||||
for managed in [[], ['main', 'main'], ['feature'], 'main', None, [True]]:
|
||||
for managed in [[], ['main', 'main'], ['feature'], 'main', None, [True],
|
||||
['release/*'], ['release/1.2/extra'], ['hotfix/1..2'],
|
||||
['release/1.2', 'release/1.2']]:
|
||||
with self.subTest(managed=managed), self.assertRaises(audit.ReadError):
|
||||
self.native(policy={'managed_branches': managed, 'required_checks': {}})
|
||||
|
||||
def test_release_and_hotfix_enforcement_requires_actual_checks(self):
|
||||
files = {'.github/workflows/build.yml': audit.qctl.generate(SHA)}
|
||||
for branch in ['release/0.1', 'release/4.1.0', 'hotfix/4.0.1']:
|
||||
with self.subTest(branch=branch):
|
||||
snapshot = Snapshot(files)
|
||||
snapshot.branch = branch
|
||||
policy = {'managed_branches': [branch],
|
||||
'required_checks': {branch: ['native / test']}}
|
||||
with patch.object(snapshot, 'checks', wraps=snapshot.checks) as read:
|
||||
report = audit.audit(ENTRY, snapshot, VERSION, SHA, policy)
|
||||
self.assertEqual('pass', report['status'])
|
||||
read.assert_called_once_with(branch)
|
||||
snapshot.required = []
|
||||
report = audit.audit(ENTRY, snapshot, VERSION, SHA, policy)
|
||||
self.assertEqual('missing', report['required_checks'][branch]['state'])
|
||||
self.assertEqual('findings', report['status'])
|
||||
policy['required_checks'] = {}
|
||||
self.assertEqual('unknown', audit.audit(ENTRY, snapshot, VERSION, SHA, policy)['status'])
|
||||
|
||||
def test_gh_is_explicit_get_and_never_changes_settings(self):
|
||||
with patch.object(audit, 'command', return_value='{}') as command:
|
||||
audit.gh('repos/QRun-IO/example')
|
||||
|
||||
@@ -58,12 +58,14 @@ class SpecialistNativeTests(unittest.TestCase):
|
||||
def worker(self, kind):
|
||||
return yaml.safe_load((ROOT / '.github/workflows' / (kind + '.yml')).read_text(encoding='utf-8'))
|
||||
|
||||
def test_business_database_job_and_path_filters_are_unchanged(self):
|
||||
def test_business_database_gate_reports_for_every_pr_and_preserves_validation(self):
|
||||
original = self.originals['business-db']
|
||||
worker = self.worker('business-db')
|
||||
self.assertEqual(worker['jobs'], original['workflow']['jobs'])
|
||||
caller = bridge.canonical('business-db', SHA)
|
||||
self.assertEqual(caller['on'], original['events'])
|
||||
self.assertEqual(caller['on'], {'pull_request': {}, 'push': {'branches': ['main']}})
|
||||
self.assertEqual(caller['concurrency']['cancel-in-progress'], "${{ github.event_name == 'pull_request' }}")
|
||||
self.assertIn('github.event.pull_request.number || github.ref', caller['concurrency']['group'])
|
||||
self.assertEqual(set(worker['jobs']), {'empty-database-migration'})
|
||||
self.assertNotIn('secrets', caller['jobs']['validate'])
|
||||
self.assertNotIn('schedule', caller['on'])
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
# Exercise real browser scripts without sudo, package installation or network.
|
||||
set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
mkdir "$scratch/bin"
|
||||
export INSTALL_LOG="$scratch/install.log"
|
||||
cat > "$scratch/bin/sudo" <<'STUB'
|
||||
#!/bin/bash
|
||||
printf '%s\n' "$*" >> "$INSTALL_LOG"
|
||||
if [[ "$*" == *'apt-get update'* ]]; then
|
||||
[[ "$*" == *'Dir::Etc::sourceparts=-'* ]] || { echo 'unrelated image PPA: 503' >&2; exit 100; }
|
||||
[[ "$*" == *'APT::Get::List-Cleanup=0'* ]] || exit 91
|
||||
[[ ${REQUIRED_REPO_FAIL:-0} == 0 ]] || exit 100
|
||||
fi
|
||||
if [[ "$1" == tee ]]; then cat >> "$INSTALL_LOG"; fi
|
||||
if [[ "$1" == gpg ]]; then cat >/dev/null; fi
|
||||
STUB
|
||||
chmod +x "$scratch/bin/sudo"
|
||||
export PATH="$scratch/bin:$PATH"
|
||||
bash "$root/src/scripts/frontend_test_dependencies.sh"
|
||||
echo 'PASS: browser dependencies ignore unavailable image PPAs'
|
||||
# Model Chrome presence without depending on the host's installed browsers.
|
||||
export CHROME_MARKER="$scratch/chrome-installed"
|
||||
export BASH_ENV="$scratch/shell-env"
|
||||
cat > "$BASH_ENV" <<'STUB'
|
||||
command() {
|
||||
if [[ "$*" == '-v google-chrome' ]]; then [[ -f "$CHROME_MARKER" ]]; else builtin command "$@"; fi
|
||||
}
|
||||
google-chrome() { [[ -f "$CHROME_MARKER" ]] && echo 'Google Chrome test'; }
|
||||
STUB
|
||||
cat > "$scratch/bin/curl" <<'STUB'
|
||||
#!/bin/bash
|
||||
printf '%s\n' "$*" >> "$INSTALL_LOG"
|
||||
[[ ${KEY_FAIL:-0} == 0 ]] || exit 22
|
||||
printf 'signing-key\n'
|
||||
STUB
|
||||
cat > "$scratch/bin/dpkg" <<'STUB'
|
||||
#!/bin/bash
|
||||
echo amd64
|
||||
STUB
|
||||
# Successful installation makes Chrome available to the final version check.
|
||||
# shellcheck disable=SC2016
|
||||
sed -i '/if \[\[ "$1" == tee/i if [[ "$*" == *"apt-get install -y google-chrome-stable"* ]]; then touch "$CHROME_MARKER"; fi' "$scratch/bin/sudo"
|
||||
chmod +x "$scratch/bin/"*
|
||||
: > "$INSTALL_LOG"
|
||||
bash "$root/src/scripts/install_chrome.sh" >/dev/null
|
||||
grep -q 'Dir::Etc::sourcelist=/etc/apt/sources.list.d/qqq-google-chrome.list' "$INSTALL_LOG"
|
||||
grep -q 'gpg --batch --yes --dearmor -o /usr/share/keyrings/qqq-google-chrome.gpg' "$INSTALL_LOG"
|
||||
grep -q 'signed-by=/usr/share/keyrings/qqq-google-chrome.gpg] https://dl.google.com/linux/chrome/deb/' "$INSTALL_LOG"
|
||||
grep -q 'apt-get install -y google-chrome-stable' "$INSTALL_LOG"
|
||||
: > "$INSTALL_LOG"
|
||||
bash "$root/src/scripts/install_chrome.sh" >/dev/null
|
||||
[[ ! -s "$INSTALL_LOG" ]] # Existing Chrome needs no network/package work.
|
||||
rm "$CHROME_MARKER"
|
||||
for script in install_chrome frontend_test_dependencies; do
|
||||
: > "$INSTALL_LOG"
|
||||
if REQUIRED_REPO_FAIL=1 bash "$root/src/scripts/$script.sh" >/dev/null 2>&1; then
|
||||
echo "Required repository outage accepted: $script" >&2; exit 1
|
||||
fi
|
||||
if grep -q 'apt-get install' "$INSTALL_LOG"; then exit 1; fi
|
||||
done
|
||||
: > "$INSTALL_LOG"
|
||||
if KEY_FAIL=1 bash "$root/src/scripts/install_chrome.sh" >/dev/null 2>&1; then
|
||||
echo 'Signing key failure accepted' >&2; exit 1
|
||||
fi
|
||||
if grep -q 'apt-get' "$INSTALL_LOG"; then exit 1; fi
|
||||
echo 'PASS: Chrome ignores unrelated PPAs; required repositories and signing key fail closed; existing browser reused'
|
||||
@@ -29,11 +29,14 @@ class BusinessE2ETests(unittest.TestCase):
|
||||
self.assertEqual('always()', self.worker['jobs']['e2e']['steps'][-1]['if'])
|
||||
self.assertEqual('docker compose -f deploy/compose/docker-compose.yml down -v', self.worker['jobs']['e2e']['steps'][-1]['run'])
|
||||
|
||||
def test_events_permissions_path_and_single_concurrency_owner_preserved(self):
|
||||
self.assertEqual(self.original['events'], self.caller['on'])
|
||||
def test_code_changes_and_manual_runs_keep_full_gate_without_nightly_duplicates(self):
|
||||
expected_events = copy.deepcopy(self.original['events'])
|
||||
del expected_events['schedule']
|
||||
self.assertEqual(expected_events, self.caller['on'])
|
||||
self.assertEqual(self.original['workflow']['permissions'], self.caller['permissions'])
|
||||
self.assertEqual(self.caller['permissions'], self.worker['permissions'])
|
||||
self.assertEqual(self.original['workflow']['concurrency'], self.caller['concurrency'])
|
||||
self.assertEqual(self.original['workflow']['concurrency']['group'], self.caller['concurrency']['group'])
|
||||
self.assertEqual("${{ github.event_name == 'pull_request' }}", self.caller['concurrency']['cancel-in-progress'])
|
||||
self.assertNotIn('concurrency', self.worker)
|
||||
self.assertEqual('.github/workflows/e2e.yml', aux.BRIDGES['business-e2e']['path'])
|
||||
self.assertEqual({'workflow_call': {'secrets': {'E2E_ALERT_WEBHOOK_URL': {'required': False}}}}, self.worker['on'])
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""CI maintenance retains Maven verification and never becomes a release request."""
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location('pipeline', ROOT/'src/scripts/generate_pipeline.py')
|
||||
pipeline = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pipeline)
|
||||
|
||||
|
||||
class MaintenanceTests(unittest.TestCase):
|
||||
def git_result(self, *args):
|
||||
return {'log': self.subject, 'rev-list': self.parents, 'diff': self.paths}.get(args[0])
|
||||
|
||||
def setUp(self):
|
||||
self.subject = b'ci: use central pipeline\n'
|
||||
self.parents = b'a'*40 + b' ' + b'b'*40 + b'\n'
|
||||
self.paths = b'.qqq-ci.yml\0.circleci/config.yml\0.github/workflows/codeql.yml\0.github/workflows/release-rc.yml\0'
|
||||
|
||||
def test_publishing_branches_keep_unfiltered_maven_tests(self):
|
||||
manifest = {'profile': 'qbit', 'orb_version': '0.7.14', 'default_branch': 'develop',
|
||||
'release_on_main': True, 'test_branch_policy': 'feature-no-hotfix',
|
||||
'publish': {'hotfix': True}}
|
||||
with mock.patch.object(Path, 'read_text', return_value=yaml.safe_dump(manifest)):
|
||||
config = pipeline.load_manifest('unused', '0.7.14')
|
||||
with mock.patch.object(pipeline, 'git', side_effect=self.git_result):
|
||||
for branch in ('main', 'develop', 'release/0.1', 'hotfix/1.2.3'):
|
||||
with self.subTest(branch=branch):
|
||||
result = pipeline.qbit_template(config, branch, '')
|
||||
job = result['workflows']['test_only']['jobs'][0]['qqq-orb/mvn_test_only']
|
||||
self.assertEqual(job['filters']['branches'], {'ignore': 'automation/qbit-rc-pending'})
|
||||
self.assertNotIn('branch_type', job)
|
||||
|
||||
def test_nonpublishing_branches_verify_maintenance_but_keep_application_policy(self):
|
||||
manifest = {'profile': 'qbit', 'orb_version': '0.8', 'default_branch': 'main',
|
||||
'release_on_main': False, 'test_branch_policy': 'feature-no-hotfix',
|
||||
'publish': {'snapshot': False, 'release_candidate': False, 'hotfix': False}}
|
||||
with mock.patch.object(Path, 'read_text', return_value=yaml.safe_dump(manifest)):
|
||||
config = pipeline.load_manifest('unused', '0.8')
|
||||
maintenance_paths = self.paths
|
||||
with mock.patch.object(pipeline, 'git', side_effect=self.git_result):
|
||||
for branch in ('main', 'develop', 'release/0.1', 'hotfix/1.2.3'):
|
||||
for maintenance in (True, False):
|
||||
with self.subTest(branch=branch, maintenance=maintenance):
|
||||
self.paths = maintenance_paths if maintenance else b'src/Main.java\0'
|
||||
result = pipeline.qbit_template(config, branch, '')
|
||||
job = result['workflows']['test_only']['jobs'][0]['qqq-orb/mvn_test_only']
|
||||
ignored = job['filters']['branches']['ignore']
|
||||
if maintenance:
|
||||
self.assertEqual(ignored, 'automation/qbit-rc-pending')
|
||||
else:
|
||||
self.assertEqual(ignored, ['automation/qbit-rc-pending',
|
||||
'/(develop|main|release/.*|hotfix/.*|integration.*)/'])
|
||||
self.assertNotIn('branch_type', job)
|
||||
|
||||
def test_frontend_maintenance_verifies_without_publishing(self):
|
||||
for profile, prefix in (('frontend-core', 'node_'), ('frontend-material', 'mvn_frontend_')):
|
||||
manifest = {'profile': profile, 'orb_version': '0.8', 'default_branch': 'develop'}
|
||||
with mock.patch.object(Path, 'read_text', return_value=yaml.safe_dump(manifest)):
|
||||
config = pipeline.load_manifest('unused', '0.8')
|
||||
for branch in ('main', 'develop', 'release/1.0', 'hotfix/1.0.1'):
|
||||
with self.subTest(profile=profile, branch=branch), mock.patch.object(pipeline, 'git', side_effect=self.git_result):
|
||||
result = pipeline.frontend_template(config, branch, '')
|
||||
job = result['workflows']['test_only']['jobs'][0]['qqq-orb/' + prefix + 'test_only']
|
||||
self.assertNotIn('branch_type', job)
|
||||
self.assertEqual(job['filters']['branches'], {'ignore': 'automation/qbit-rc-pending'})
|
||||
self.paths = b'package.json\0'
|
||||
with mock.patch.object(pipeline, 'git', side_effect=self.git_result):
|
||||
result = pipeline.frontend_template(config, 'develop', '')
|
||||
self.assertIn('qqq-orb/' + prefix + 'publish', result['workflows']['publish_snapshot']['jobs'][0])
|
||||
self.paths = b'.qqq-ci.yml\0'
|
||||
with mock.patch.object(pipeline, 'git', return_value=b'verified-head\n'):
|
||||
result = pipeline.frontend_template(config, 'main', 'v1.2.3')
|
||||
job = result['workflows']['publish_release']['jobs'][0]['qqq-orb/' + prefix + 'publish']
|
||||
self.assertEqual(job['branch_type'], 'release')
|
||||
|
||||
def test_application_changes_and_unknown_files_publish_normally(self):
|
||||
for paths in (b'pom.xml\0', b'.qqq-ci.yml\0src/Main.java\0', b'.github/workflows/custom-release.yml\0', b'', None):
|
||||
self.paths = paths
|
||||
with mock.patch.object(pipeline, 'git', side_effect=self.git_result):
|
||||
self.assertFalse(pipeline.proven_ci_maintenance('release/0.1', ''))
|
||||
|
||||
def test_schema_caller_migration_is_maintenance_but_schema_inputs_are_not(self):
|
||||
self.paths = b'.github/workflows/schema-drift.yml\0'
|
||||
with mock.patch.object(pipeline, 'git', side_effect=self.git_result):
|
||||
self.assertTrue(pipeline.proven_ci_maintenance('release/0.1', ''))
|
||||
for path in (b'pom.xml\0', b'src/test/resources/schema.yml\0', b'src/test/java/SchemaDriftTest.java\0'):
|
||||
self.paths = b'.github/workflows/schema-drift.yml\0' + path
|
||||
self.assertFalse(pipeline.proven_ci_maintenance('release/0.1', ''))
|
||||
|
||||
def test_merge_root_and_unavailable_history_do_not_suppress_release(self):
|
||||
for parents in (None, b'a'*40+b'\n', self.parents.rstrip()+b' '+b'c'*40+b'\n'):
|
||||
self.parents = parents
|
||||
with mock.patch.object(pipeline, 'git', side_effect=self.git_result):
|
||||
self.assertFalse(pipeline.proven_ci_maintenance('release/0.1', ''))
|
||||
|
||||
def test_tags_and_release_requests_do_not_suppress_release(self):
|
||||
with mock.patch.object(pipeline, 'git', side_effect=self.git_result):
|
||||
self.assertFalse(pipeline.proven_ci_maintenance('main', 'v1.2.3'))
|
||||
for subject in (b'Merge develop for next RC\n', b'fix: application\n', None):
|
||||
self.subject = subject
|
||||
self.assertFalse(pipeline.proven_ci_maintenance('release/0.1', ''))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Discovery covers all current branches and never turns an unknown into a target."""
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
import discover_enrollment as discovery
|
||||
|
||||
|
||||
def entry(name, archived=False):
|
||||
return {'nameWithOwner': 'QRun-IO/' + name, 'isArchived': archived}
|
||||
|
||||
|
||||
class DiscoveryTests(unittest.TestCase):
|
||||
def test_paginates_branches_and_ignores_historical_releases(self):
|
||||
requests = []
|
||||
|
||||
def request(path):
|
||||
requests.append(path)
|
||||
if path.endswith('page=1'):
|
||||
return [{'name': 'release/' + str(i)} for i in range(99)] + [{'name': 'main'}]
|
||||
return [{'name': 'develop'}]
|
||||
|
||||
inspect = mock.Mock(side_effect=lambda target, version: {**target, 'version': version})
|
||||
plans, findings = discovery.discover('QRun-IO', '0.8', inspect, request, [entry('example')])
|
||||
self.assertEqual(['develop', 'main'], [p['branch'] for p in plans])
|
||||
self.assertEqual(2, len(requests))
|
||||
self.assertEqual([], findings)
|
||||
|
||||
def test_unknown_branch_and_profile_stay_unresolved_while_other_targets_progress(self):
|
||||
def inspect(target, version):
|
||||
if target['repository'].endswith('/custom'):
|
||||
raise ValueError('Unknown orchestration')
|
||||
return target
|
||||
|
||||
def request(path):
|
||||
return [{'name': 'feature/new'}] if '/new/' in path else [{'name': 'main'}]
|
||||
|
||||
plans, findings = discovery.discover('QRun-IO', '0.8', inspect, request,
|
||||
[entry('custom'), entry('new'), entry('good')])
|
||||
self.assertEqual([{'repository': 'QRun-IO/good', 'branch': 'main'}], plans)
|
||||
self.assertEqual(2, len(findings))
|
||||
self.assertTrue(all(f['state'] == 'unresolved' for f in findings))
|
||||
|
||||
def test_archived_and_owner_repositories_are_never_modified(self):
|
||||
inspect, request = mock.Mock(), mock.Mock()
|
||||
plans, findings = discovery.discover('QRun-IO', '0.8', inspect, request,
|
||||
[entry('old', True), entry('qqq-orb')])
|
||||
self.assertEqual([], plans)
|
||||
self.assertEqual(['archived', 'owner-bootstrap'], [f['state'] for f in findings])
|
||||
inspect.assert_not_called()
|
||||
request.assert_not_called()
|
||||
|
||||
def test_api_errors_and_duplicate_branches_never_become_empty_success(self):
|
||||
for result in (ValueError('GitHub read unavailable'), [{'name': 'main'}, {'name': 'main'}]):
|
||||
request = mock.Mock(side_effect=result) if isinstance(result, Exception) else mock.Mock(return_value=result)
|
||||
inspect = mock.Mock()
|
||||
plans, findings = discovery.discover('QRun-IO', '0.8', inspect, request, [entry('example')])
|
||||
self.assertEqual([], plans)
|
||||
self.assertEqual('unresolved', findings[0]['state'])
|
||||
inspect.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise the real version script and Maven install; never publish remotely."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NS = {'m': 'http://maven.apache.org/POM/4.0.0'}
|
||||
BASH = os.environ.get('TEST_BASH') or shutil.which('bash')
|
||||
MAVEN = shutil.which('mvn') or shutil.which('mvn.cmd')
|
||||
|
||||
|
||||
def run(args, directory, env=None):
|
||||
subprocess.run(args, cwd=directory, env=env, check=True, timeout=300)
|
||||
|
||||
|
||||
def project(body):
|
||||
return '<project xmlns="http://maven.apache.org/POM/4.0.0">\n<modelVersion>4.0.0</modelVersion>\n' + body + '\n</project>\n'
|
||||
|
||||
|
||||
def main():
|
||||
assert BASH and MAVEN, 'Real Bash, Java and Maven are required'
|
||||
with tempfile.TemporaryDirectory(prefix='qqq-version-') as temporary:
|
||||
base = Path(temporary)
|
||||
for name, version, revision, branch, expected, reactor in (
|
||||
('literal', '0.4.0-SNAPSHOT', '0.4.0-RC.1', 'release/0.4', '0.4.0-RC.2', False),
|
||||
('friendly', '${revision}', '0.4.0-RC.1', 'release/0.4', '0.4.0-RC.2', False),
|
||||
('reactor', '0.4.0-SNAPSHOT', '0.4.0-RC.1', 'release/0.4', '0.4.0-RC.2', True),
|
||||
('unchanged', '0.3.0-SNAPSHOT', '0.4.0-SNAPSHOT', 'develop', '0.4.0-SNAPSHOT', False),
|
||||
):
|
||||
root = base / name
|
||||
root.mkdir()
|
||||
modules = '<modules><module>child</module></modules>' if reactor else ''
|
||||
(root / 'pom.xml').write_text(project(f'''<groupId>example.version</groupId>
|
||||
<artifactId>{name}</artifactId><version>{version}</version><packaging>pom</packaging>
|
||||
<properties>
|
||||
<revision>{revision}</revision>
|
||||
</properties>
|
||||
{modules}'''), encoding='utf-8')
|
||||
if reactor:
|
||||
(root / 'child').mkdir()
|
||||
(root / 'child/pom.xml').write_text(project(f'''<parent><groupId>example.version</groupId>
|
||||
<artifactId>{name}</artifactId><version>{version}</version></parent>
|
||||
<artifactId>child</artifactId><packaging>pom</packaging>'''), encoding='utf-8')
|
||||
run(['git', 'init', '-q'], root)
|
||||
env = dict(os.environ, CIRCLE_BRANCH=branch, CIRCLE_TAG='')
|
||||
run([BASH, str(ROOT / 'src/scripts/calculate_version.sh')], root, env)
|
||||
if name == 'friendly':
|
||||
assert ET.parse(root / 'pom.xml').findtext('m:version', namespaces=NS) == '${revision}'
|
||||
# Install to a temporary repository so the actual artifact coordinate
|
||||
# proves the fix, rather than merely checking the revision property.
|
||||
repository = base / 'repository'
|
||||
run([MAVEN, '-B', '-ntp', '-Dmaven.repo.local=' + str(repository),
|
||||
'-Dflatten.mode=resolveCiFriendliesOnly', '-DupdatePomFile=true',
|
||||
'org.codehaus.mojo:flatten-maven-plugin:1.7.3:flatten', 'install'], root)
|
||||
for artifact in ([name, 'child'] if reactor else [name]):
|
||||
installed = repository / 'example/version' / artifact / expected / f'{artifact}-{expected}.pom'
|
||||
assert installed.exists(), installed
|
||||
pom = ET.parse(installed)
|
||||
actual = pom.findtext('m:version', namespaces=NS) or pom.findtext('m:parent/m:version', namespaces=NS)
|
||||
assert actual == expected, (artifact, actual, expected)
|
||||
print(f'PASS: {name} installed at {expected}', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Shared enforcement only adds proven gates and preserves unrelated policy."""
|
||||
import copy
|
||||
import base64
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
import yaml
|
||||
|
||||
spec = importlib.util.spec_from_file_location('enforcement', Path(__file__).resolve().parents[1] / 'tools/enforce_orb_checks.py')
|
||||
enforcement = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(enforcement)
|
||||
|
||||
|
||||
def runs():
|
||||
return [{'id': index, 'name': name, 'app': {'id': 18001}, 'conclusion': 'success'}
|
||||
for index, name in enumerate(('setup', 'verify'), 1)]
|
||||
|
||||
|
||||
class EnforcementTests(unittest.TestCase):
|
||||
def test_check_comparison_ignores_only_order(self):
|
||||
checks = [*enforcement.DESIRED, {'context': 'scanner', 'app_id': 15368}]
|
||||
self.assertTrue(enforcement.same_checks(checks, list(reversed(checks))))
|
||||
self.assertFalse(enforcement.same_checks(checks, checks[:-1]))
|
||||
self.assertFalse(enforcement.same_checks(checks, checks + checks[:1]))
|
||||
self.assertFalse(enforcement.same_checks(checks, [*enforcement.DESIRED, {'context': 'scanner', 'app_id': 999}]))
|
||||
|
||||
def test_preserves_security_gate_and_binds_orb_checks(self):
|
||||
before = {'required_status_checks': {'strict': False, 'contexts': ['security', 'setup'],
|
||||
'checks': [{'context': 'security', 'app_id': 15368}, {'context': 'setup', 'app_id': -1}]}}
|
||||
original = copy.deepcopy(before)
|
||||
result = enforcement.required_checks(before, runs())
|
||||
self.assertEqual(original, before)
|
||||
self.assertEqual(result, {'strict': True, 'checks': [{'context': 'security', 'app_id': 15368}, *enforcement.DESIRED]})
|
||||
|
||||
def test_wrong_app_failed_latest_and_missing_checks_reject(self):
|
||||
cases = [[], [runs()[0]], [dict(run, app={'id': 999}) for run in runs()],
|
||||
runs() + [dict(runs()[1], id=3, conclusion='failure')]]
|
||||
for candidate in cases:
|
||||
with self.subTest(candidate=candidate), self.assertRaises(ValueError):
|
||||
enforcement.required_checks(None, candidate)
|
||||
with self.assertRaises(ValueError):
|
||||
enforcement.required_checks({'required_status_checks': {'checks': [{'context': 'verify', 'app_id': 999}]}}, runs())
|
||||
|
||||
def test_existing_unrestricted_contexts_are_retained(self):
|
||||
result = enforcement.required_checks({'required_status_checks': {'contexts': ['legacy-build']}}, runs())
|
||||
self.assertIn({'context': 'legacy-build', 'app_id': -1}, result['checks'])
|
||||
|
||||
def fixture(self, before):
|
||||
state = {'before': copy.deepcopy(before), 'head': 'a' * 40, 'writes': []}
|
||||
def request(path, method='GET', payload=None):
|
||||
if method != 'GET':
|
||||
state['writes'].append((method, path, payload))
|
||||
if method == 'PUT':
|
||||
state['before'] = {**payload, 'enforce_admins': {'enabled': True}}
|
||||
elif method == 'PATCH':
|
||||
state['before']['required_status_checks'] = copy.deepcopy(payload)
|
||||
elif method == 'POST':
|
||||
state['before']['enforce_admins'] = {'enabled': True}
|
||||
return None
|
||||
if '/rules/' in path:
|
||||
return []
|
||||
if path.endswith('/protection'):
|
||||
return copy.deepcopy(state['before'])
|
||||
return {'commit': {'sha': state['head']}}
|
||||
plan = {'repository': 'QRun-IO/example', 'branch': 'develop', 'head': 'a' * 40,
|
||||
'before': copy.deepcopy(before), 'required_status_checks': enforcement.required_checks(before, runs()), 'changed': True}
|
||||
return plan, request, state
|
||||
|
||||
def test_narrow_updates_preserve_reviews_signatures_and_restrictions(self):
|
||||
before = {'required_status_checks': {'strict': False, 'checks': []}, 'enforce_admins': {'enabled': False},
|
||||
'required_signatures': {'enabled': True}, 'required_linear_history': {'enabled': True},
|
||||
'required_pull_request_reviews': {'required_approving_review_count': 2},
|
||||
'restrictions': {'users': [{'login': 'owner'}]}}
|
||||
plan, request, state = self.fixture(before)
|
||||
enforcement.enforce(plan, request)
|
||||
self.assertEqual(['PATCH', 'POST'], [write[0] for write in state['writes']])
|
||||
for key in ('required_signatures', 'required_linear_history', 'required_pull_request_reviews', 'restrictions'):
|
||||
self.assertEqual(before[key], state['before'][key])
|
||||
|
||||
def test_new_protection_and_idempotent_noop(self):
|
||||
plan, request, state = self.fixture(None)
|
||||
enforcement.enforce(plan, request)
|
||||
self.assertEqual('PUT', state['writes'][0][0])
|
||||
self.assertIsNone(state['before']['required_pull_request_reviews'])
|
||||
plan['before'] = copy.deepcopy(state['before'])
|
||||
plan['changed'] = False
|
||||
state['writes'] = []
|
||||
enforcement.enforce(plan, request)
|
||||
self.assertEqual([], state['writes'])
|
||||
|
||||
def test_github_may_reorder_required_checks_on_readback(self):
|
||||
before = {'required_status_checks': {'strict': True, 'checks': [
|
||||
*enforcement.DESIRED, {'context': 'scanner', 'app_id': 15368}]},
|
||||
'enforce_admins': {'enabled': False}, 'required_signatures': {'enabled': True}}
|
||||
plan, request, state = self.fixture(before)
|
||||
|
||||
def reordered(path, method='GET', payload=None):
|
||||
value = request(path, method, payload)
|
||||
if method == 'PATCH':
|
||||
state['before']['required_status_checks']['checks'].reverse()
|
||||
return value
|
||||
|
||||
enforcement.enforce(plan, reordered)
|
||||
self.assertTrue(state['before']['enforce_admins']['enabled'])
|
||||
self.assertEqual(before['required_signatures'], state['before']['required_signatures'])
|
||||
|
||||
def test_head_or_protection_drift_never_mutates(self):
|
||||
for field, value in [('head', 'b' * 40), ('before', {'unexpected': True})]:
|
||||
plan, request, state = self.fixture(None)
|
||||
state[field] = value
|
||||
with self.assertRaises(ValueError):
|
||||
enforcement.enforce(plan, request)
|
||||
self.assertEqual([], state['writes'])
|
||||
|
||||
def ruleset_fixture(self):
|
||||
parameters = {'strict_required_status_checks_policy': False,
|
||||
'required_status_checks': [{'context': c['context'], 'integration_id': c['app_id']}
|
||||
for c in enforcement.DESIRED] + [{'context': 'scanner', 'integration_id': 15368}]}
|
||||
record = {'id': 123, 'enforcement': 'active', 'target': 'branch', 'source_type': 'Repository',
|
||||
'source': 'QRun-IO/example', 'bypass_actors': [],
|
||||
'rules': [{'type': 'required_status_checks', 'parameters': parameters}]}
|
||||
effective = [{'type': 'required_status_checks', 'ruleset_source_type': 'Repository',
|
||||
'ruleset_source': 'QRun-IO/example', 'ruleset_id': 123, 'parameters': copy.deepcopy(parameters)}]
|
||||
return record, effective
|
||||
|
||||
def test_ruleset_verification_preserves_freshness_scanner_and_never_writes(self):
|
||||
record, effective = self.ruleset_fixture()
|
||||
original = copy.deepcopy(record)
|
||||
|
||||
def request(path, method='GET', payload=None):
|
||||
self.assertEqual('GET', method)
|
||||
if path.endswith('/rulesets/123'):
|
||||
return copy.deepcopy(record)
|
||||
if '/rules/branches/' in path:
|
||||
return copy.deepcopy(effective)
|
||||
if path.endswith('/protection'):
|
||||
return None
|
||||
if '/check-runs?' in path:
|
||||
return {'check_runs': runs()}
|
||||
if '/contents/' in path:
|
||||
data = ({'profile': 'qbit', 'orb_version': '0.8', 'default_branch': 'develop'}
|
||||
if '/.qqq-ci.yml?' in path else enforcement.standard.canonical_caller('qbit', '0.8'))
|
||||
return {'encoding': 'base64', 'type': 'file', 'content': base64.b64encode(yaml.safe_dump(data).encode()).decode()}
|
||||
if '/branches/' in path:
|
||||
return {'commit': {'sha': 'a' * 40}}
|
||||
return {'archived': False}
|
||||
|
||||
plan = enforcement.inspect('QRun-IO/example', 'develop', '0.8', request)
|
||||
self.assertFalse(plan['changed'])
|
||||
self.assertFalse(plan['required_status_checks']['strict'])
|
||||
self.assertEqual('rulesets', plan['protection_mode'])
|
||||
enforcement.enforce(plan, request)
|
||||
self.assertEqual(original, record)
|
||||
record['bypass_actors'] = [{'actor_type': 'RepositoryRole', 'actor_id': 5}]
|
||||
with self.assertRaises(ValueError):
|
||||
enforcement.enforce(plan, request)
|
||||
|
||||
def test_inactive_bypass_unbound_and_missing_ruleset_gates_reject(self):
|
||||
record, effective = self.ruleset_fixture()
|
||||
cases = [dict(record, enforcement='evaluate'), dict(record, bypass_actors=[{'actor_id': 1}]),
|
||||
dict(record, rules=[])]
|
||||
unbound = copy.deepcopy(record)
|
||||
unbound['rules'][0]['parameters']['required_status_checks'][1]['integration_id'] = None
|
||||
cases.append(unbound)
|
||||
for candidate in cases:
|
||||
with self.subTest(candidate=candidate), self.assertRaises(ValueError):
|
||||
enforcement.verified_rulesets('QRun-IO/example', effective, lambda path: candidate)
|
||||
|
||||
def test_unsupported_ruleset_sources_are_not_inferred_as_protected(self):
|
||||
record, effective = self.ruleset_fixture()
|
||||
effective[0]['ruleset_source_type'] = 'Organization'
|
||||
with self.assertRaises(ValueError):
|
||||
enforcement.verified_rulesets('QRun-IO/example', effective, lambda path: record)
|
||||
|
||||
def test_empty_push_allowlist_gains_ruleset_without_changing_access(self):
|
||||
classic = {'restrictions': {'users': [], 'teams': [], 'apps': []},
|
||||
'enforce_admins': {'enabled': False}, 'required_signatures': {'enabled': True}}
|
||||
state = {'ruleset': None, 'writes': []}
|
||||
|
||||
def request(path, method='GET', payload=None):
|
||||
if method != 'GET':
|
||||
self.assertEqual(('POST', 'repos/QRun-IO/example/rulesets'), (method, path))
|
||||
state['writes'].append(copy.deepcopy(payload))
|
||||
state['ruleset'] = {**copy.deepcopy(payload), 'id': 123,
|
||||
'source_type': 'Repository', 'source': 'QRun-IO/example'}
|
||||
return state['ruleset']
|
||||
if '/rulesets?' in path:
|
||||
return []
|
||||
if path.endswith('/rulesets/123'):
|
||||
return copy.deepcopy(state['ruleset'])
|
||||
if '/rules/branches/' in path:
|
||||
if state['ruleset'] is None:
|
||||
return []
|
||||
return [{**state['ruleset']['rules'][0], 'ruleset_source_type': 'Repository',
|
||||
'ruleset_source': 'QRun-IO/example', 'ruleset_id': 123}]
|
||||
if path.endswith('/protection'):
|
||||
return copy.deepcopy(classic)
|
||||
if '/check-runs?' in path:
|
||||
return {'check_runs': runs()}
|
||||
if '/contents/' in path:
|
||||
data = ({'profile': 'core', 'orb_version': '0.8', 'default_branch': 'develop'}
|
||||
if '/.qqq-ci.yml?' in path else enforcement.standard.canonical_caller('core', '0.8'))
|
||||
return {'encoding': 'base64', 'type': 'file', 'content': base64.b64encode(yaml.safe_dump(data).encode()).decode()}
|
||||
if '/branches/' in path:
|
||||
return {'commit': {'sha': 'a' * 40}}
|
||||
return {'archived': False}
|
||||
|
||||
plan = enforcement.inspect('QRun-IO/example', 'main', '0.8', request)
|
||||
self.assertEqual('create-ruleset', plan['protection_mode'])
|
||||
enforcement.enforce(plan, request)
|
||||
self.assertEqual(1, len(state['writes']))
|
||||
self.assertEqual([], state['writes'][0]['bypass_actors'])
|
||||
self.assertEqual({'include': ['refs/heads/main'], 'exclude': []}, state['writes'][0]['conditions']['ref_name'])
|
||||
self.assertFalse(plan['before']['enforce_admins']['enabled'])
|
||||
self.assertEqual(classic, plan['before'])
|
||||
repeat = enforcement.inspect('QRun-IO/example', 'main', '0.8', request)
|
||||
self.assertFalse(repeat['changed'])
|
||||
enforcement.enforce(repeat, request)
|
||||
self.assertEqual(1, len(state['writes']))
|
||||
|
||||
def test_existing_named_ruleset_is_not_overwritten_or_duplicated(self):
|
||||
before = {'restrictions': {'users': [], 'teams': [], 'apps': []}, 'enforce_admins': {'enabled': False}}
|
||||
plan, base_request, state = self.fixture(before)
|
||||
plan['protection_mode'] = 'create-ruleset'
|
||||
|
||||
def request(path, method='GET', payload=None):
|
||||
if '/rulesets?' in path:
|
||||
return [{'name': 'QQQ automated verification: develop'}]
|
||||
return base_request(path, method, payload)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, 'already exists'):
|
||||
enforcement.enforce(plan, request)
|
||||
self.assertEqual([], state['writes'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Batch enrollment preserves policy and rejects unsafe or unknown declarations."""
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
import yaml
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
import enroll_consumers as enrollment
|
||||
|
||||
|
||||
class EnrollmentTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
|
||||
def legacy(self, name):
|
||||
root = self.root / name
|
||||
(root / '.circleci').mkdir(parents=True)
|
||||
config = {'version': 2.1, 'orbs': {'qqq-orb': 'kingsrook/qqq-orb@0.5.0'},
|
||||
'workflows': {'verify': {'jobs': [{'qqq-orb/mvn_test_only': {}}]}}}
|
||||
(root / '.circleci/config.yml').write_text(yaml.safe_dump(config), encoding='utf-8')
|
||||
return {'checkout': str(root), 'default_branch': 'main'}
|
||||
|
||||
def canonical(self, profile):
|
||||
target = self.legacy(profile)
|
||||
root = Path(target['checkout'])
|
||||
data = {'profile': profile, 'orb_version': '0.7.4', 'default_branch': 'main'}
|
||||
(root / '.qqq-ci.yml').write_text(yaml.safe_dump(data), encoding='utf-8')
|
||||
(root / '.circleci/config.yml').write_text(yaml.safe_dump(
|
||||
enrollment.rollout.standard.canonical_caller(profile, '0.7.4')), encoding='utf-8')
|
||||
return target
|
||||
|
||||
def test_mixed_fleet_preview_write_and_idempotence(self):
|
||||
legacy = self.legacy('legacy')
|
||||
targets = [legacy] + [self.canonical(p) for p in enrollment.rollout.standard.PROFILE_PARAMETERS]
|
||||
preview = enrollment.enroll(targets, '0.8')
|
||||
self.assertTrue(all(item['changed'] for item in preview))
|
||||
self.assertFalse((Path(legacy['checkout']) / '.qqq-ci.yml').exists())
|
||||
result = enrollment.enroll(targets, '0.8', write=True)
|
||||
self.assertEqual(1, sum(item['enrollment'] for item in result))
|
||||
data = yaml.safe_load((Path(legacy['checkout']) / '.qqq-ci.yml').read_text())
|
||||
self.assertFalse(any(data['publish'].values()))
|
||||
self.assertEqual('all', data['test_branch_policy'])
|
||||
self.assertTrue(all(not item['changed'] for item in enrollment.enroll(targets, '0.8', write=True)))
|
||||
|
||||
def test_unknown_orchestration_rejects_entire_batch(self):
|
||||
good, bad = self.legacy('good'), self.legacy('bad')
|
||||
path = Path(bad['checkout']) / '.circleci/config.yml'
|
||||
path.write_text(path.read_text() + '\njobs: {custom: {steps: [checkout]}}\n')
|
||||
original = (Path(good['checkout']) / '.circleci/config.yml').read_bytes()
|
||||
with self.assertRaises(ValueError):
|
||||
enrollment.enroll([good, bad], '0.8', write=True)
|
||||
self.assertEqual(original, (Path(good['checkout']) / '.circleci/config.yml').read_bytes())
|
||||
self.assertFalse((Path(good['checkout']) / '.qqq-ci.yml').exists())
|
||||
|
||||
def test_duplicate_targets_and_wrong_default_reject_before_write(self):
|
||||
target = self.canonical('core')
|
||||
for targets in ([target, target], [{**target, 'default_branch': 'develop'}]):
|
||||
with self.assertRaises(ValueError):
|
||||
enrollment.enroll(targets, '0.8', write=True)
|
||||
self.assertEqual('0.7.4', yaml.safe_load((Path(target['checkout']) / '.qqq-ci.yml').read_text())['orb_version'])
|
||||
|
||||
def test_invalid_manifest_and_unstable_versions_rejected(self):
|
||||
target = self.canonical('qbit')
|
||||
path = Path(target['checkout']) / '.qqq-ci.yml'
|
||||
path.write_text(path.read_text() + '\nunknown_policy: true\n')
|
||||
for version in ('0.8', 'dev:snapshot', 'latest', 0.8):
|
||||
with self.assertRaises(ValueError):
|
||||
enrollment.enroll([target], version, write=True)
|
||||
|
||||
def test_drift_after_preflight_is_not_overwritten(self):
|
||||
target = self.legacy('drift')
|
||||
path = Path(target['checkout']) / '.circleci/config.yml'
|
||||
original_plan = enrollment.plan
|
||||
|
||||
def changed_plan(*args):
|
||||
result = original_plan(*args)
|
||||
path.write_bytes(b'concurrent edit\n')
|
||||
return result
|
||||
|
||||
with mock.patch.object(enrollment, 'plan', side_effect=changed_plan):
|
||||
with self.assertRaises(ValueError):
|
||||
enrollment.enroll([target], '0.8', write=True)
|
||||
self.assertEqual(b'concurrent edit\n', path.read_bytes())
|
||||
self.assertFalse((Path(target['checkout']) / '.qqq-ci.yml').exists())
|
||||
|
||||
def test_manifest_symlink_is_rejected(self):
|
||||
target = self.legacy('redirect')
|
||||
outside = self.root / 'outside.yml'
|
||||
outside.write_text('untouched\n')
|
||||
try:
|
||||
(Path(target['checkout']) / '.qqq-ci.yml').symlink_to(outside)
|
||||
except OSError as error:
|
||||
self.skipTest('Symlink creation unavailable: ' + str(error))
|
||||
with self.assertRaises(ValueError):
|
||||
enrollment.enroll([target], '0.8', write=True)
|
||||
self.assertEqual('untouched\n', outside.read_text())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Automatic completion requires qualified source and preserves normal merge policy."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
import finish_enrollment as finish
|
||||
|
||||
|
||||
class FinishTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.plan = {'repository': 'QRun-IO/example', 'branch': 'main', 'head': 'a' * 40,
|
||||
'profile': 'qbit', 'version': '0.8'}
|
||||
self.result = {'state': 'existing-pr', 'head': 'b' * 40, 'url': 'https://github.com/QRun-IO/example/pull/1'}
|
||||
self.pr = {'number': 1, 'state': 'OPEN', 'headRefOid': 'b' * 40,
|
||||
'baseRefOid': 'a' * 40, 'baseRefName': 'main', 'mergeCommit': None}
|
||||
|
||||
def checks(self):
|
||||
return [{'id': i, 'name': name, 'app': {'id': 18001}, 'status': 'completed', 'conclusion': 'success'}
|
||||
for i, name in enumerate(('setup', 'verify'))]
|
||||
|
||||
def test_all_checks_and_statuses_must_pass_and_gates_must_use_circle_app(self):
|
||||
checks = self.checks()
|
||||
checks[1]['app']['id'] = 123
|
||||
checks.append({'id': 3, 'name': 'scanner', 'app': {'id': 15368}, 'status': 'completed', 'conclusion': 'failure'})
|
||||
with mock.patch.object(finish.remote, 'api', side_effect=[{'check_runs': checks},
|
||||
{'statuses': [{'context': 'legacy-test', 'state': 'pending'}]}]):
|
||||
self.assertEqual(['legacy-test', 'scanner', 'verify'], finish.qualification('QRun-IO/example', 'b' * 40))
|
||||
|
||||
def test_failed_or_missing_checks_never_merge(self):
|
||||
with mock.patch.object(finish.remote, 'verify_submission'), mock.patch.object(finish.remote, 'command', return_value=json.dumps(self.pr)) as command, mock.patch.object(finish.remote, 'api', return_value={'commit': {'sha': self.plan['head']}}), mock.patch.object(finish, 'qualification', return_value=['verify']):
|
||||
self.assertEqual('waiting-checks', finish.merge_ready(self.plan, self.result)['state'])
|
||||
self.assertEqual(1, command.call_count)
|
||||
self.assertEqual(['gh', 'pr', 'view'], command.call_args.args[0][:3])
|
||||
|
||||
def test_stale_base_never_merges_even_when_checks_were_green(self):
|
||||
self.pr['baseRefOid'] = 'c' * 40
|
||||
with mock.patch.object(finish.remote, 'verify_submission'), mock.patch.object(finish.remote, 'command', return_value=json.dumps(self.pr)) as command:
|
||||
self.assertEqual('stale-base', finish.merge_ready(self.plan, self.result)['state'])
|
||||
self.assertEqual(1, command.call_count)
|
||||
|
||||
def test_success_uses_exact_head_normal_merge_and_verifies_result(self):
|
||||
calls = []
|
||||
|
||||
def command(args):
|
||||
calls.append(args)
|
||||
if args[2] == 'merge':
|
||||
return ''
|
||||
if len(calls) == 1:
|
||||
return json.dumps(self.pr)
|
||||
return json.dumps({'state': 'MERGED', 'mergeCommit': {'oid': 'd' * 40}})
|
||||
|
||||
def api(path):
|
||||
if '/branches/' in path:
|
||||
return {'commit': {'sha': self.plan['head']}}
|
||||
return {'commit': {'tree': {'sha': 'e' * 40}, 'verification': {'verified': True}}}
|
||||
|
||||
with mock.patch.object(finish.remote, 'verify_submission'), mock.patch.object(finish.remote, 'command', side_effect=command), mock.patch.object(finish.remote, 'api', side_effect=api), mock.patch.object(finish, 'qualification', return_value=[]):
|
||||
result = finish.merge_ready(self.plan, self.result)
|
||||
self.assertEqual('merged', result['state'])
|
||||
self.assertEqual(['gh', 'pr', 'merge', '1', '--repo', 'QRun-IO/example', '--squash', '--match-head-commit', 'b' * 40], calls[1])
|
||||
|
||||
def test_enforcement_waits_for_current_head_and_uses_shared_preserving_writer(self):
|
||||
with mock.patch.object(finish.enforcement, 'inspect', side_effect=ValueError('Current branch has no successful verify')), mock.patch.object(finish.enforcement, 'enforce') as enforce:
|
||||
self.assertEqual('waiting-enforcement', finish.enforce_current(self.plan)['state'])
|
||||
enforce.assert_not_called()
|
||||
current = {'head': 'd' * 40, 'changed': True}
|
||||
with mock.patch.object(finish.enforcement, 'inspect', return_value=current), mock.patch.object(finish.enforcement, 'enforce') as enforce:
|
||||
self.assertEqual('enforced', finish.enforce_current(self.plan)['state'])
|
||||
enforce.assert_called_once_with(current)
|
||||
|
||||
def test_unknown_gate_profile_never_merges_or_enforces(self):
|
||||
self.plan['profile'] = 'all'
|
||||
with mock.patch.object(finish.remote, 'command') as command, mock.patch.object(finish.enforcement, 'inspect') as inspect:
|
||||
self.assertEqual('unsupported-gates', finish.merge_ready(self.plan, self.result)['state'])
|
||||
self.assertEqual('unsupported-gates', finish.enforce_current(self.plan)['state'])
|
||||
command.assert_not_called()
|
||||
inspect.assert_not_called()
|
||||
|
||||
def test_merged_tree_mismatch_is_not_reported_as_success(self):
|
||||
self.pr.update(state='MERGED', mergeCommit={'oid': 'd' * 40})
|
||||
with mock.patch.object(finish.remote, 'verify_submission'), mock.patch.object(finish.remote, 'command', return_value=json.dumps(self.pr)), mock.patch.object(finish.remote, 'api', side_effect=[
|
||||
{'commit': {'tree': {'sha': 'e' * 40}}},
|
||||
{'commit': {'tree': {'sha': 'f' * 40}, 'verification': {'verified': True}}}]):
|
||||
with self.assertRaises(ValueError):
|
||||
finish.merge_ready(self.plan, self.result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
# Exercise the actual test runner with no server, browser, network or Maven work.
|
||||
set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
mkdir "$scratch/bin"
|
||||
export RUN_LOG="$scratch/calls"
|
||||
cat > "$scratch/bin/npm" <<'STUB'
|
||||
#!/bin/bash
|
||||
printf 'npm %s\n' "$*" >> "$RUN_LOG"
|
||||
if [[ "$*" == 'run build' && ${BUILD_FAIL:-0} == 1 ]]; then exit 12; fi
|
||||
STUB
|
||||
for tool in mvn dockerize; do
|
||||
# shellcheck disable=SC2016
|
||||
printf '#!/bin/bash\nprintf "%s %%s\\n" "$*" >> "$RUN_LOG"\n' "$tool" > "$scratch/bin/$tool"
|
||||
done
|
||||
chmod +x "$scratch/bin/"*
|
||||
export PATH="$scratch/bin:$PATH"
|
||||
cd "$scratch"
|
||||
PARAM_BUILD_BEFORE_TEST=false bash "$root/src/scripts/frontend_test_run.sh" >/dev/null
|
||||
if grep -qx 'npm run build' "$RUN_LOG"; then
|
||||
echo 'FAIL: combined job rebuilt the already-built React app' >&2; exit 1
|
||||
fi
|
||||
grep -q '^mvn .* -B test$' "$RUN_LOG"
|
||||
echo 'PASS: combined job retains tests without a duplicate build'
|
||||
|
||||
: > "$RUN_LOG"
|
||||
bash "$root/src/scripts/frontend_test_run.sh" >/dev/null
|
||||
grep -qx 'npm run build' "$RUN_LOG"
|
||||
grep -q '^mvn .* -B test$' "$RUN_LOG"
|
||||
: > "$RUN_LOG"
|
||||
if BUILD_FAIL=1 bash "$root/src/scripts/frontend_test_run.sh" >/dev/null 2>&1; then
|
||||
echo 'FAIL: required standalone build failure ignored' >&2; exit 1
|
||||
fi
|
||||
if grep -q '^mvn ' "$RUN_LOG"; then exit 1; fi
|
||||
echo 'PASS: standalone command still builds and fails before tests when its build fails'
|
||||
+141
-13
@@ -192,14 +192,45 @@ class PipelineTests(unittest.TestCase):
|
||||
return result
|
||||
|
||||
def assert_full(self, result, workflow="test_only"):
|
||||
self.assertEqual(list(result["workflows"]), [workflow])
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
self.assertNotIn("docs_only", result.get("jobs", {}))
|
||||
job, options = next(iter(result["workflows"][workflow]["jobs"][0].items()))
|
||||
job, options = next(iter(result["workflows"]["verify"]["jobs"][0].items()))
|
||||
self.assertEqual(job, "qqq-orb/mvn_test_only" if workflow == "test_only" else "qqq-orb/mvn_publish")
|
||||
if workflow != "test_only":
|
||||
self.assertEqual(options["branch_type"], workflow.removeprefix("publish_"))
|
||||
self.assertEqual(options["context"], ["qqq-maven-registry-credentials"])
|
||||
self.assertFalse(options["compile_before_verify"])
|
||||
return options
|
||||
|
||||
def test_ci_maintenance_source_and_packed_keep_verification(self):
|
||||
self.write_manifest(test_branch_policy="feature-no-hotfix", publish={"hotfix": True})
|
||||
workflow = self.repo / ".github/workflows/release-rc.yml"
|
||||
workflow.parent.mkdir(parents=True)
|
||||
workflow.write_text("# central RC caller migration\n", encoding="utf-8")
|
||||
self.git("add", ".qqq-ci.yml", ".github/workflows/release-rc.yml")
|
||||
self.git("-c", "commit.gpgsign=false", "commit", "-qm", "ci: adopt central pipeline")
|
||||
for branch in ("develop", "release/0.1", "hotfix/0.1.1"):
|
||||
result = self.render(CIRCLE_BRANCH=branch)
|
||||
options = self.assert_full(result)
|
||||
self.assertEqual(options["filters"]["branches"], {"ignore": "automation/qbit-rc-pending"})
|
||||
if PACKED:
|
||||
self.assertEqual(result, self.render(packed=True, CIRCLE_BRANCH=branch))
|
||||
schema = self.repo / ".github/workflows/schema-drift.yml"
|
||||
schema.write_text("# central schema drift caller migration\n", encoding="utf-8")
|
||||
self.git("add", ".github/workflows/schema-drift.yml")
|
||||
self.git("-c", "commit.gpgsign=false", "commit", "-qm", "ci: centralize schema drift")
|
||||
result = self.render(CIRCLE_BRANCH="release/0.1")
|
||||
self.assert_full(result)
|
||||
if PACKED:
|
||||
self.assertEqual(result, self.render(packed=True, CIRCLE_BRANCH="release/0.1"))
|
||||
(self.repo / "App.java").write_text("application change\n", encoding="utf-8")
|
||||
self.git("add", "App.java")
|
||||
self.git("-c", "commit.gpgsign=false", "commit", "-qm", "ci: misleading title")
|
||||
result = self.render(CIRCLE_BRANCH="release/0.1")
|
||||
self.assert_full(result, "publish_release_candidate")
|
||||
if PACKED:
|
||||
self.assertEqual(result, self.render(packed=True, CIRCLE_BRANCH="release/0.1"))
|
||||
|
||||
def test_defaults_and_options(self):
|
||||
options = self.assert_full(self.render())
|
||||
self.assertTrue(options["install_browsers"])
|
||||
@@ -210,6 +241,15 @@ class PipelineTests(unittest.TestCase):
|
||||
self.assertTrue(options["check_middleware_api_versions"])
|
||||
self.assertEqual(options["java_version"], "17")
|
||||
|
||||
def test_minor_track_reaches_continuation_without_changing_jobs(self):
|
||||
expected = self.render()
|
||||
expected['orbs']['qqq-orb'] = 'kingsrook/qqq-orb@0.7'
|
||||
self.write_manifest(orb_version='0.7')
|
||||
self.assertEqual(expected, self.render(QQQ_CI_ORB_VERSION='0.7'))
|
||||
if PACKED:
|
||||
self.assertEqual(expected, self.render(packed=True, QQQ_CI_ORB_VERSION='0.7'))
|
||||
self.render(ok=False, QQQ_CI_ORB_VERSION='0.8')
|
||||
|
||||
def test_branch_matrix(self):
|
||||
for branch, workflow in [("develop", "publish_snapshot"), ("release/1.0", "publish_release_candidate"),
|
||||
("main", "test_only"), ("hotfix/fix", "test_only"),
|
||||
@@ -225,6 +265,46 @@ class PipelineTests(unittest.TestCase):
|
||||
self.write_manifest(publish={"asciidoc": True})
|
||||
self.assertTrue(self.assert_full(self.render(CIRCLE_BRANCH="develop"), "publish_snapshot")["publish_asciidoc"])
|
||||
|
||||
def test_qbit_hotfix_is_opt_in_in_source_and_packed_generator(self):
|
||||
for packed in ([False, True] if PACKED else [False]):
|
||||
self.write_manifest()
|
||||
self.assert_full(self.render(packed=packed, CIRCLE_BRANCH="hotfix/1.2.3"))
|
||||
self.write_manifest(publish={"hotfix": True})
|
||||
options = self.assert_full(self.render(packed=packed, CIRCLE_BRANCH="hotfix/1.2.3"), "publish_hotfix")
|
||||
self.assertNotIn("notify_rc_published", options)
|
||||
self.assert_full(self.render(packed=packed, CIRCLE_BRANCH="main"))
|
||||
self.assert_full(self.render(packed=packed, CIRCLE_BRANCH="hotfix/"))
|
||||
self.write_manifest(publish={"hotfix": "true"})
|
||||
self.render(packed=packed, CIRCLE_BRANCH="hotfix/1.2.3", ok=False)
|
||||
|
||||
def test_qbit_feature_tags_source_and_packed_require_opt_in_and_head(self):
|
||||
self.git("tag", "publish-test")
|
||||
for packed in ([False, True] if PACKED else [False]):
|
||||
self.write_manifest()
|
||||
self.assert_full(self.render(packed=packed, CIRCLE_BRANCH="", CIRCLE_TAG="publish-test"))
|
||||
self.write_manifest(publish={"feature": True})
|
||||
self.assert_full(self.render(packed=packed, CIRCLE_BRANCH="", CIRCLE_TAG="publish-test"), "publish_feature")
|
||||
self.assert_full(self.render(packed=packed, CIRCLE_BRANCH="feature/example"))
|
||||
self.assert_full(self.render(packed=packed, CIRCLE_BRANCH="", CIRCLE_TAG="publish-missing"))
|
||||
self.commit()
|
||||
for packed in ([False, True] if PACKED else [False]):
|
||||
self.assert_full(self.render(packed=packed, CIRCLE_BRANCH="", CIRCLE_TAG="publish-test"))
|
||||
|
||||
def test_repository_policy_source_and_packed_never_select_publication(self):
|
||||
self.write_manifest(profile="repository-policy")
|
||||
for packed in ([False, True] if PACKED else [False]):
|
||||
for branch, tag in (("main", ""), ("develop", ""), ("release/1", ""), ("", "v1.0.0")):
|
||||
result = self.render(packed=packed, CIRCLE_BRANCH=branch, CIRCLE_TAG=tag)
|
||||
self.assertEqual({"verify": {"jobs": ["validate-policy"]}}, result["workflows"])
|
||||
self.assertEqual({"validate-policy"}, set(result["jobs"]))
|
||||
self.assertNotIn("orbs", result)
|
||||
|
||||
def test_stable_tags_require_ascii_digits_in_source_and_packed_generator(self):
|
||||
tag = "v\u0661.\u0662.\u0663"
|
||||
self.git("tag", tag)
|
||||
for packed in ([False, True] if PACKED else [False]):
|
||||
self.assert_full(self.render(packed=packed, CIRCLE_TAG=tag))
|
||||
|
||||
def test_qbit_rc_completion_is_explicit_and_rc_only(self):
|
||||
options = self.assert_full(self.render(CIRCLE_BRANCH="release/1"), "publish_release_candidate")
|
||||
self.assertFalse(options["notify_rc_published"])
|
||||
@@ -298,8 +378,8 @@ class PipelineTests(unittest.TestCase):
|
||||
self.change_docs()
|
||||
for packed in ([False, True] if PACKED else [False]):
|
||||
result = self.render(packed=packed)
|
||||
self.assertEqual(list(result["workflows"]), ["test_only"])
|
||||
options = result["workflows"]["test_only"]["jobs"][0]["docs_only"]
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
options = result["workflows"]["verify"]["jobs"][0]["docs_only"]
|
||||
self.assertEqual(options["name"], "qqq-orb/mvn_test_only")
|
||||
self.assertIn("git diff --check", result["jobs"]["docs_only"]["steps"][1]["run"]["command"])
|
||||
(self.repo / "App.java").write_text("changed\n", encoding="utf-8")
|
||||
@@ -349,9 +429,27 @@ class PipelineTests(unittest.TestCase):
|
||||
self.change_docs()
|
||||
self.assertIn("docs_only", self.render()["jobs"])
|
||||
|
||||
def test_core_snapshot_publisher_waits_for_all_acceptance_and_analysis(self):
|
||||
self.write_manifest(profile="core")
|
||||
result = self.render(CIRCLE_BRANCH="develop")
|
||||
jobs = {options.get("name", kind): options
|
||||
for entry in result["workflows"]["verify"]["jobs"]
|
||||
for kind, options in entry.items()}
|
||||
def ancestors(name, path=()):
|
||||
self.assertNotIn(name, path, "Workflow dependencies must be acyclic")
|
||||
required = jobs[name].get("requires", [])
|
||||
return set(required).union(*(ancestors(parent, path + (name,)) for parent in required))
|
||||
self.assertEqual(ancestors("qqq-orb/mvn_publish"), set(jobs) - {"qqq-orb/mvn_publish"})
|
||||
if PACKED:
|
||||
self.assertEqual(result, self.render(packed=True, CIRCLE_BRANCH="develop"))
|
||||
|
||||
def test_core_workflow_graphs_match_baseline(self):
|
||||
self.write_manifest(profile="core")
|
||||
baseline = yaml.safe_load((ROOT / "tests/fixtures/core_pipeline_baseline.yml").read_text(encoding="utf-8"))
|
||||
# Snapshot publication must now wait for the existing sample and analysis gates.
|
||||
for entry in baseline["workflows"]["publish_snapshot"]["jobs"]:
|
||||
if "qqq-orb/mvn_publish" in entry:
|
||||
entry["qqq-orb/mvn_publish"]["requires"] += ["snapshot_sample_coverage_gate", "qqq-orb/static_analysis"]
|
||||
self.git("tag", "v1.0.0")
|
||||
self.git("tag", "publish-fixture")
|
||||
events = [("feature/test", "", "test_only"), ("develop", "", "publish_snapshot"),
|
||||
@@ -360,8 +458,8 @@ class PipelineTests(unittest.TestCase):
|
||||
for branch, tag, workflow in events:
|
||||
with self.subTest(workflow=workflow):
|
||||
result = self.render(CIRCLE_BRANCH=branch, CIRCLE_TAG=tag)
|
||||
self.assertEqual(list(result["workflows"]), [workflow])
|
||||
actual = result["workflows"][workflow]["jobs"]
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
actual = result["workflows"]["verify"]["jobs"]
|
||||
expected = baseline["workflows"][workflow]["jobs"]
|
||||
self.assertEqual(len(actual), len(expected))
|
||||
for before, after in zip(expected, actual):
|
||||
@@ -422,8 +520,8 @@ class PipelineTests(unittest.TestCase):
|
||||
with self.subTest(branch=branch, enabled=enabled):
|
||||
result = self.render(CIRCLE_BRANCH=branch, QQQ_CI_RUN_PLAYWRIGHT=str(enabled).lower())
|
||||
workflow = base + ("_with_playwright" if enabled else "")
|
||||
self.assertEqual(list(result["workflows"]), [workflow])
|
||||
jobs = result["workflows"][workflow]["jobs"]
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
jobs = result["workflows"]["verify"]["jobs"]
|
||||
self.assertEqual(len(jobs), 2 if enabled else 1)
|
||||
maven = [j for j in jobs if next(iter(j)).startswith("qqq-orb/mvn_frontend_")]
|
||||
self.assertEqual(len(maven), 1)
|
||||
@@ -454,8 +552,10 @@ class PipelineTests(unittest.TestCase):
|
||||
("hotfix/fix", "", "publish_hotfix_release"), ("", "publish-fixture", "publish_feature"),
|
||||
("", "v1.0.0", "publish_release")]:
|
||||
result = self.render(CIRCLE_BRANCH=branch, CIRCLE_TAG=tag)
|
||||
self.assertEqual(list(result["workflows"]), [workflow])
|
||||
options = next(iter(result["workflows"][workflow]["jobs"][0].values()))
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
options = next(iter(result["workflows"]["verify"]["jobs"][0].values()))
|
||||
expected_kind = {"publish_hotfix_release": "hotfix"}.get(workflow, workflow.removeprefix("publish_"))
|
||||
self.assertEqual(options["branch_type"], expected_kind)
|
||||
if profile == "frontend-core":
|
||||
self.assertTrue(options["npm_trusted_publishing"])
|
||||
self.assertEqual(options["node_version"], "24.21")
|
||||
@@ -463,9 +563,9 @@ class PipelineTests(unittest.TestCase):
|
||||
else:
|
||||
self.assertEqual(options["context"], ["qqq-maven-registry-credentials"])
|
||||
main = self.render(CIRCLE_BRANCH="main")
|
||||
self.assertEqual(list(main["workflows"]), ["publish_release" if profile == "frontend-core" else "test_only"])
|
||||
self.assertEqual(list(main["workflows"]), ["verify"])
|
||||
rc_tag = self.render(CIRCLE_BRANCH="", CIRCLE_TAG="v1.0.0-RC.1")
|
||||
self.assertEqual(list(rc_tag["workflows"]), ["publish_release" if profile == "frontend-core" else "test_only"])
|
||||
self.assertEqual(list(rc_tag["workflows"]), ["verify"])
|
||||
|
||||
def test_profile_specific_options_rejected(self):
|
||||
for profile, options in [("qbit", {"node_version": "22"}), ("frontend-core", {"browsers": False}),
|
||||
@@ -523,6 +623,33 @@ class PipelineTests(unittest.TestCase):
|
||||
if PACKED:
|
||||
self.assertEqual(result, self.render(packed=True, CIRCLE_BRANCH="main"))
|
||||
|
||||
def test_frontend_ci_only_commits_verify_with_packed_parity(self):
|
||||
for profile, job_name in (("frontend-core", "qqq-orb/node_test_only"),
|
||||
("frontend-material", "qqq-orb/mvn_frontend_test_only")):
|
||||
self.write_manifest(profile=profile)
|
||||
self.git("add", ".qqq-ci.yml")
|
||||
self.git("-c", "commit.gpgsign=false", "commit", "-qm", "ci: adopt managed orb")
|
||||
for branch in ("main", "develop"):
|
||||
with self.subTest(profile=profile, branch=branch):
|
||||
result = self.render(CIRCLE_BRANCH=branch)
|
||||
job = result["workflows"]["verify"]["jobs"][0][job_name]
|
||||
self.assertNotIn("branch_type", job)
|
||||
self.assertEqual(job["filters"]["branches"], {"ignore": "automation/qbit-rc-pending"})
|
||||
if PACKED:
|
||||
self.assertEqual(result, self.render(packed=True, CIRCLE_BRANCH=branch))
|
||||
|
||||
def test_nonpublishing_main_ci_change_runs_verification_with_packed_parity(self):
|
||||
self.write_manifest(default_branch="main", release_on_main=False,
|
||||
test_branch_policy="feature")
|
||||
self.git("add", ".qqq-ci.yml")
|
||||
self.git("-c", "commit.gpgsign=false", "commit", "-qm", "ci: use central pipeline")
|
||||
result = self.render(CIRCLE_BRANCH="main")
|
||||
job = result["workflows"]["verify"]["jobs"][0]["qqq-orb/mvn_test_only"]
|
||||
self.assertEqual(job["filters"]["branches"], {"ignore": "automation/qbit-rc-pending"})
|
||||
self.assertNotIn("branch_type", job)
|
||||
if PACKED:
|
||||
self.assertEqual(result, self.render(packed=True, CIRCLE_BRANCH="main"))
|
||||
|
||||
def test_qbit_publication_policy_packed_parity(self):
|
||||
self.write_manifest(release_on_main=True, release_tag_policy="v-prefix",
|
||||
test_branch_policy="feature")
|
||||
@@ -532,7 +659,8 @@ class PipelineTests(unittest.TestCase):
|
||||
{"CIRCLE_BRANCH": "integration-example"}):
|
||||
result = self.render(**environment)
|
||||
if environment.get("CIRCLE_BRANCH") != "integration-example":
|
||||
self.assertIn("publish_release", result["workflows"])
|
||||
self.assertEqual(list(result["workflows"]), ["verify"])
|
||||
self.assertEqual(result["workflows"]["verify"]["jobs"][0]["qqq-orb/mvn_publish"]["branch_type"], "release")
|
||||
if PACKED:
|
||||
self.assertEqual(result, self.render(packed=True, **environment))
|
||||
|
||||
|
||||
@@ -35,13 +35,16 @@ class RcBridgeTests(unittest.TestCase):
|
||||
data = rc.canonical(SHA)
|
||||
inputs = data["on"]["workflow_dispatch"]["inputs"]
|
||||
passed = data["jobs"]["release-rc"]["with"]
|
||||
self.assertEqual({"dry_run", "finalize", "merge_sha", "bump_sha", "branch", "version"}, set(inputs))
|
||||
self.assertEqual({"dry_run", "finalize", "recover", "circle_build", "merge_sha", "bump_sha", "branch", "version"}, set(inputs))
|
||||
self.assertEqual(set(inputs), set(passed))
|
||||
for name in ("dry_run", "finalize"):
|
||||
for name in ("dry_run", "finalize", "recover"):
|
||||
self.assertEqual("boolean", inputs[name]["type"])
|
||||
self.assertIs(False, inputs[name]["default"])
|
||||
self.assertEqual("${{ github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_dispatch' && inputs.finalize) }}", passed["finalize"])
|
||||
self.assertEqual("${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false }}", passed["dry_run"])
|
||||
self.assertEqual("${{ github.event_name == 'workflow_dispatch' && inputs.recover || false }}", passed['recover'])
|
||||
self.assertEqual("${{ github.event_name == 'workflow_dispatch' && inputs.circle_build || '' }}", passed['circle_build'])
|
||||
self.assertEqual({'description': 'Original Circle publisher build number (recovery only)', 'type': 'string', 'default': ''}, inputs['circle_build'])
|
||||
for name in ("merge_sha", "bump_sha", "branch", "version"):
|
||||
self.assertEqual("string", inputs[name]["type"])
|
||||
self.assertEqual("", inputs[name]["default"])
|
||||
|
||||
@@ -6,6 +6,7 @@ scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
mkdir "$scratch/bin"
|
||||
export INSTALL_LOG="$scratch/install.log"
|
||||
export APT_CONFIG_CAPTURE="$scratch/apt.conf"
|
||||
export BASH_ENV="$scratch/bash-env"
|
||||
export QQQ_MAVEN_BIN_DIR="$scratch/maven-bin"
|
||||
: > "$BASH_ENV"
|
||||
@@ -14,6 +15,29 @@ cat > "$scratch/bin/sudo" <<'STUB'
|
||||
printf '%s\n' "$*" >> "$INSTALL_LOG"
|
||||
if [[ "$*" == *apt-get* ]]; then
|
||||
[[ "$1" == env && "$2" == DEBIAN_FRONTEND=noninteractive && "$3" == NEEDRESTART_MODE=l ]] || exit 90
|
||||
[[ -f "$APT_CONFIG_CAPTURE" ]] && grep -qx 'Acquire::Retries "3";' "$APT_CONFIG_CAPTURE" || exit 91
|
||||
[[ ${APT_FAIL:-0} == 0 ]] || exit "$APT_FAIL"
|
||||
if [[ "$*" == *'apt-get update'* ]]; then
|
||||
updates=$(grep -c 'apt-get update' "$INSTALL_LOG")
|
||||
if [[ $updates -eq 1 ]]; then
|
||||
# An unavailable image PPA must not participate in base-tool setup.
|
||||
[[ "$*" == *'Dir::Etc::sourceparts=-'* ]] || exit 100
|
||||
[[ "$*" == *'Dir::Etc::sourcelist=/etc/apt/sources.list'* || "$*" == *'Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources'* ]] || exit 93
|
||||
[[ "$*" == *'APT::Get::List-Cleanup=0'* ]] || exit 92
|
||||
[[ ${UBUNTU_UPDATE_FAIL:-0} == 0 ]] || exit "$UBUNTU_UPDATE_FAIL"
|
||||
fi
|
||||
if [[ $updates -eq 2 ]]; then
|
||||
# Model a PPA outage after the initial base-package installation. Only
|
||||
# the new signed repository may be contacted; existing indexes survive.
|
||||
[[ "$*" == *'Dir::Etc::sourcelist=/etc/apt/sources.list.d/adoptium.list'* ]] || exit 100
|
||||
[[ "$*" == *'Dir::Etc::sourceparts=-'* ]] || exit 100
|
||||
[[ "$*" == *'APT::Get::List-Cleanup=0'* ]] || exit 92
|
||||
[[ ${TEMURIN_UPDATE_FAIL:-0} == 0 ]] || exit "$TEMURIN_UPDATE_FAIL"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if [[ "$1" == tee && "$2" == /etc/apt/apt.conf.d/80-qqq-download-retries ]]; then
|
||||
cat > "$APT_CONFIG_CAPTURE"; exit 0
|
||||
fi
|
||||
if [[ "$1" == tee || "$1" == gpg || "$1" == -E ]]; then cat >/dev/null; fi
|
||||
STUB
|
||||
@@ -32,9 +56,29 @@ grep -q 'tee /etc/needrestart/conf.d/99-qqq-ci.conf' "$INSTALL_LOG"
|
||||
grep -q 'export JAVA_HOME=' "$BASH_ENV"
|
||||
: > "$INSTALL_LOG"
|
||||
: > "$BASH_ENV"
|
||||
rm "$APT_CONFIG_CAPTURE"
|
||||
PARAM_INSTALL_NODE=true bash "$script_dir/install_java.sh" >/dev/null 2>&1
|
||||
[[ $(grep -c 'apt-get' "$INSTALL_LOG") -eq 5 ]]
|
||||
grep -q 'apt-get install -y -qq nodejs' "$INSTALL_LOG"
|
||||
# An outage of the required repository must still stop before Java installation.
|
||||
: > "$INSTALL_LOG"
|
||||
if UBUNTU_UPDATE_FAIL=100 PARAM_INSTALL_NODE=false bash "$script_dir/install_java.sh" >/dev/null 2>&1; then
|
||||
echo 'required Ubuntu repository failure accepted' >&2; exit 1
|
||||
else
|
||||
[[ $? -eq 100 ]]
|
||||
fi
|
||||
if grep -q 'apt-get install' "$INSTALL_LOG"; then
|
||||
echo 'package installation continued after Ubuntu repository failure' >&2; exit 1
|
||||
fi
|
||||
: > "$INSTALL_LOG"
|
||||
if TEMURIN_UPDATE_FAIL=100 PARAM_INSTALL_NODE=false bash "$script_dir/install_java.sh" >/dev/null 2>&1; then
|
||||
echo 'required Temurin repository failure accepted' >&2; exit 1
|
||||
else
|
||||
[[ $? -eq 100 ]]
|
||||
fi
|
||||
if grep -q 'apt-get install -y -qq temurin-' "$INSTALL_LOG"; then
|
||||
echo 'Java installation continued after repository failure' >&2; exit 1
|
||||
fi
|
||||
# Execute the actual generated selector. Arguments containing spaces stay intact.
|
||||
export SELECTOR_LOG="$scratch/selector.log"
|
||||
cat > "$scratch/bin/mvn" <<'NATIVE'
|
||||
@@ -82,4 +126,9 @@ if PARAM_INSTALL_NODE=false bash "$script_dir/install_java.sh" >/dev/null 2>&1;
|
||||
echo 'invalid native Maven accepted' >&2; exit 1
|
||||
fi
|
||||
unset -f mvn
|
||||
if APT_FAIL=100 PARAM_INSTALL_NODE=false bash "$script_dir/install_java.sh" >/dev/null 2>&1; then
|
||||
echo 'persistent apt failure accepted' >&2; exit 1
|
||||
else
|
||||
[[ $? -eq 100 ]]
|
||||
fi
|
||||
printf 'PASS: unattended installation and native/executable/nonexecutable wrapper selection\n'
|
||||
@@ -38,7 +38,9 @@ class MobilePolicyTests(unittest.TestCase):
|
||||
data = json.loads((ROOT / "src/scripts/templates/android.json").read_text())
|
||||
steps = data["jobs"]["maven-deploy"]["steps"]
|
||||
self.assertEqual(len(steps), 3)
|
||||
command = steps[2]["run"]["command"]
|
||||
raw = steps[2]["run"]["command"]
|
||||
self.assertIn("python3 \\<<'PYTHON'", raw)
|
||||
command = raw.replace("\\<<", "<<")
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
runner = root / "gradlew"
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).parents[1]
|
||||
|
||||
class BranchLanguages(unittest.TestCase):
|
||||
def select(self, tracked, untracked=()):
|
||||
worker = yaml.safe_load((ROOT / '.github/workflows/qbit-codeql.yml').read_text())
|
||||
script = next(s['run'] for s in worker['jobs']['analyze']['steps'] if s.get('id') == 'languages')
|
||||
script = script.split("python3 <<'PY'\n", 1)[1].rsplit('\nPY', 1)[0]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
subprocess.run(['git', 'init', '-q', directory], check=True)
|
||||
for name in (*tracked, *untracked):
|
||||
path = root / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text('fixture')
|
||||
if tracked:
|
||||
subprocess.run(['git', 'add', '--', *tracked], cwd=root, check=True)
|
||||
output = root / 'outputs'
|
||||
result = subprocess.run([sys.executable, '-c', script], cwd=root,
|
||||
env={**os.environ, 'GITHUB_OUTPUT': str(output)}, capture_output=True, text=True)
|
||||
return result, output.read_text() if output.exists() else ''
|
||||
|
||||
def test_historical_java_branch_does_not_scan_absent_actions(self):
|
||||
result, output = self.select(['src/Example.java'])
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertIn('languages=java-kotlin\n', output)
|
||||
self.assertIn('build-mode=none\n', output)
|
||||
|
||||
def test_current_branch_preserves_all_present_supported_languages(self):
|
||||
result, output = self.select(['src/Example.java', '.github/workflows/ci.yaml', 'web/app.tsx', 'tools/check.py'])
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertIn('languages=java-kotlin,javascript-typescript,python,actions\n', output)
|
||||
|
||||
def test_kotlin_requires_a_build(self):
|
||||
result, output = self.select(['src/Example.kt', '.github/workflows/ci.yml'])
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertIn('build-mode=autobuild\n', output)
|
||||
|
||||
def test_untracked_files_do_not_select_languages(self):
|
||||
result, output = self.select(['src/Example.java'], ['web/generated.js', '.github/workflows/untracked.yml'])
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertIn('languages=java-kotlin\n', output)
|
||||
|
||||
def test_placeholder_fails_explicitly(self):
|
||||
result, output = self.select(['README.md'])
|
||||
self.assertNotEqual(0, result.returncode)
|
||||
self.assertIn('No supported source', result.stderr)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Preserve Workflows' audited publish-tag contract without broadening branches."""
|
||||
import unittest
|
||||
from unittest import mock
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
from test_qbit_hotfix_profile import load, migration, pipeline, selects, ROOT
|
||||
|
||||
standard = load('standard', 'src/scripts/validate_standard.py')
|
||||
|
||||
|
||||
class FeaturePublicationTests(unittest.TestCase):
|
||||
def config(self, manifest):
|
||||
with mock.patch.object(Path, 'read_text', return_value=yaml.safe_dump(manifest)):
|
||||
return pipeline.load_manifest('unused', '0.7.16')
|
||||
|
||||
def test_audited_event_selection_matches_legacy(self):
|
||||
legacy = yaml.safe_load((ROOT / 'tests/fixtures/qbit-feature-publication.yml').read_text())
|
||||
manifest = migration.infer_manifest(legacy, '0.7.16', 'main')
|
||||
self.assertTrue(manifest['publish']['feature'])
|
||||
caller = migration.canonical_caller('0.7.16', manifest['release_tag_policy'], True)
|
||||
standard.validate(yaml.safe_dump(caller), yaml.safe_dump(manifest), '0.7.16')
|
||||
setup = caller['workflows']['setup']['jobs'][0]['qqq-orb/generate_pipeline']
|
||||
cases = [(branch, '') for branch in ['main', 'develop', 'feature/example', 'release/0.3.0', 'hotfix/0.3.1', 'integration-test']]
|
||||
cases += [('', tag) for tag in ['publish', 'publish-example', 'v1.2.3', 'v1.2.3-RC.1', 'other']]
|
||||
for branch, tag in cases:
|
||||
with self.subTest(branch=branch, tag=tag):
|
||||
expected = sorted((job, args.get('branch_type')) for workflow in legacy['workflows'].values()
|
||||
for entry in workflow['jobs'] for job, args in entry.items()
|
||||
if selects(args.get('filters', {}), branch, tag))
|
||||
with mock.patch.object(pipeline, 'git', return_value=b'verified-head\n'):
|
||||
rendered = pipeline.qbit_template(self.config(manifest), branch, tag)
|
||||
actual = sorted((job, args.get('branch_type')) for workflow in rendered['workflows'].values()
|
||||
for entry in workflow['jobs'] for job, args in entry.items()
|
||||
if selects(args.get('filters', {}), branch, tag))
|
||||
if not selects(setup['filters'], branch, tag):
|
||||
actual = []
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
def test_feature_is_opt_in_and_requires_exact_tag_head(self):
|
||||
manifest = {'profile': 'qbit', 'orb_version': '0.7.16', 'default_branch': 'main'}
|
||||
self.assertFalse(self.config(manifest)['publish']['feature'])
|
||||
manifest['publish'] = {'feature': True}
|
||||
for tagged in [None, b'wrong-head\n']:
|
||||
def git(*args):
|
||||
return b'head\n' if args == ('rev-parse', 'HEAD') else tagged
|
||||
with self.subTest(tagged=tagged), mock.patch.object(pipeline, 'git', side_effect=git):
|
||||
rendered = pipeline.qbit_template(self.config(manifest), '', 'publish-test')
|
||||
self.assertIn('qqq-orb/mvn_test_only', rendered['workflows']['test_only']['jobs'][0])
|
||||
|
||||
def test_feature_tag_recovery_checks_fetched_commit(self):
|
||||
manifest = {'profile': 'qbit', 'orb_version': '0.7.16', 'default_branch': 'main', 'publish': {'feature': True}}
|
||||
with mock.patch.object(pipeline, 'git', side_effect=[None, b'', b'head\n', b'head\n']):
|
||||
rendered = pipeline.qbit_template(self.config(manifest), '', 'publish-test')
|
||||
self.assertEqual('feature', rendered['workflows']['publish_feature']['jobs'][0]['qqq-orb/mvn_publish']['branch_type'])
|
||||
|
||||
def test_setup_preserves_release_policy_and_rejects_drift(self):
|
||||
for policy in ['stable', 'v-prefix', 'none']:
|
||||
manifest = {'profile': 'qbit', 'orb_version': '0.7.16', 'default_branch': 'main',
|
||||
'release_tag_policy': policy, 'publish': {'feature': True}}
|
||||
caller = migration.canonical_caller('0.7.16', policy, True)
|
||||
standard.validate(yaml.safe_dump(caller), yaml.safe_dump(manifest), '0.7.16')
|
||||
self.assertEqual(caller, standard.canonical_caller('qbit', '0.7.16', release_tag_policy=policy, publish_feature=True))
|
||||
with self.assertRaises(standard.StandardError):
|
||||
standard.validate(yaml.safe_dump(migration.canonical_caller('0.7.16', policy)), yaml.safe_dump(manifest), '0.7.16')
|
||||
|
||||
def test_unrecognized_feature_branch_filters_are_rejected(self):
|
||||
for branches in [{'only': ['/feature/.*/']}, {'only': ['main']}, {'ignore': '/.*/'}]:
|
||||
with self.subTest(branches=branches), self.assertRaises(ValueError):
|
||||
migration.publication_filters('feature', {'branches': branches, 'tags': {'only': ['/publish.*/']}})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Preserve the audited main-branch hotfix contracts in the shared profile."""
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
def load(name, path):
|
||||
spec = importlib.util.spec_from_file_location(name, ROOT / path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
migration = load('migration', 'tools/standardize_qbit.py')
|
||||
pipeline = load('pipeline', 'src/scripts/generate_pipeline.py')
|
||||
|
||||
def selects(filters, branch, tag):
|
||||
selector = filters.get('tags' if tag else 'branches', {})
|
||||
if tag and not selector:
|
||||
return False
|
||||
def matches(patterns):
|
||||
patterns = patterns if isinstance(patterns, list) else [patterns]
|
||||
return any(re.fullmatch(p[1:-1], tag or branch, flags=re.ASCII) is not None if p.startswith('/') and p.endswith('/') else p == (tag or branch) for p in patterns)
|
||||
return ('only' not in selector or matches(selector['only'])) and ('ignore' not in selector or not matches(selector['ignore']))
|
||||
|
||||
class HotfixMigrationTests(unittest.TestCase):
|
||||
def test_audited_main_job_selection_is_preserved(self):
|
||||
fixtures = json.loads((ROOT/'tests/fixtures/qbit-main-hotfix-policies.json').read_text())
|
||||
for fixture in fixtures:
|
||||
manifest = migration.infer_manifest(fixture['config'], '0.7.13', 'develop')
|
||||
with mock.patch.object(Path, 'read_text', return_value=yaml.safe_dump(manifest)):
|
||||
config = pipeline.load_manifest('unused', '0.7.13')
|
||||
cases = [(branch, '') for branch in ('main','develop','feature/a','release/1','hotfix/1.2.3','integration-test','staging')]
|
||||
cases += [('', tag) for tag in ('v1.2.3','v1.2.3-RC.1','vanything','publish1','invalid','v\u0661.\u0662.\u0663')]
|
||||
for branch, tag in cases:
|
||||
with self.subTest(repository=fixture['repository'], branch=branch, tag=tag):
|
||||
expected = sorted((job, options.get('branch_type')) for workflow in fixture['config']['workflows'].values() for entry in workflow['jobs'] for job, options in entry.items() if selects(options.get('filters', {}), branch, tag))
|
||||
with mock.patch.object(pipeline, 'git', return_value=b'verified-head\n'):
|
||||
generated = pipeline.qbit_template(config, branch, tag)
|
||||
actual = sorted((job, options.get('branch_type')) for workflow in generated['workflows'].values() for entry in workflow['jobs'] for job, options in entry.items() if selects(options.get('filters', {}), branch, tag))
|
||||
# The canonical setup's tag filter prevents non-release tag runs.
|
||||
caller = migration.canonical_caller('0.7.13', manifest['release_tag_policy'])
|
||||
setup = caller['workflows']['setup']['jobs'][0]['qqq-orb/generate_pipeline']
|
||||
if not selects(setup['filters'], branch, tag): actual = []
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_cms_repair_only_quotes_the_existing_tag_pattern(self):
|
||||
fixtures = json.loads((ROOT/'tests/fixtures/qbit-main-hotfix-policies.json').read_text())
|
||||
fixture = next(row for row in fixtures if row['repository'] == 'QRun-IO/qbit-cms')
|
||||
with self.assertRaises(yaml.YAMLError):
|
||||
yaml.safe_load(fixture['original_yaml'])
|
||||
repaired = fixture['original_yaml'].replace(r'only: [/^v[0-9]+\.[0-9]+\.[0-9]+$/]',
|
||||
r"only: ['/^v[0-9]+\.[0-9]+\.[0-9]+$/']")
|
||||
self.assertEqual(yaml.safe_load(repaired), fixture['config'])
|
||||
|
||||
def test_stable_tag_selection_uses_ascii_digits(self):
|
||||
with mock.patch.object(pipeline, 'git', return_value=b'verified-head\n'):
|
||||
self.assertEqual(pipeline.event_kind('', 'v\u0661.\u0662.\u0663'), 'test')
|
||||
self.assertEqual(pipeline.event_kind('', 'v1.2.3'), 'release')
|
||||
|
||||
def test_existing_manifest_does_not_enable_hotfix_publication(self):
|
||||
raw = 'profile: qbit\norb_version: 0.7.13\ndefault_branch: develop\n'
|
||||
with mock.patch.object(Path, 'read_text', return_value=raw):
|
||||
config = pipeline.load_manifest('unused','0.7.13')
|
||||
self.assertFalse(config['publish'].get('hotfix', False))
|
||||
result = pipeline.qbit_template(config, 'hotfix/1.2.3', '')
|
||||
self.assertIn('qqq-orb/mvn_test_only', result['workflows']['test_only']['jobs'][0])
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -58,6 +58,17 @@ class FixtureAPI:
|
||||
|
||||
|
||||
class SweepTests(unittest.TestCase):
|
||||
def test_stable_gate_and_mixed_name_latest_failure(self):
|
||||
api = FixtureAPI()
|
||||
api.status['check_runs'][0]['name'] = 'verify'
|
||||
self.assertEqual('dispatched', sweep.reconcile(api, REPO))
|
||||
api.sent.clear()
|
||||
api.status['check_runs'].append(dict(api.status['check_runs'][0],
|
||||
id=10, name='publish_release_candidate', conclusion='failure'))
|
||||
with self.assertRaises(sweep.InvalidIntent):
|
||||
sweep.reconcile(api, REPO)
|
||||
self.assertEqual([], api.sent)
|
||||
|
||||
def test_idle_never_dispatches_or_reads_artifacts(self):
|
||||
api = FixtureAPI(); api.lock = None
|
||||
self.assertEqual('idle', sweep.reconcile(api, REPO))
|
||||
|
||||
@@ -23,6 +23,17 @@ def bump(parent=SHA, version='1.2.3-RC.4'):
|
||||
|
||||
|
||||
class ReleaseGuardTests(unittest.TestCase):
|
||||
def test_stable_workflow_preserves_exact_source_and_latest_attempt_checks(self):
|
||||
result = status()
|
||||
result['check_runs'][0]['name'] = 'verify'
|
||||
self.assertEqual('1.2.3-RC.4', evaluate(SHA, result, bump()))
|
||||
result['check_runs'].append(dict(result['check_runs'][0], id=10, conclusion='failure'))
|
||||
with self.assertRaisesRegex(Failure, 'CircleCI failed'):
|
||||
evaluate(SHA, result, bump())
|
||||
result['check_runs'] = [dict(result['check_runs'][0], head_sha=OTHER)]
|
||||
with self.assertRaises(Failure):
|
||||
evaluate(SHA, result, bump())
|
||||
|
||||
def test_success_exact_built_source(self):
|
||||
self.assertEqual('1.2.3-RC.4', evaluate(SHA, status(), bump()))
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Exercise actual shell handling of gh error JSON, including the observed 404."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
SOURCE = Path(__file__).parents[1].joinpath('.github/workflows/qbit-release-rc.yml').read_text()
|
||||
FUNCTION = textwrap.dedent(SOURCE.split(' # BEGIN RC REF LOOKUP\n')[1].split(' # END RC REF LOOKUP')[0])
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which('bash') and shutil.which('jq'), 'Bash and jq required')
|
||||
class RefLookupTests(unittest.TestCase):
|
||||
def lookup(self, body, code):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
env = dict(os.environ, BODY=body, CODE=str(code), REPO='QRun-IO/qbit-alerts', workdir=directory)
|
||||
script = 'set -euo pipefail\ngh() { printf "%s" "$BODY"; return "$CODE"; }\n' + FUNCTION + '\nresult=$(read_ref tags/v0.1.0-RC.17)\nprintf "%s" "$result"\n'
|
||||
return subprocess.run(['bash', '-c', script], env=env, text=True, capture_output=True)
|
||||
|
||||
def test_absent_tag_returns_empty_not_error_json(self):
|
||||
result = self.lookup('{"message":"Not Found","documentation_url":"https://docs.github.com/rest/git/refs#get-a-reference","status":"404"}', 1)
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertEqual('', result.stdout)
|
||||
|
||||
def test_existing_tag_returns_only_valid_object_sha(self):
|
||||
result = self.lookup('{"object":{"sha":"' + 'a' * 40 + '"}}', 0)
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertEqual('a' * 40, result.stdout)
|
||||
|
||||
def test_permission_network_and_server_errors_are_not_missing_refs(self):
|
||||
for body in ('{"status":"403"}', '{"status":"500"}', '', ' ', '\n', 'not json'):
|
||||
with self.subTest(body=body):
|
||||
self.assertNotEqual(0, self.lookup(body, 1).returncode)
|
||||
|
||||
def test_malformed_success_response_is_rejected(self):
|
||||
for body in ('{}', '{"object":{"sha":null}}', '{"object":{"sha":"not-a-sha"}}'):
|
||||
with self.subTest(body=body):
|
||||
self.assertNotEqual(0, self.lookup(body, 0).returncode)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,136 @@
|
||||
"""A timeout recovery must prove the original upload, not forgive a failed build."""
|
||||
import copy
|
||||
import unittest
|
||||
|
||||
from test_rc_guard import scope, SHA, OTHER, Failure, bump, status, evaluate
|
||||
|
||||
REPO = 'QRun-IO/qbit-alerts'
|
||||
WORKFLOW = '86efe478-29d6-41a4-93f0-d74b178bf329'
|
||||
DEPLOYMENT = '993e5a6d-23ce-4696-8e5a-9176ae19e670'
|
||||
VERSION = '0.1.0-RC.17'
|
||||
PURL = 'pkg:maven/com.kingsrook.qbits/qbit-alerts@' + VERSION
|
||||
|
||||
|
||||
def evidence():
|
||||
publisher = status('failure')['check_runs'][0]
|
||||
publisher['details_url'] = 'https://app.circleci.com/workflow/' + WORKFLOW + '?utm_source=github'
|
||||
job = {'username': 'QRun-IO', 'reponame': 'qbit-alerts', 'vcs_revision': SHA,
|
||||
'branch': 'release/0.1', 'build_num': 148, 'status': 'failed', 'outcome': 'failed',
|
||||
'start_time': '2026-10-21T00:01:00Z',
|
||||
'workflows': {'workflow_name': 'publish_release_candidate',
|
||||
'job_name': 'qqq-orb/mvn_publish', 'workflow_id': WORKFLOW},
|
||||
'steps': [{'actions': [{'name': 'Run Maven Verify', 'status': 'success'}]},
|
||||
{'actions': [{'name': 'Publish to Sonatype Central (releases and SNAPSHOTs)',
|
||||
'status': 'timedout', 'timedout': True, 'step': 124, 'index': 0}]}]}
|
||||
return job, publisher
|
||||
|
||||
|
||||
def validate(job, publisher):
|
||||
return scope['validate_timeout'](job, publisher, REPO, SHA, 'release/0.1', '148', '2026-10-21T00:00:00Z')
|
||||
|
||||
|
||||
class TimeoutRecoveryTests(unittest.TestCase):
|
||||
def test_stable_gate_must_match_the_actual_publishing_workflow(self):
|
||||
job, publisher = evidence()
|
||||
job['workflows']['workflow_name'] = 'verify'
|
||||
publisher['name'] = 'verify'
|
||||
validate(job, publisher)
|
||||
publisher['name'] = 'publish_release_candidate'
|
||||
with self.assertRaises(Failure):
|
||||
validate(job, publisher)
|
||||
|
||||
def test_exact_upload_timeout_after_successful_verify(self):
|
||||
self.assertEqual((124, 0), validate(*evidence()))
|
||||
|
||||
def test_other_source_branch_repository_or_attempt_is_rejected(self):
|
||||
for key, value in [('vcs_revision', OTHER), ('branch', 'develop'),
|
||||
('username', 'untrusted'), ('reponame', 'other'), ('build_num', 149)]:
|
||||
with self.subTest(key=key), self.assertRaises(Failure):
|
||||
job, publisher = evidence()
|
||||
job[key] = value
|
||||
validate(job, publisher)
|
||||
|
||||
def test_non_circle_or_unrelated_workflow_cannot_authorize_recovery(self):
|
||||
for key, value in [('head_sha', OTHER), ('app', {'id': 1}), ('status', 'in_progress'),
|
||||
('conclusion', 'success'), ('name', 'test_only'),
|
||||
('details_url', 'https://example.com/workflow/' + WORKFLOW),
|
||||
('details_url', 'https://app.circleci.com/workflow/another')]:
|
||||
with self.subTest(key=key, value=value), self.assertRaises(Failure):
|
||||
job, publisher = evidence()
|
||||
publisher[key] = value
|
||||
validate(job, publisher)
|
||||
|
||||
def test_prior_failure_or_missing_verify_never_becomes_release_success(self):
|
||||
for action in [{'name': 'Run Maven Verify', 'status': 'failed'},
|
||||
{'name': 'Run Maven Verify', 'status': 'skipped'},
|
||||
{'name': 'Some other build', 'status': 'success'}]:
|
||||
with self.subTest(action=action), self.assertRaises(Failure):
|
||||
job, publisher = evidence()
|
||||
job['steps'][0]['actions'][0] = action
|
||||
validate(job, publisher)
|
||||
|
||||
def test_only_a_terminal_deployment_timeout_is_eligible(self):
|
||||
for change in [{'name': 'Run Maven Verify'}, {'timedout': False}, {'status': 'failed'},
|
||||
{'step': -1}, {'index': '0'}]:
|
||||
with self.subTest(change=change), self.assertRaises(Failure):
|
||||
job, publisher = evidence()
|
||||
job['steps'][-1]['actions'][0].update(change)
|
||||
validate(job, publisher)
|
||||
|
||||
def test_failed_later_git_push_is_not_an_upload_timeout(self):
|
||||
job, publisher = evidence()
|
||||
job['steps'].append({'actions': [{'name': 'Commit version', 'status': 'failed'}]})
|
||||
with self.assertRaises(Failure):
|
||||
validate(job, publisher)
|
||||
|
||||
def test_publisher_cannot_predate_the_cut(self):
|
||||
job, publisher = evidence()
|
||||
job['start_time'] = '2026-10-20T23:59:00Z'
|
||||
with self.assertRaises(Failure):
|
||||
validate(job, publisher)
|
||||
|
||||
def test_receipt_requires_one_upload_and_expected_staged_coordinates(self):
|
||||
log = ('[INFO] Installing x to /home/circleci/project/target/central-staging/'
|
||||
'com/kingsrook/qbits/qbit-alerts/' + VERSION + '/qbit-alerts-' + VERSION + '.pom\n'
|
||||
'[INFO] Uploaded bundle successfully, deployment name: Deployment, deploymentId: '
|
||||
+ DEPLOYMENT + '. Deployment will publish automatically\n')
|
||||
self.assertEqual((DEPLOYMENT, [PURL]), scope['upload_receipt'](log, 'qbit-alerts', VERSION))
|
||||
for invalid in [log + log, log.replace('Uploaded bundle successfully', 'Upload failed'),
|
||||
log.replace('qbit-alerts', 'qbit-other'), log.replace(VERSION, '0.1.0-RC.16')]:
|
||||
with self.subTest(invalid=invalid), self.assertRaises(Failure):
|
||||
scope['upload_receipt'](invalid, 'qbit-alerts', VERSION)
|
||||
|
||||
def test_central_must_confirm_the_exact_receipt_and_all_coordinates(self):
|
||||
result = {'deploymentId': DEPLOYMENT, 'deploymentState': 'PUBLISHED', 'purls': [PURL]}
|
||||
scope['validate_deployment'](result, DEPLOYMENT, [PURL])
|
||||
for key, value in [('deploymentId', 'other'), ('deploymentState', 'PUBLISHING'),
|
||||
('deploymentState', 'FAILED'), ('purls', []), ('purls', [PURL, 'other'])]:
|
||||
with self.subTest(key=key, value=value), self.assertRaises(Failure):
|
||||
invalid = copy.deepcopy(result)
|
||||
invalid[key] = value
|
||||
scope['validate_deployment'](invalid, DEPLOYMENT, [PURL])
|
||||
|
||||
def test_retry_bump_cannot_hide_other_source_changes(self):
|
||||
source = '<project><revision>0.1.0-RC.16</revision><name>Alerts</name></project>'
|
||||
actual = source.replace('0.1.0-RC.16', VERSION)
|
||||
commit = bump(version=VERSION)
|
||||
commit['files'] = [{'filename': 'pom.xml', 'status': 'modified'}]
|
||||
scope['validate_recovery_bump'](commit, source, actual, VERSION, SHA)
|
||||
for changed in [actual.replace('Alerts', 'Other'), actual + '<extra/>', source]:
|
||||
with self.subTest(changed=changed), self.assertRaises(Failure):
|
||||
scope['validate_recovery_bump'](commit, source, changed, VERSION, SHA)
|
||||
commit['files'].append({'filename': 'other.java', 'status': 'modified'})
|
||||
with self.assertRaises(Failure):
|
||||
scope['validate_recovery_bump'](commit, source, actual, VERSION, SHA)
|
||||
|
||||
def test_recovery_still_requires_exact_bump_parent_and_verified_version(self):
|
||||
self.assertEqual(VERSION, evaluate(SHA, status('failure'), bump(version=VERSION), VERSION))
|
||||
for commit in [bump(parent=OTHER, version=VERSION), bump(version='0.1.0-RC.18')]:
|
||||
with self.subTest(commit=commit), self.assertRaises(Failure):
|
||||
evaluate(SHA, status('failure'), commit, VERSION)
|
||||
with self.assertRaises(Failure):
|
||||
evaluate(SHA, status('failure'), bump(version=VERSION))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Remote enrollment keeps immutable source checks and resumes signed submissions."""
|
||||
import base64
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
import yaml
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
import reconcile_enrollment as remote
|
||||
|
||||
|
||||
class ReconcileTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.head = 'a' * 40
|
||||
self.new = 'b' * 40
|
||||
self.repo = 'QRun-IO/example'
|
||||
self.target = {'repository': self.repo, 'branch': 'main'}
|
||||
self.manifest = {'profile': 'qbit', 'orb_version': '0.7.4', 'default_branch': 'main',
|
||||
'publish': {'snapshot': False, 'release_candidate': False, 'release': False}}
|
||||
self.sources = {
|
||||
'.qqq-ci.yml': yaml.safe_dump(self.manifest).encode(),
|
||||
'.circleci/config.yml': yaml.safe_dump(remote.enroll_consumers.rollout.standard.canonical_caller('qbit', '0.7.4')).encode()}
|
||||
|
||||
def request(self, path, missing=False):
|
||||
if path == 'repos/' + self.repo:
|
||||
return {'archived': False, 'default_branch': 'main'}
|
||||
if path.endswith('/branches/main'):
|
||||
return {'commit': {'sha': self.head}}
|
||||
if '/contents/' in path:
|
||||
self.assertTrue(path.endswith('?ref=' + self.head))
|
||||
name = path.split('/contents/')[1].split('?')[0]
|
||||
raw = self.sources.get(name)
|
||||
return None if raw is None else {'type': 'file', 'encoding': 'base64', 'content': base64.b64encode(raw).decode()}
|
||||
raise AssertionError(path)
|
||||
|
||||
def plan(self):
|
||||
with mock.patch.object(remote, 'api', side_effect=self.request):
|
||||
return remote.inspect(self.target, '0.8')
|
||||
|
||||
def test_immutable_plan_is_deterministic_and_preserves_policy(self):
|
||||
first, second = self.plan(), self.plan()
|
||||
self.assertEqual(first['automation_branch'], second['automation_branch'])
|
||||
self.assertEqual(self.sources, first['originals'])
|
||||
self.assertEqual({**self.manifest, 'orb_version': '0.8'},
|
||||
yaml.safe_load(first['replacements']['.qqq-ci.yml']))
|
||||
self.assertTrue(first['changed'])
|
||||
|
||||
def test_missing_manifest_uses_legacy_converter(self):
|
||||
self.sources.pop('.qqq-ci.yml')
|
||||
self.sources['.circleci/config.yml'] = yaml.safe_dump({'version': 2.1,
|
||||
'orbs': {'qqq-orb': 'kingsrook/qqq-orb@0.5.0'},
|
||||
'workflows': {'verify': {'jobs': [{'qqq-orb/mvn_test_only': {}}]}}}).encode()
|
||||
plan = self.plan()
|
||||
self.assertIsNone(plan['originals']['.qqq-ci.yml'])
|
||||
self.assertFalse(any(yaml.safe_load(plan['replacements']['.qqq-ci.yml'])['publish'].values()))
|
||||
|
||||
def submission_api(self, plan, path, missing=False):
|
||||
if '/branches/main' in path:
|
||||
return {'commit': {'sha': self.head}}
|
||||
if '/git/ref/' in path:
|
||||
return {'object': {'sha': self.new}}
|
||||
if '/commits/' in path:
|
||||
return {'commit': {'verification': {'verified': True}},
|
||||
'parents': [{'sha': self.head}], 'files': [{'filename': p} for p in remote.PATHS]}
|
||||
if '/contents/' in path:
|
||||
self.assertTrue(path.endswith('?ref=' + self.new))
|
||||
name = path.split('/contents/')[1].split('?')[0]
|
||||
return {'type': 'file', 'encoding': 'base64', 'content': base64.b64encode(plan['replacements'][name]).decode()}
|
||||
if '/pulls?' in path:
|
||||
return [{'head': {'sha': self.new}, 'html_url': 'https://github.com/QRun-IO/example/pull/1'}]
|
||||
raise AssertionError(path)
|
||||
|
||||
def test_matching_existing_submission_uses_no_git_or_pr_mutations(self):
|
||||
plan = self.plan()
|
||||
with mock.patch.object(remote, 'api', side_effect=lambda *a, **k: self.submission_api(plan, *a, **k)), mock.patch.object(remote, 'command') as command:
|
||||
result = remote.apply(plan)
|
||||
self.assertEqual('existing-pr', result['state'])
|
||||
command.assert_not_called()
|
||||
|
||||
def test_unsigned_existing_branch_is_rejected_without_mutation(self):
|
||||
plan = self.plan()
|
||||
|
||||
def request(path, missing=False):
|
||||
value = self.submission_api(plan, path, missing)
|
||||
if '/commits/' in path:
|
||||
value['commit']['verification']['verified'] = False
|
||||
return value
|
||||
|
||||
with mock.patch.object(remote, 'api', side_effect=request), mock.patch.object(remote, 'command') as command:
|
||||
with self.assertRaises(ValueError):
|
||||
remote.apply(plan)
|
||||
command.assert_not_called()
|
||||
|
||||
def test_source_drift_prevents_submission(self):
|
||||
plan = self.plan()
|
||||
with mock.patch.object(remote, 'api', return_value={'commit': {'sha': 'c' * 40}}), mock.patch.object(remote, 'command') as command:
|
||||
with self.assertRaises(ValueError):
|
||||
remote.apply(plan)
|
||||
command.assert_not_called()
|
||||
|
||||
def test_interrupted_push_creates_only_missing_pr(self):
|
||||
plan = self.plan()
|
||||
seen = []
|
||||
|
||||
def request(path, missing=False):
|
||||
return [] if '/pulls?' in path else self.submission_api(plan, path, missing)
|
||||
|
||||
def command(args, cwd=None):
|
||||
seen.append(args[:3])
|
||||
self.assertEqual(['gh', 'pr', 'create'], args[:3])
|
||||
self.assertIn(self.head, Path(args[args.index('--body-file') + 1]).read_text())
|
||||
return 'https://github.com/QRun-IO/example/pull/2'
|
||||
|
||||
with mock.patch.object(remote, 'api', side_effect=request), mock.patch.object(remote, 'command', side_effect=command):
|
||||
result = remote.apply(plan)
|
||||
self.assertEqual('submitted', result['state'])
|
||||
self.assertEqual([['gh', 'pr', 'create']], seen)
|
||||
|
||||
def test_new_api_submission_is_atomic_and_exact(self):
|
||||
plan = self.plan()
|
||||
mutations = []
|
||||
|
||||
def request(path, missing=False):
|
||||
if '/git/ref/' in path:
|
||||
return None
|
||||
return self.submission_api(plan, path, missing)
|
||||
|
||||
def mutate(path, payload):
|
||||
mutations.append((path, payload))
|
||||
return {'data': {'createCommitOnBranch': {'commit': {'oid': self.new}}}}
|
||||
|
||||
with mock.patch.object(remote, 'api', side_effect=request), mock.patch.object(remote, 'mutation', side_effect=mutate), mock.patch.object(remote, 'command') as command:
|
||||
self.assertEqual('existing-pr', remote.apply(plan)['state'])
|
||||
command.assert_not_called()
|
||||
self.assertEqual(('repos/' + self.repo + '/git/refs', {'ref': 'refs/heads/' + plan['automation_branch'], 'sha': self.head}), mutations[0])
|
||||
commit = mutations[1][1]['variables']['input']
|
||||
self.assertEqual(self.head, commit['expectedHeadOid'])
|
||||
self.assertEqual({'repositoryNameWithOwner': self.repo, 'branchName': plan['automation_branch']}, commit['branch'])
|
||||
self.assertEqual(plan['replacements'], {item['path']: base64.b64decode(item['contents']) for item in commit['fileChanges']['additions']})
|
||||
|
||||
def test_base_only_branch_resumes_without_creating_another_ref(self):
|
||||
plan = self.plan()
|
||||
result = {'data': {'createCommitOnBranch': {'commit': {'oid': self.new}}}}
|
||||
with mock.patch.object(remote, 'api', return_value={'commit': {'sha': self.head}}), mock.patch.object(remote, 'mutation', return_value=result) as mutation:
|
||||
self.assertEqual(self.new, remote.signed_api_commit(plan, {'object': {'sha': self.head}}))
|
||||
self.assertEqual(1, mutation.call_count)
|
||||
self.assertEqual('graphql', mutation.call_args.args[0])
|
||||
|
||||
def test_existing_completed_branch_is_not_mutated(self):
|
||||
with mock.patch.object(remote, 'mutation') as mutation:
|
||||
self.assertEqual(self.new, remote.signed_api_commit(self.plan(), {'object': {'sha': self.new}}))
|
||||
mutation.assert_not_called()
|
||||
|
||||
def test_target_drift_prevents_commit_on_base_only_branch(self):
|
||||
plan = self.plan()
|
||||
with mock.patch.object(remote, 'api', return_value={'commit': {'sha': 'c' * 40}}), mock.patch.object(remote, 'mutation') as mutation:
|
||||
with self.assertRaises(ValueError):
|
||||
remote.signed_api_commit(plan, {'object': {'sha': self.head}})
|
||||
mutation.assert_not_called()
|
||||
|
||||
def test_graphql_errors_fail_closed(self):
|
||||
response = mock.Mock(returncode=0, stdout='{"errors": [{"message": "conflict"}]}')
|
||||
with mock.patch.object(remote.subprocess, 'run', return_value=response):
|
||||
with self.assertRaises(ValueError):
|
||||
remote.mutation('graphql', {})
|
||||
|
||||
def test_unsigned_api_commit_cannot_create_pr(self):
|
||||
plan = self.plan()
|
||||
def request(path, missing=False):
|
||||
if '/git/ref/' in path:
|
||||
return {'object': {'sha': self.head}}
|
||||
value = self.submission_api(plan, path, missing)
|
||||
if '/commits/' in path:
|
||||
value['commit']['verification']['verified'] = False
|
||||
return value
|
||||
with mock.patch.object(remote, 'api', side_effect=request), mock.patch.object(remote, 'mutation', return_value={'data': {'createCommitOnBranch': {'commit': {'oid': self.new}}}}), mock.patch.object(remote, 'command') as command:
|
||||
with self.assertRaises(ValueError):
|
||||
remote.apply(plan)
|
||||
command.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Execute the policy job against real git indexes, including application promotion."""
|
||||
import importlib.util
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location('pipeline', ROOT / 'src/scripts/generate_pipeline.py')
|
||||
pipeline = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pipeline)
|
||||
TEMPLATE = json.loads((ROOT / 'src/scripts/templates/repository-policy.json').read_text())
|
||||
SCRIPT = TEMPLATE['jobs']['validate-policy']['steps'][-1]['run']['command'].split("<<'PYTHON'\n", 1)[1].rsplit('PYTHON', 1)[0]
|
||||
|
||||
|
||||
class RepositoryPolicyTests(unittest.TestCase):
|
||||
def test_packing_is_independent_of_checkout_line_endings(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
shutil.copytree(ROOT / 'src/scripts', root / 'src/scripts')
|
||||
(root / 'scripts').mkdir()
|
||||
shutil.copy(ROOT / 'scripts/pack_pipeline.py', root / 'scripts/pack_pipeline.py')
|
||||
resources = list((root / 'src/scripts/templates').glob('*.json'))
|
||||
for resource in resources:
|
||||
resource.write_bytes(resource.read_bytes().replace(b'\r\n', b'\n'))
|
||||
subprocess.run([sys.executable, str(root / 'scripts/pack_pipeline.py')], check=True)
|
||||
packed = (root / 'src/scripts/packed_generate_pipeline.py').read_bytes()
|
||||
for resource in resources:
|
||||
resource.write_bytes(resource.read_bytes().replace(b'\n', b'\r\n'))
|
||||
subprocess.run([sys.executable, str(root / 'scripts/pack_pipeline.py'), '--check'], check=True)
|
||||
self.assertEqual(packed, (root / 'src/scripts/packed_generate_pipeline.py').read_bytes())
|
||||
|
||||
def execute(self, files, untracked=None):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
subprocess.run(['git', 'init', '-q'], cwd=root, check=True)
|
||||
for name, content in files.items():
|
||||
file = root / name
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file.write_text(content, encoding='utf-8')
|
||||
subprocess.run(['git', 'add', '.'], cwd=root, check=True)
|
||||
for name, content in (untracked or {}).items():
|
||||
(root / name).write_text(content, encoding='utf-8')
|
||||
return subprocess.run([sys.executable, '-c', SCRIPT], cwd=root, capture_output=True, text=True)
|
||||
|
||||
def test_placeholder_with_wrappers_and_workflows(self):
|
||||
result = self.execute({'LICENSE': 'Apache-2.0', 'docs/DESIGN.md': '# Planned application',
|
||||
'.github/workflows/release-rc.yml': 'name: Release\non: workflow_dispatch\n',
|
||||
'.mvn/wrapper/maven-wrapper.properties': 'distributionUrl=https://example.invalid/maven.zip',
|
||||
'mvnw': '#!/bin/sh\n', 'mvnw.cmd': '@echo off\n'})
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
|
||||
def test_new_application_requires_real_build_profile(self):
|
||||
for name in ('pom.xml', 'package.json', 'src/App.java', 'main.py', 'main.go', 'Dockerfile'):
|
||||
with self.subTest(name=name):
|
||||
self.assertNotEqual(0, self.execute({'LICENSE': 'license', name: '{}'}).returncode)
|
||||
|
||||
def test_malformed_configuration_fails(self):
|
||||
for name in ('.github/dependabot.yml', 'settings.json'):
|
||||
with self.subTest(name=name):
|
||||
self.assertNotEqual(0, self.execute({name: '[unterminated'}).returncode)
|
||||
|
||||
def test_unused_checkstyle_policy_is_validated_without_allowing_application_xml(self):
|
||||
self.assertEqual(0, self.execute({'checkstyle/config.xml': '<module name="Checker"/>'}).returncode)
|
||||
self.assertNotEqual(0, self.execute({'checkstyle/config.xml': '<module>'}).returncode)
|
||||
self.assertNotEqual(0, self.execute({'application.xml': '<application/>'}).returncode)
|
||||
self.assertNotEqual(0, self.execute({'pom.xml': '<project/>'}).returncode)
|
||||
|
||||
def test_only_tracked_files_are_policy_inputs(self):
|
||||
self.assertEqual(0, self.execute({'README.md': '# Docs'}, {'main.py': 'temporary tool'}).returncode)
|
||||
|
||||
def test_manifest_cannot_enable_publication(self):
|
||||
manifest = {'profile': 'repository-policy', 'orb_version': '0.7.14', 'default_branch': 'develop'}
|
||||
with patch.object(Path, 'read_text', return_value=yaml.safe_dump(manifest)):
|
||||
config = pipeline.load_manifest('unused', '0.7.14')
|
||||
self.assertFalse(any(config['publish'].values()))
|
||||
self.assertEqual({'verify': {'jobs': ['validate-policy']}}, pipeline.repository_policy_template(config, 'main', '')['workflows'])
|
||||
manifest['publish'] = {'release': True}
|
||||
with patch.object(Path, 'read_text', return_value=yaml.safe_dump(manifest)), self.assertRaises(pipeline.ManifestError):
|
||||
pipeline.load_manifest('unused', '0.7.14')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Version rollouts preserve policy and reject drift before changing any repo."""
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import yaml
|
||||
|
||||
spec = importlib.util.spec_from_file_location('rollout', Path(__file__).resolve().parents[1] / 'tools/rollout_version.py')
|
||||
rollout = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(rollout)
|
||||
|
||||
|
||||
class RolloutTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
|
||||
def checkout(self, name, profile='qbit', explicit=False, **policy):
|
||||
root = self.root / name
|
||||
(root / '.circleci').mkdir(parents=True)
|
||||
manifest = {'profile': profile, 'orb_version': '0.7.4', 'default_branch': 'develop', **policy}
|
||||
caller = rollout.standard.canonical_caller(profile, '0.7.4', explicit_manifest=explicit,
|
||||
release_tag_policy=manifest.get('release_tag_policy'))
|
||||
(root / '.qqq-ci.yml').write_text(yaml.safe_dump(manifest), encoding='utf-8')
|
||||
(root / '.circleci/config.yml').write_text(yaml.safe_dump(caller), encoding='utf-8')
|
||||
return root
|
||||
|
||||
def contents(self, root):
|
||||
return [(root / path).read_bytes() for path in ('.circleci/config.yml', '.qqq-ci.yml')]
|
||||
|
||||
def test_batch_preserves_opt_outs_filters_contexts_and_forwarded_parameters(self):
|
||||
qbit = self.checkout('qbit', release_tag_policy='none', release_on_main=False,
|
||||
test_branch_policy='feature', browsers=False,
|
||||
publish={'snapshot': False, 'release_candidate': False, 'release': False})
|
||||
core = self.checkout('core', profile='core', explicit=True)
|
||||
cli = self.checkout('cli', profile='cli', publish={'release': True}, publish_context='npm-publisher')
|
||||
roots = [qbit, core, cli]
|
||||
originals = [yaml.safe_load((root / '.qqq-ci.yml').read_text()) for root in roots]
|
||||
results = rollout.rollout(roots, '0.7.11', write=True)
|
||||
self.assertTrue(all(result['changed'] for result in results))
|
||||
for root, original in zip(roots, originals):
|
||||
expected = {**original, 'orb_version': '0.7.11'}
|
||||
self.assertEqual(expected, yaml.safe_load((root / '.qqq-ci.yml').read_text()))
|
||||
rollout.standard.validate(*(data.decode() for data in self.contents(root)), '0.7.11')
|
||||
rollout.pipeline.load_manifest(root / '.qqq-ci.yml', '0.7.11')
|
||||
core_caller = yaml.safe_load((core / '.circleci/config.yml').read_text())
|
||||
options = core_caller['workflows']['setup']['jobs'][0]['qqq-orb/generate_pipeline']
|
||||
self.assertEqual('.qqq-ci.yml', options['manifest'])
|
||||
self.assertEqual('<< pipeline.parameters.mongo_bind_diagnostics >>', options['mongo_bind_diagnostics'])
|
||||
|
||||
def test_all_supported_profiles_remain_valid_after_the_same_batch_upgrade(self):
|
||||
roots = [self.checkout(profile, profile=profile, default_branch='main') for profile in rollout.standard.PROFILE_PARAMETERS]
|
||||
results = rollout.rollout(roots, '0.7.11', write=True)
|
||||
self.assertEqual(set(rollout.standard.PROFILE_PARAMETERS), {item['profile'] for item in results})
|
||||
for root in roots:
|
||||
rollout.pipeline.load_manifest(root / '.qqq-ci.yml', '0.7.11')
|
||||
rollout.standard.validate(*(data.decode() for data in self.contents(root)), '0.7.11')
|
||||
|
||||
def test_dry_run_and_repeated_application_do_not_rewrite_files(self):
|
||||
root = self.checkout('qbit')
|
||||
before = self.contents(root)
|
||||
self.assertTrue(rollout.rollout([root], '0.7.11')[0]['changed'])
|
||||
self.assertEqual(before, self.contents(root))
|
||||
rollout.rollout([root], '0.7.11', write=True)
|
||||
applied = self.contents(root)
|
||||
self.assertFalse(rollout.rollout([root], '0.7.11', write=True)[0]['changed'])
|
||||
self.assertEqual(applied, self.contents(root))
|
||||
|
||||
def test_managed_patch_track_and_exact_pin_rollback_preserve_every_profile(self):
|
||||
roots = [self.checkout(profile, profile=profile, default_branch='main')
|
||||
for profile in rollout.standard.PROFILE_PARAMETERS]
|
||||
originals = [yaml.safe_load((root / '.qqq-ci.yml').read_text()) for root in roots]
|
||||
rollout.rollout(roots, '0.7', write=True)
|
||||
for root, original in zip(roots, originals):
|
||||
self.assertEqual({**original, 'orb_version': '0.7'},
|
||||
yaml.safe_load((root / '.qqq-ci.yml').read_text()))
|
||||
rollout.standard.validate(*(data.decode() for data in self.contents(root)), '0.7')
|
||||
rollout.pipeline.load_manifest(root / '.qqq-ci.yml', '0.7')
|
||||
with self.assertRaises(ValueError):
|
||||
rollout.pipeline.load_manifest(root / '.qqq-ci.yml', '0.8')
|
||||
self.assertTrue(all(not r['changed'] for r in rollout.rollout(roots, '0.7', write=True)))
|
||||
rollout.rollout(roots, '0.7.4', write=True)
|
||||
for root, original in zip(roots, originals):
|
||||
self.assertEqual(original, yaml.safe_load((root / '.qqq-ci.yml').read_text()))
|
||||
|
||||
def test_unquoted_track_is_rejected_as_numeric_yaml(self):
|
||||
root = self.checkout('numeric')
|
||||
manifest = root / '.qqq-ci.yml'
|
||||
manifest.write_text(manifest.read_text().replace('0.7.4', '0.7'))
|
||||
with self.assertRaises(ValueError):
|
||||
rollout.pipeline.load_manifest(manifest, '0.7')
|
||||
|
||||
def test_one_custom_caller_rejects_entire_batch_before_any_write(self):
|
||||
good = self.checkout('good')
|
||||
bad = self.checkout('bad')
|
||||
before = self.contents(good)
|
||||
with (bad / '.circleci/config.yml').open('a') as stream:
|
||||
stream.write('\njobs: {custom: {steps: [checkout]}}\n')
|
||||
with self.assertRaises(ValueError):
|
||||
rollout.rollout([good, bad], '0.7.11', write=True)
|
||||
self.assertEqual(before, self.contents(good))
|
||||
|
||||
def test_duplicate_manifest_keys_unknown_settings_and_mismatched_pins_are_rejected(self):
|
||||
for suffix in ['orb_version: 0.7.4\n', 'unknown_setting: true\n', 'publish: {release: maybe}\n']:
|
||||
with self.subTest(suffix=suffix):
|
||||
root = self.checkout(str(len(list(self.root.iterdir()))))
|
||||
with (root / '.qqq-ci.yml').open('a') as stream:
|
||||
stream.write(suffix)
|
||||
before = self.contents(root)
|
||||
with self.assertRaises(ValueError):
|
||||
rollout.rollout([root], '0.7.11', write=True)
|
||||
self.assertEqual(before, self.contents(root))
|
||||
root = self.checkout('mismatch')
|
||||
path = root / '.circleci/config.yml'
|
||||
path.write_text(path.read_text().replace('0.7.4', '0.7.3'))
|
||||
with self.assertRaises(ValueError):
|
||||
rollout.rollout([root], '0.7.11', write=True)
|
||||
|
||||
def test_invalid_target_and_duplicate_checkout_are_rejected(self):
|
||||
root = self.checkout('qbit')
|
||||
before = self.contents(root)
|
||||
for version in ['latest', 'volatile', 'dev:test', '0', '0.7.*', '0.7.11\n']:
|
||||
with self.assertRaises(ValueError):
|
||||
rollout.rollout([root], version, write=True)
|
||||
with self.assertRaises(ValueError):
|
||||
rollout.rollout([root, root], '0.7.11', write=True)
|
||||
self.assertEqual(before, self.contents(root))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Central execution rejects untrusted branches and preserves partial evidence."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||||
import run_enrollment
|
||||
|
||||
|
||||
class RunnerTests(unittest.TestCase):
|
||||
def environment(self):
|
||||
return {'CIRCLE_PROJECT_USERNAME': 'QRun-IO', 'CIRCLE_PROJECT_REPONAME': 'qqq-orb',
|
||||
'CIRCLE_BRANCH': 'main', 'GITHUB_TOKEN': 'test-only-secret'}
|
||||
|
||||
def test_wrong_owner_branch_repo_tag_or_missing_auth_cannot_execute(self):
|
||||
for change in ({'CIRCLE_PROJECT_USERNAME': 'other'}, {'CIRCLE_PROJECT_REPONAME': 'other'},
|
||||
{'CIRCLE_BRANCH': 'develop'}, {'CIRCLE_TAG': 'v1.0.0'}, {'GITHUB_TOKEN': ''}):
|
||||
execute = Mock()
|
||||
with self.subTest(change=change), self.assertRaises(ValueError):
|
||||
run_enrollment.run({**self.environment(), **change}, execute)
|
||||
execute.assert_not_called()
|
||||
|
||||
def test_timeout_retains_partial_report_without_secrets(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
def execute(args, **kwargs):
|
||||
output = Path(args[args.index('--output') + 1])
|
||||
output.write_text('[{"state":"unresolved"}]')
|
||||
self.assertEqual(1200, kwargs['timeout'])
|
||||
self.assertEqual('test-only-secret', kwargs['env']['GH_TOKEN'])
|
||||
raise subprocess.TimeoutExpired(args, 1200)
|
||||
self.assertEqual(124, run_enrollment.run(self.environment(), execute, directory))
|
||||
output = Path(directory) / 'artifacts/enrollment'
|
||||
self.assertEqual([{'state': 'unresolved'}], json.loads((output / 'result.json').read_text()))
|
||||
self.assertEqual('timed-out', json.loads((output / 'run.json').read_text())['state'])
|
||||
self.assertNotIn('test-only-secret', (output / 'run.json').read_text())
|
||||
|
||||
def test_failed_pass_propagates_nonzero_exit(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
self.assertEqual(2, run_enrollment.run(self.environment(), Mock(return_value=Mock(returncode=2)), directory))
|
||||
self.assertEqual('failed', json.loads((Path(directory) / 'artifacts/enrollment/run.json').read_text())['state'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -3,6 +3,7 @@ import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
@@ -15,6 +16,38 @@ spec.loader.exec_module(migration)
|
||||
|
||||
|
||||
class MigrationTests(unittest.TestCase):
|
||||
def test_hotfix_and_integration_exclusion_preserves_main_tests_and_publications(self):
|
||||
pattern = "/(develop|release/.*|hotfix/.*|integration.*)/"
|
||||
config = self.config(filters={"branches": {"ignore": pattern}, "tags": {"only": []}})
|
||||
for kind, branch in (("snapshot", "develop"), ("release_candidate", r"/release\/.*/"), ("hotfix", r"/hotfix\/.*/")):
|
||||
config["workflows"][kind] = {"jobs": [{"qqq-orb/mvn_publish": {
|
||||
"branch_type": kind, "filters": {"branches": {"only": [branch]}}}}]}
|
||||
config["workflows"]["release"] = {"jobs": [{"qqq-orb/mvn_publish": {
|
||||
"branch_type": "release", "filters": {"branches": {"ignore": "/.*/"},
|
||||
"tags": {"only": [r"/^v\d+\.\d+\.\d+$/"]}}}}]}
|
||||
manifest = migration.infer_manifest(config, "0.8", "develop")
|
||||
self.assertEqual("non-release-no-hotfix-integration", manifest["test_branch_policy"])
|
||||
self.assertFalse(manifest["release_on_main"])
|
||||
self.assertEqual("stable", manifest["release_tag_policy"])
|
||||
self.assertTrue(all(manifest["publish"][kind] for kind in ("snapshot", "release_candidate", "release", "hotfix")))
|
||||
spec = importlib.util.spec_from_file_location("filter_pipeline", Path(__file__).resolve().parents[1] / "src/scripts/generate_pipeline.py")
|
||||
pipeline = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pipeline)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "manifest.yml"
|
||||
path.write_text(yaml.safe_dump(manifest), encoding="utf-8")
|
||||
loaded = pipeline.load_manifest(path, "0.8")
|
||||
with mock.patch.object(pipeline, "proven_ci_maintenance", return_value=False), mock.patch.object(pipeline, "proven_docs_only", return_value=False), mock.patch.object(pipeline, "git", return_value=b"qualified-head\n"):
|
||||
for branch, tag, kind in (("main", "", None), ("feature/test", "", None),
|
||||
("integration-test", "", None), ("develop", "", "snapshot"),
|
||||
("release/1.0", "", "release_candidate"), ("hotfix/1.0.1", "", "hotfix"),
|
||||
("", "v1.0.0", "release")):
|
||||
result = pipeline.qbit_template(loaded, branch, tag)
|
||||
options = next(iter(next(iter(result["workflows"].values()))["jobs"][0].values()))
|
||||
self.assertEqual(kind, options.get("branch_type"))
|
||||
if kind is None:
|
||||
self.assertEqual(["automation/qbit-rc-pending", pattern], options["filters"]["branches"]["ignore"])
|
||||
|
||||
def test_630_original_filter_intersections_equal_generated_publications(self):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"publication_pipeline", Path(__file__).resolve().parents[1] / "src/scripts/generate_pipeline.py")
|
||||
@@ -76,6 +109,24 @@ class MigrationTests(unittest.TestCase):
|
||||
manifest = migration.infer_manifest(self.config(), "0.7.0", "main")
|
||||
self.assertFalse(any(manifest["publish"].values()))
|
||||
|
||||
def test_managed_track_preserves_policy_and_caller_pins(self):
|
||||
fixtures = json.loads((Path(__file__).parent / "fixtures/qbit-publication-policies.json").read_text(encoding="utf-8"))
|
||||
for fixture in fixtures:
|
||||
with self.subTest(repository=fixture["name"]):
|
||||
expected = migration.infer_manifest(fixture["config"], "0.8.1", fixture["default_branch"])
|
||||
expected["orb_version"] = "0.8"
|
||||
actual = migration.infer_manifest(fixture["config"], "0.8", fixture["default_branch"])
|
||||
self.assertEqual(actual, expected)
|
||||
caller = migration.canonical_caller("0.8", actual["release_tag_policy"], actual["publish"].get("feature", False))
|
||||
self.assertEqual(caller["orbs"]["qqq-orb"], "kingsrook/qqq-orb@0.8")
|
||||
self.assertEqual(caller["workflows"]["setup"]["jobs"][0]["qqq-orb/generate_pipeline"]["orb_version"], "0.8")
|
||||
self.assertEqual(yaml.safe_load(yaml.safe_dump(actual))["orb_version"], "0.8")
|
||||
|
||||
def test_reject_ambiguous_or_invalid_versions(self):
|
||||
for version in (0.8, None, "0", "latest", "volatile", "0.8.*", "0.8.1.2", ""):
|
||||
with self.subTest(version=version), self.assertRaises(ValueError):
|
||||
migration.infer_manifest(self.config(), version, "main")
|
||||
|
||||
def test_snapshot_does_not_enable_rc_or_release(self):
|
||||
manifest = migration.infer_manifest(
|
||||
self.config("qqq-orb/mvn_publish", branch_type="snapshot",
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Maven failures must survive tee without overriding project analysis policy."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = [('mvn_spotbugs.sh', 'SPOTBUGS_FAIL_ON_ERROR', 'spotbugs.failOnError'),
|
||||
('mvn_pmd.sh', 'PMD_FAIL_ON_VIOLATION', 'pmd.failOnViolation')]
|
||||
|
||||
|
||||
class StaticAnalysisExitTests(unittest.TestCase):
|
||||
def invoke(self, script, variable, setting, status):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
binary = root / 'mvn'
|
||||
binary.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$ARGUMENT_FILE"\necho analysis-output\nexit "$FAKE_MAVEN_EXIT"\n')
|
||||
binary.chmod(0o755)
|
||||
env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ['PATH'],
|
||||
ARGUMENT_FILE=str(root / 'arguments'), FAKE_MAVEN_EXIT=str(status))
|
||||
env.pop(variable, None)
|
||||
if setting is not None:
|
||||
env[variable] = setting
|
||||
result = subprocess.run(['bash', str(ROOT / 'src/scripts' / script)], cwd=root,
|
||||
env=env, text=True, capture_output=True, timeout=15)
|
||||
return result, (root / 'arguments').read_text().splitlines()
|
||||
|
||||
def test_maven_failure_is_never_hidden_by_successful_tee(self):
|
||||
for script, variable, _ in SCRIPTS:
|
||||
for setting in ('true', 'false'):
|
||||
with self.subTest(script=script, setting=setting):
|
||||
result, _ = self.invoke(script, variable, setting, 17)
|
||||
self.assertEqual(17, result.returncode)
|
||||
self.assertIn('analysis-output', result.stdout)
|
||||
|
||||
def test_success_preserves_explicit_flag_and_project_defaults(self):
|
||||
for script, variable, property_name in SCRIPTS:
|
||||
for setting in ('true', 'false', None):
|
||||
with self.subTest(script=script, setting=setting):
|
||||
result, arguments = self.invoke(script, variable, setting, 0)
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
if setting == 'true':
|
||||
self.assertIn('-D' + property_name + '=true', arguments)
|
||||
else:
|
||||
self.assertFalse(any(arg.startswith('-D' + property_name + '=') for arg in arguments))
|
||||
self.assertIn('-DskipTests', arguments)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
+99
-22
@@ -6,6 +6,10 @@ Example: python tools/audit_estate.py --inventory inventory.json --orb-version 0
|
||||
Policy: {"profiles":{"qbit":{"required_checks":{"main":["observed context"],
|
||||
"develop":["observed context"]}}},"repositories":{"QRun-IO/example":{...}}}.
|
||||
Only use contexts verified on hosted canaries. Absent policy is unknown, not pass.
|
||||
Use --live --source-branch main (or release/...) to inspect that branch's source;
|
||||
omitting it retains default-branch auditing. Unavailable branches remain unknown.
|
||||
A profile/repository policy may set rc_worker_sha independently of --worker-sha
|
||||
so an RC-only worker change does not require updating unchanged native workers.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
@@ -62,6 +66,38 @@ def gh(path):
|
||||
return json.loads(command(['gh', 'api', '--method', 'GET', path]))
|
||||
|
||||
|
||||
def discover_inventory(organization):
|
||||
"""Discover every repository visible to existing gh authentication, across all pages."""
|
||||
if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9-]*', organization):
|
||||
raise ReadError('invalid organization name')
|
||||
pages = json.loads(command(['gh', 'api', '--method', 'GET', '--paginate', '--slurp',
|
||||
f'orgs/{organization}/repos?type=all&per_page=100']))
|
||||
if not isinstance(pages, list) or any(not isinstance(page, list) for page in pages):
|
||||
raise ReadError('invalid organization inventory response')
|
||||
entries, seen = [], set()
|
||||
for page in pages:
|
||||
for repo in page:
|
||||
if not isinstance(repo, dict):
|
||||
raise ReadError('invalid organization repository')
|
||||
name = repo.get('name')
|
||||
full_name = repo.get('full_name')
|
||||
archived = repo.get('archived')
|
||||
branch = repo.get('default_branch')
|
||||
if (not isinstance(name, str) or not re.fullmatch(r'[A-Za-z0-9_.-]+', name)
|
||||
or not isinstance(full_name, str)
|
||||
or full_name.lower() != f'{organization}/{name}'.lower()
|
||||
or type(archived) is not bool
|
||||
or not isinstance(branch, str) or not branch
|
||||
or full_name.lower() in seen):
|
||||
raise ReadError('invalid or duplicate organization repository')
|
||||
seen.add(full_name.lower())
|
||||
entries.append({'name': name, 'nameWithOwner': full_name,
|
||||
'isArchived': archived, 'defaultBranchRef': {'name': branch}})
|
||||
if not entries:
|
||||
raise ReadError('empty organization inventory; coverage is unproven')
|
||||
return sorted(entries, key=lambda entry: entry['nameWithOwner'].lower())
|
||||
|
||||
|
||||
class Snapshot:
|
||||
def __init__(self, entry, live=False):
|
||||
self.repo = entry['nameWithOwner']
|
||||
@@ -69,11 +105,17 @@ class Snapshot:
|
||||
raise ReadError('invalid repository name')
|
||||
self.live = live
|
||||
self.path = entry.get('localPath', '')
|
||||
self.branch = (entry.get('defaultBranchRef') or {}).get('name')
|
||||
requested = entry.get('sourceBranch')
|
||||
if requested is not None and (not live or not isinstance(requested, str)
|
||||
or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._/-]*', requested)):
|
||||
raise ReadError('explicit source branch requires a valid branch name and live lookup')
|
||||
self.default_branch = (entry.get('defaultBranchRef') or {}).get('name')
|
||||
self.branch = requested or self.default_branch
|
||||
self.sha = entry.get('sha', '')
|
||||
if live:
|
||||
info = gh('repos/' + self.repo)
|
||||
self.branch = info['default_branch']
|
||||
self.default_branch = info['default_branch']
|
||||
self.branch = requested or self.default_branch
|
||||
if info['archived']:
|
||||
raise ReadError('repository is now archived; refresh inventory')
|
||||
self.sha = gh('repos/' + self.repo + '/commits/' + quote(self.branch, safe=''))['sha']
|
||||
@@ -165,9 +207,10 @@ def owner_gate_errors(snapshot):
|
||||
|
||||
def audit(entry, snapshot, version, worker_sha, policy):
|
||||
name = entry['name']
|
||||
profile = policy.get('profile') or ('qbit' if name.startswith('qbit-') else PROFILES.get(name))
|
||||
profile = policy.get('branch_profiles', {}).get(snapshot.branch) or policy.get('profile') or ('qbit' if name.startswith('qbit-') else PROFILES.get(name))
|
||||
result = {'repository': entry['nameWithOwner'], 'sha': snapshot.sha,
|
||||
'default_branch': snapshot.branch, 'profile': profile, 'findings': [], 'required_checks': {}}
|
||||
'default_branch': getattr(snapshot, 'default_branch', snapshot.branch),
|
||||
'source_branch': snapshot.branch, 'profile': profile, 'findings': [], 'required_checks': {}}
|
||||
findings = result['findings']
|
||||
def finding(code, path=None, detail=None):
|
||||
findings.append({k: v for k, v in {'code': code, 'path': path, 'detail': detail}.items() if v is not None})
|
||||
@@ -196,7 +239,7 @@ def audit(entry, snapshot, version, worker_sha, policy):
|
||||
manifest = Path(directory) / 'manifest.yml'
|
||||
manifest.write_text(raw, encoding='utf-8')
|
||||
generator.load_manifest(manifest, pin)
|
||||
if data.get('default_branch') != snapshot.branch:
|
||||
if data.get('default_branch') != result['default_branch']:
|
||||
finding('default_branch_drift', '.qqq-ci.yml')
|
||||
if caller is not None:
|
||||
standard.validate(caller, raw, pin)
|
||||
@@ -206,14 +249,15 @@ def audit(entry, snapshot, version, worker_sha, policy):
|
||||
native_paths = set()
|
||||
rc_source = snapshot.read(rc_worker.CALLER_PATH)
|
||||
existing_rc = rc_source is not None and 'qbit-release-rc.yml@' in rc_source
|
||||
if profile == 'qbit' and (rc_enabled or existing_rc):
|
||||
if profile in ('qbit', 'repository-policy') and (rc_enabled or existing_rc):
|
||||
path = rc_worker.CALLER_PATH
|
||||
raw = rc_source
|
||||
rc_sha = policy.get("rc_worker_sha", worker_sha)
|
||||
if raw is None:
|
||||
finding('missing_rc_bridge', path)
|
||||
elif not re.fullmatch(r'[0-9a-f]{40}', worker_sha or ''):
|
||||
elif not re.fullmatch(r'[0-9a-f]{40}', rc_sha or ''):
|
||||
finding('native_pin_policy_missing', path)
|
||||
elif raw != rc_worker.generate(worker_sha):
|
||||
elif raw != rc_worker.generate(rc_sha):
|
||||
finding('rc_bridge_drift', path)
|
||||
else:
|
||||
native_paths.add(path)
|
||||
@@ -221,17 +265,30 @@ def audit(entry, snapshot, version, worker_sha, policy):
|
||||
if other != path and 'qbit-release-rc.yml@' in snapshot.read(other):
|
||||
finding('duplicate_rc_bridge', other)
|
||||
for kind, definition in auxiliary.BRIDGES.items():
|
||||
if definition['repository'] != entry['nameWithOwner']:
|
||||
bridge_sha = worker_sha
|
||||
if kind == 'qbit-codeql':
|
||||
if profile not in ('qbit', 'repository-policy', 'all') or 'codeql_worker_sha' not in policy:
|
||||
continue
|
||||
bridge_sha = policy['codeql_worker_sha']
|
||||
elif definition['repository'] != entry['nameWithOwner']:
|
||||
continue
|
||||
path = definition['path']
|
||||
raw = snapshot.read(path)
|
||||
if kind == 'all-release' and profile == 'repository-policy' and raw is None:
|
||||
continue
|
||||
# Older release branches can predate Authentik's schema contract test.
|
||||
# Keep the default branch required, and validate every existing bridge.
|
||||
if (kind == 'qbit-schema-drift' and raw is None
|
||||
and snapshot.branch != result['default_branch']
|
||||
and not any(path.endswith('/SchemaDriftTest.java') for path in snapshot.paths)):
|
||||
continue
|
||||
if raw is None:
|
||||
finding('missing_native_maintenance_bridge', path)
|
||||
continue
|
||||
if not re.fullmatch(r'[0-9a-f]{40}', worker_sha or ''):
|
||||
if not re.fullmatch(r'[0-9a-f]{40}', bridge_sha or ''):
|
||||
finding('native_pin_policy_missing', path)
|
||||
continue
|
||||
expected = generator.yaml.safe_dump(auxiliary.canonical(kind, worker_sha), sort_keys=False)
|
||||
expected = generator.yaml.safe_dump(auxiliary.canonical(kind, bridge_sha), sort_keys=False)
|
||||
if raw != expected:
|
||||
finding('native_maintenance_bridge_drift', path)
|
||||
else:
|
||||
@@ -293,9 +350,11 @@ def audit(entry, snapshot, version, worker_sha, policy):
|
||||
branches = policy.get('required_checks', {})
|
||||
managed = policy.get('managed_branches', ['main', 'develop'])
|
||||
if (not isinstance(managed, list) or not managed
|
||||
or any(branch not in ('main', 'develop') for branch in managed)
|
||||
or any(not isinstance(branch, str) or not re.fullmatch(
|
||||
r'(?:main|develop|(?:release|hotfix)/[0-9]+\.[0-9]+(?:\.[0-9]+)?)', branch)
|
||||
for branch in managed)
|
||||
or len(set(managed)) != len(managed)):
|
||||
raise ReadError('managed_branches must be a nonempty unique main/develop list')
|
||||
raise ReadError('managed_branches must be a nonempty unique main/develop or numeric release/hotfix list')
|
||||
result['managed_branches'] = managed
|
||||
for branch in managed:
|
||||
if branch not in branches:
|
||||
@@ -321,9 +380,13 @@ def audit(entry, snapshot, version, worker_sha, policy):
|
||||
return result
|
||||
|
||||
|
||||
def run(entries, version, worker_sha, policies, live=False, snapshot_factory=Snapshot):
|
||||
def run(entries, version, worker_sha, policies, live=False, snapshot_factory=Snapshot, source_branch=None):
|
||||
if source_branch is not None and not live:
|
||||
raise ValueError('Explicit source branch audits require --live')
|
||||
results = []
|
||||
for entry in entries:
|
||||
if source_branch is not None:
|
||||
entry = dict(entry, sourceBranch=source_branch)
|
||||
if entry.get('isArchived'):
|
||||
results.append({'repository': entry['nameWithOwner'], 'status': 'archived_skip'})
|
||||
continue
|
||||
@@ -334,27 +397,41 @@ def run(entries, version, worker_sha, policies, live=False, snapshot_factory=Sna
|
||||
try:
|
||||
results.append(audit(entry, snapshot_factory(entry, live), version, worker_sha, policy))
|
||||
except (ReadError, OSError, ValueError, KeyError, subprocess.TimeoutExpired):
|
||||
results.append({'repository': entry['nameWithOwner'], 'status': 'unknown', 'reason': 'source/API/policy read failed'})
|
||||
results.append({'repository': entry['nameWithOwner'], 'source_branch': source_branch,
|
||||
'status': 'unknown', 'reason': 'source/API/policy read failed'})
|
||||
return {'inventory_count': len(entries), 'active_count': sum(not e.get('isArchived', False) for e in entries),
|
||||
'covered_count': len(results), 'unique_repository_count': len({e['nameWithOwner'] for e in entries}),
|
||||
'live': live, 'results': results}
|
||||
'live': live, 'source_branch_selector': source_branch or 'default', 'results': results}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--inventory', type=Path, required=True)
|
||||
inventory = parser.add_mutually_exclusive_group(required=True)
|
||||
inventory.add_argument('--inventory', type=Path)
|
||||
inventory.add_argument('--organization', help='Discover current repositories using existing gh authentication; requires --live')
|
||||
parser.add_argument('--orb-version', required=True)
|
||||
parser.add_argument('--worker-sha')
|
||||
parser.add_argument('--policy', type=Path)
|
||||
parser.add_argument('--live', action='store_true', help='Read current source and branch rules using existing gh authentication')
|
||||
parser.add_argument('--expected-active', type=int, default=68)
|
||||
parser.add_argument('--source-branch', help='Inspect this branch instead of the repository default; requires --live')
|
||||
parser.add_argument('--expected-active', type=int, help='Optional independent coverage check; saved inventories default to 68')
|
||||
parser.add_argument('--output', type=Path)
|
||||
args = parser.parse_args()
|
||||
entries = json.loads(args.inventory.read_text(encoding='utf-8'))
|
||||
if args.source_branch is not None and not args.live:
|
||||
parser.error('--source-branch requires --live')
|
||||
if args.organization and not args.live:
|
||||
parser.error('--organization requires --live')
|
||||
entries = (discover_inventory(args.organization) if args.organization else
|
||||
json.loads(args.inventory.read_text(encoding='utf-8')))
|
||||
expected_active = args.expected_active
|
||||
if expected_active is None and not args.organization:
|
||||
expected_active = 68
|
||||
policies = json.loads(args.policy.read_text(encoding='utf-8')) if args.policy else {}
|
||||
report = run(entries, args.orb_version, args.worker_sha, policies, args.live)
|
||||
report['expected_active_count'] = args.expected_active
|
||||
report['coverage_matches_expected'] = (report['active_count'] == args.expected_active
|
||||
report = run(entries, args.orb_version, args.worker_sha, policies, args.live, source_branch=args.source_branch)
|
||||
report['inventory_source'] = ('organization-api' if args.organization else 'saved-inventory')
|
||||
report['inventory_visibility'] = 'authenticated repositories; inaccessible repositories cannot be certified'
|
||||
report['expected_active_count'] = expected_active
|
||||
report['coverage_matches_expected'] = ((expected_active is None or report['active_count'] == expected_active)
|
||||
and report['unique_repository_count'] == report['inventory_count'])
|
||||
text = json.dumps(report, indent=2) + '\n'
|
||||
if args.output:
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Discover current enrollment targets without assuming unknown repositories conform."""
|
||||
import audit_estate
|
||||
|
||||
|
||||
def discover(organization, version, inspect, request, inventory=None):
|
||||
"""Inspect every active main/develop branch; retain unsupported results explicitly."""
|
||||
entries = audit_estate.discover_inventory(organization) if inventory is None else inventory
|
||||
plans, findings = [], []
|
||||
for entry in entries:
|
||||
repo = entry['nameWithOwner']
|
||||
if entry['isArchived']:
|
||||
findings.append({'repository': repo, 'state': 'archived'})
|
||||
continue
|
||||
if repo.lower() == (organization + '/qqq-orb').lower():
|
||||
findings.append({'repository': repo, 'state': 'owner-bootstrap',
|
||||
'reason': 'Orb publisher is not its own consumer'})
|
||||
continue
|
||||
try:
|
||||
branches, page = [], 1
|
||||
while True:
|
||||
batch = request(f'repos/{repo}/branches?per_page=100&page={page}')
|
||||
if not isinstance(batch, list) or any(not isinstance(b, dict) or not isinstance(b.get('name'), str) for b in batch):
|
||||
raise ValueError('Invalid branch inventory')
|
||||
branches.extend(b['name'] for b in batch)
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
if len(set(branches)) != len(branches):
|
||||
raise ValueError('Duplicate branch inventory')
|
||||
selected = sorted(set(branches) & {'main', 'develop'})
|
||||
if not selected:
|
||||
raise ValueError('No current main/develop branch; explicit profile decision required')
|
||||
except (ValueError, OSError) as error:
|
||||
findings.append({'repository': repo, 'state': 'unresolved', 'reason': str(error)})
|
||||
continue
|
||||
for branch in selected:
|
||||
target = {'repository': repo, 'branch': branch}
|
||||
try:
|
||||
plans.append(inspect(target, version))
|
||||
except (ValueError, OSError) as error:
|
||||
findings.append({**target, 'state': 'unresolved', 'reason': str(error)})
|
||||
return plans, findings
|
||||
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Batch-enforce observed orb checks; dry-run unless --apply. Never changes source."""
|
||||
import argparse
|
||||
import base64
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location('standard', ROOT / 'src/scripts/validate_standard.py')
|
||||
standard = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(standard)
|
||||
DESIRED = [{'context': name, 'app_id': 18001} for name in ('setup', 'verify')]
|
||||
PROFILES = {'qbit', 'core', 'frontend-core', 'frontend-material', 'repository-policy'}
|
||||
|
||||
|
||||
def same_checks(left, right):
|
||||
"""GitHub may reorder checks; retain exact bindings and duplicate counts."""
|
||||
return sorted(json.dumps(check, sort_keys=True) for check in left) == sorted(
|
||||
json.dumps(check, sort_keys=True) for check in right)
|
||||
|
||||
|
||||
def api(path, method='GET', payload=None):
|
||||
"""Use existing gh authentication; never expose response bodies on failure."""
|
||||
args = ['gh', 'api', '--method', method, path]
|
||||
if payload is not None:
|
||||
args += ['--input', '-']
|
||||
result = subprocess.run(args, input=None if payload is None else json.dumps(payload),
|
||||
text=True, encoding='utf-8', capture_output=True, timeout=60)
|
||||
if result.returncode:
|
||||
try:
|
||||
error = json.loads(result.stdout)
|
||||
except ValueError:
|
||||
error = {}
|
||||
if method == 'GET' and path.endswith('/protection') and error.get('status') == '404' and error.get('message') == 'Branch not protected':
|
||||
return None
|
||||
raise ValueError('GitHub operation unavailable: ' + method + ' ' + path.split('?')[0])
|
||||
return json.loads(result.stdout) if result.stdout.strip() else None
|
||||
|
||||
|
||||
def required_checks(before, runs):
|
||||
"""Append app-bound gates only after both succeeded; preserve all other gates."""
|
||||
for desired in DESIRED:
|
||||
matches = [run for run in runs if run['name'] == desired['context'] and run['app']['id'] == desired['app_id']]
|
||||
if not matches or max(matches, key=lambda run: run['id'])['conclusion'] != 'success':
|
||||
raise ValueError('Current branch head has no successful CircleCI ' + desired['context'])
|
||||
prior = (before or {}).get('required_status_checks') or {}
|
||||
checks = [dict(check) for check in prior.get('checks', [])]
|
||||
represented = {check['context'] for check in checks}
|
||||
checks += [{'context': name, 'app_id': -1} for name in prior.get('contexts', []) if name not in represented]
|
||||
for desired in DESIRED:
|
||||
matches = [check for check in checks if check['context'] == desired['context']]
|
||||
if any(check.get('app_id') not in (None, -1, desired['app_id']) for check in matches):
|
||||
raise ValueError('Existing gate uses a different app: ' + desired['context'])
|
||||
checks = [check for check in checks if check['context'] != desired['context']]
|
||||
checks.append(dict(desired))
|
||||
return {'strict': True, 'checks': checks}
|
||||
|
||||
|
||||
def verified_rulesets(repo, effective, request=api):
|
||||
"""Recognize active repository rulesets with bound gates and no bypass actors."""
|
||||
selected = [rule for rule in effective if rule['type'] == 'required_status_checks']
|
||||
if not selected or any(rule.get('ruleset_source_type') != 'Repository'
|
||||
or rule.get('ruleset_source') != repo for rule in selected):
|
||||
raise ValueError('Required rulesets need unsupported source reconciliation')
|
||||
records = []
|
||||
for identifier in sorted({rule['ruleset_id'] for rule in selected}):
|
||||
record = request(f'repos/{repo}/rulesets/{identifier}')
|
||||
if (record.get('enforcement') != 'active' or record.get('target') != 'branch'
|
||||
or record.get('source_type') != 'Repository' or record.get('source') != repo
|
||||
or record.get('bypass_actors') != []):
|
||||
raise ValueError('Ruleset must be active with no bypass actors')
|
||||
records.append(record)
|
||||
requirements = [rule['parameters'] for record in records for rule in record['rules']
|
||||
if rule['type'] == 'required_status_checks']
|
||||
for desired in DESIRED:
|
||||
if not any(check['context'] == desired['context'] and check.get('integration_id') == desired['app_id']
|
||||
for parameters in requirements for check in parameters['required_status_checks']):
|
||||
raise ValueError('Ruleset is missing app-bound gate: ' + desired['context'])
|
||||
# Retain the existing freshness policy: recognition does not add rebuilds or
|
||||
# weaken any existing scanner, merge, or bypass requirement.
|
||||
return records, any(parameters.get('strict_required_status_checks_policy', False) for parameters in requirements)
|
||||
|
||||
|
||||
def inspect(repo, branch, version, request=api):
|
||||
"""Read immutable canonical source, latest check runs, and current protections."""
|
||||
if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repo) or branch not in ('main', 'develop'):
|
||||
raise ValueError('Targets must name an exact repository and main/develop branch')
|
||||
root = 'repos/' + repo
|
||||
info = request(root)
|
||||
if info['archived']:
|
||||
raise ValueError('Archived repository')
|
||||
prefix = root + '/branches/' + branch
|
||||
head = request(prefix)['commit']['sha']
|
||||
source = []
|
||||
for path in ('.circleci/config.yml', '.qqq-ci.yml'):
|
||||
item = request(root + '/contents/' + quote(path, safe='/') + '?ref=' + head)
|
||||
if item.get('encoding') != 'base64' or item.get('type') != 'file':
|
||||
raise ValueError('Canonical declarations must be regular files')
|
||||
source.append(base64.b64decode(item['content'], validate=False).decode('utf-8'))
|
||||
standard.validate(*source, version)
|
||||
if standard.load_yaml(source[1])['profile'] not in PROFILES:
|
||||
raise ValueError('Profile does not use the shared setup/verify gate pair')
|
||||
rules = request(root + '/rules/branches/' + branch)
|
||||
rulesets, ruleset_strict = verified_rulesets(repo, rules, request) if rules else ([], False)
|
||||
before = request(prefix + '/protection')
|
||||
restrictions = (before or {}).get('restrictions')
|
||||
use_ruleset = (not rulesets and restrictions is not None
|
||||
and not any(restrictions.get(kind) for kind in ('users', 'teams', 'apps'))
|
||||
and not (before or {}).get('enforce_admins', {}).get('enabled'))
|
||||
runs = []
|
||||
page = 1
|
||||
while True:
|
||||
batch = request(root + '/commits/' + head + '/check-runs?filter=latest&per_page=100&page=' + str(page))['check_runs']
|
||||
runs += batch
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
desired = required_checks(before, runs)
|
||||
if use_ruleset:
|
||||
return {'repository': repo, 'branch': branch, 'head': head, 'before': before,
|
||||
'required_status_checks': {'strict': False, 'checks': DESIRED},
|
||||
'changed': True, 'protection_mode': 'create-ruleset'}
|
||||
if rulesets:
|
||||
desired['strict'] = ruleset_strict or bool(((before or {}).get('required_status_checks') or {}).get('strict'))
|
||||
return {'repository': repo, 'branch': branch, 'head': head, 'before': before,
|
||||
'required_status_checks': desired, 'changed': False, 'protection_mode': 'rulesets',
|
||||
'effective_rules': rules, 'rulesets': rulesets}
|
||||
current = (before or {}).get('required_status_checks') or {}
|
||||
ready = (current.get('strict') is True and same_checks(current.get('checks', []), desired['checks'])
|
||||
and (before or {}).get('enforce_admins', {}).get('enabled') is True)
|
||||
return {'repository': repo, 'branch': branch, 'head': head, 'before': before,
|
||||
'required_status_checks': desired, 'changed': not ready}
|
||||
|
||||
|
||||
def enforce(plan, request=api):
|
||||
"""Recheck drift, use narrow updates, and verify preservation of other settings."""
|
||||
root = 'repos/' + plan['repository']
|
||||
prefix = root + '/branches/' + plan['branch']
|
||||
if request(prefix)['commit']['sha'] != plan['head'] or request(prefix + '/protection') != plan['before']:
|
||||
raise ValueError('Branch or protection changed after preflight')
|
||||
rules = request(root + '/rules/branches/' + plan['branch'])
|
||||
if plan.get('protection_mode') == 'rulesets':
|
||||
if rules != plan['effective_rules'] or verified_rulesets(plan['repository'], rules, request)[0] != plan['rulesets']:
|
||||
raise ValueError('Ruleset changed after preflight')
|
||||
return
|
||||
if rules:
|
||||
raise ValueError('Ruleset changed after preflight')
|
||||
if plan.get('protection_mode') == 'create-ruleset':
|
||||
name = 'QQQ automated verification: ' + plan['branch']
|
||||
page = 1
|
||||
while True:
|
||||
existing = request(root + '/rulesets?per_page=100&page=' + str(page))
|
||||
if any(record['name'] == name for record in existing):
|
||||
raise ValueError('Named ruleset already exists; reconcile its scope explicitly')
|
||||
if len(existing) < 100:
|
||||
break
|
||||
page += 1
|
||||
request(root + '/rulesets', 'POST', {
|
||||
'name': name, 'target': 'branch', 'enforcement': 'active', 'bypass_actors': [],
|
||||
'conditions': {'ref_name': {'include': ['refs/heads/' + plan['branch']], 'exclude': []}},
|
||||
'rules': [{'type': 'required_status_checks', 'parameters': {
|
||||
'strict_required_status_checks_policy': False, 'do_not_enforce_on_create': False,
|
||||
'required_status_checks': [{'context': gate['context'], 'integration_id': gate['app_id']}
|
||||
for gate in DESIRED]}}]})
|
||||
effective = request(root + '/rules/branches/' + plan['branch'])
|
||||
verified_rulesets(plan['repository'], effective, request)
|
||||
if request(prefix + '/protection') != plan['before']:
|
||||
raise ValueError('Classic access policy changed during ruleset creation')
|
||||
return
|
||||
if not plan['changed']:
|
||||
return
|
||||
before = plan['before']
|
||||
if before is None:
|
||||
request(prefix + '/protection', 'PUT', {
|
||||
'required_status_checks': plan['required_status_checks'], 'enforce_admins': True,
|
||||
'required_pull_request_reviews': None, 'restrictions': None})
|
||||
else:
|
||||
request(prefix + '/protection/required_status_checks', 'PATCH', plan['required_status_checks'])
|
||||
if not before['enforce_admins']['enabled']:
|
||||
request(prefix + '/protection/enforce_admins', 'POST')
|
||||
after = request(prefix + '/protection')
|
||||
actual = after['required_status_checks']
|
||||
if not actual['strict'] or not same_checks(actual['checks'], plan['required_status_checks']['checks']) or not after['enforce_admins']['enabled']:
|
||||
raise ValueError('Protection readback did not match requested gates')
|
||||
if before:
|
||||
for key, value in before.items():
|
||||
if key not in ('required_status_checks', 'enforce_admins') and after.get(key) != value:
|
||||
raise ValueError('Unrelated protection changed: ' + key)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--targets', type=Path, required=True, help='JSON list of {repository, branch}')
|
||||
parser.add_argument('--orb-version', required=True)
|
||||
parser.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
targets = json.loads(args.targets.read_text(encoding='utf-8'))
|
||||
if not isinstance(targets, list) or not targets:
|
||||
raise ValueError('A nonempty explicit target list is required')
|
||||
if len({(row['repository'], row['branch']) for row in targets}) != len(targets):
|
||||
raise ValueError('Duplicate branch target')
|
||||
# Resolve the entire batch before any mutations. Failures never trigger rollback
|
||||
# that would remove successful protections; rerunning converges idempotently.
|
||||
plans = [inspect(row['repository'], row['branch'], args.orb_version) for row in targets]
|
||||
args.output.write_text(json.dumps({'state': 'planned', 'plans': plans}, indent=2), encoding='utf-8')
|
||||
if args.apply:
|
||||
for plan in plans:
|
||||
enforce(plan)
|
||||
plan['verified'] = True
|
||||
args.output.write_text(json.dumps({'state': 'applying', 'plans': plans}, indent=2), encoding='utf-8')
|
||||
args.output.write_text(json.dumps({'state': 'verified', 'plans': plans}, indent=2), encoding='utf-8')
|
||||
print(json.dumps({'targets': len(plans), 'changed': sum(plan['changed'] for plan in plans), 'applied': args.apply}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Enroll a batch of audited consumers; preview by default, write with --write.
|
||||
|
||||
Existing canonical profiles retain their manifests. Legacy QBits use the shared
|
||||
policy inference. Unknown orchestration is rejected before any checkout changes.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
|
||||
import yaml
|
||||
|
||||
import rollout_version as rollout
|
||||
import standardize_qbit as qbit
|
||||
|
||||
|
||||
def declaration(root, relative):
|
||||
"""Reject redirected declarations, including symlinked parent directories."""
|
||||
path = root / relative
|
||||
current = path
|
||||
while current != root:
|
||||
if current.is_symlink():
|
||||
raise ValueError('CI declarations must not use symlinks')
|
||||
current = current.parent
|
||||
if not path.resolve().is_relative_to(root):
|
||||
raise ValueError('CI declarations must remain inside the checkout')
|
||||
return path
|
||||
|
||||
|
||||
def plan(target, version):
|
||||
"""Preserve explicit publication policy and validate generated declarations."""
|
||||
if not isinstance(target, dict) or set(target) != {'checkout', 'default_branch'}:
|
||||
raise ValueError('Each target requires checkout and observed default_branch')
|
||||
if target['default_branch'] not in ('main', 'develop'):
|
||||
raise ValueError('Unsupported default branch')
|
||||
root = Path(target['checkout']).resolve(strict=True)
|
||||
config = declaration(root, '.circleci/config.yml')
|
||||
manifest = declaration(root, '.qqq-ci.yml')
|
||||
if manifest.exists():
|
||||
data = rollout.standard.load_yaml(manifest.read_text(encoding='utf-8'))
|
||||
if not isinstance(data, dict) or data.get('default_branch') != target['default_branch']:
|
||||
raise ValueError('Observed default branch differs from manifest')
|
||||
return {**rollout.plan(root, version), 'enrollment': False}
|
||||
original = config.read_bytes()
|
||||
data = qbit.infer_manifest(rollout.standard.load_yaml(original.decode('utf-8')),
|
||||
version, target['default_branch'])
|
||||
caller = rollout.standard.canonical_caller(data['profile'], version,
|
||||
release_tag_policy=data['release_tag_policy'], publish_feature=data['publish'].get('feature', False))
|
||||
config_raw = yaml.safe_dump(caller, sort_keys=False)
|
||||
manifest_raw = yaml.safe_dump(data, sort_keys=False)
|
||||
rollout.standard.validate(config_raw, manifest_raw, version)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
temporary_manifest = Path(directory) / 'manifest.yml'
|
||||
temporary_manifest.write_text(manifest_raw, encoding='utf-8')
|
||||
rollout.pipeline.load_manifest(temporary_manifest, version)
|
||||
return {'checkout': str(root), 'profile': data['profile'], 'from': None,
|
||||
'to': version, 'enrollment': True,
|
||||
'files': [(config, original, config_raw.encode('utf-8')),
|
||||
(manifest, None, manifest_raw.encode('utf-8'))]}
|
||||
|
||||
|
||||
def enroll(targets, version, write=False):
|
||||
"""Validate the whole batch before writing; repeat applications are no-ops."""
|
||||
if not isinstance(version, str) or not re.fullmatch(r'\d+\.\d+(?:\.\d+)?', version):
|
||||
raise ValueError('Enrollment requires a stable minor track or exact patch')
|
||||
if not isinstance(targets, list) or not targets:
|
||||
raise ValueError('A nonempty target list is required')
|
||||
plans = [plan(target, version) for target in targets]
|
||||
if len({item['checkout'] for item in plans}) != len(plans):
|
||||
raise ValueError('Each checkout may appear only once')
|
||||
if write:
|
||||
for item in plans:
|
||||
for path, original, replacement in item['files']:
|
||||
declaration(Path(item['checkout']), path.relative_to(item['checkout']))
|
||||
current = path.read_bytes() if path.exists() else None
|
||||
if current != original:
|
||||
raise ValueError('Declaration changed after preflight')
|
||||
for item in plans:
|
||||
for path, original, replacement in item['files']:
|
||||
if replacement != original:
|
||||
path.write_bytes(replacement)
|
||||
return [{**{key: value for key, value in item.items() if key != 'files'},
|
||||
'changed': any(old != new for _, old, new in item['files']),
|
||||
'written': write} for item in plans]
|
||||
|
||||
|
||||
def main():
|
||||
"""Read one fleet target file and emit an auditable result."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('targets', type=Path)
|
||||
parser.add_argument('--orb-version', required=True)
|
||||
parser.add_argument('--write', action='store_true')
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(enroll(json.loads(args.targets.read_text(encoding='utf-8')),
|
||||
args.orb_version, args.write), indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Qualify signed enrollment PRs and enforce only successful current branch heads."""
|
||||
import json
|
||||
|
||||
import enforce_orb_checks as enforcement
|
||||
import reconcile_enrollment as remote
|
||||
|
||||
|
||||
def qualification(repo, head):
|
||||
"""Require both app-bound orb gates and every advertised latest check/status."""
|
||||
checks, page = [], 1
|
||||
while True:
|
||||
batch = remote.api(f'repos/{repo}/commits/{head}/check-runs?filter=latest&per_page=100&page={page}')['check_runs']
|
||||
checks.extend(batch)
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
reasons = []
|
||||
for name in ('setup', 'verify'):
|
||||
found = [check for check in checks if check['name'] == name and check['app']['id'] == 18001]
|
||||
if not found or max(found, key=lambda check: check['id'])['conclusion'] != 'success':
|
||||
reasons.append(name)
|
||||
for check in checks:
|
||||
if check['status'] != 'completed' or check['conclusion'] not in ('success', 'neutral', 'skipped'):
|
||||
reasons.append(check['name'])
|
||||
page = 1
|
||||
while True:
|
||||
batch = remote.api(f'repos/{repo}/commits/{head}/status?per_page=100&page={page}')['statuses']
|
||||
reasons.extend(status['context'] for status in batch if status['state'] != 'success')
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
return sorted(set(reasons))
|
||||
|
||||
|
||||
def merge_ready(plan, result):
|
||||
"""Normal exact-head squash merge only; no administrative or force bypass."""
|
||||
if plan['profile'] not in enforcement.PROFILES:
|
||||
return {'state': 'unsupported-gates', 'profile': plan['profile']}
|
||||
if result['state'] == 'unchanged':
|
||||
return {'state': 'unchanged'}
|
||||
repo, head = plan['repository'], result['head']
|
||||
remote.verify_submission(plan, head)
|
||||
pr = json.loads(remote.command(['gh', 'pr', 'view', result['url'], '--repo', repo,
|
||||
'--json', 'number,state,headRefOid,baseRefOid,baseRefName,mergeCommit']))
|
||||
if pr['headRefOid'] != head or pr['baseRefName'] != plan['branch']:
|
||||
raise ValueError('PR differs from enrollment target')
|
||||
if pr['state'] == 'CLOSED':
|
||||
return {'state': 'closed', 'url': result['url']}
|
||||
if pr['state'] != 'MERGED':
|
||||
if pr['baseRefOid'] != plan['head'] or remote.api(f'repos/{repo}/branches/{plan["branch"]}')['commit']['sha'] != plan['head']:
|
||||
return {'state': 'stale-base', 'url': result['url']}
|
||||
pending = qualification(repo, head)
|
||||
if pending:
|
||||
return {'state': 'waiting-checks', 'checks': pending, 'url': result['url']}
|
||||
remote.command(['gh', 'pr', 'merge', str(pr['number']), '--repo', repo, '--squash', '--match-head-commit', head])
|
||||
pr = json.loads(remote.command(['gh', 'pr', 'view', result['url'], '--repo', repo,
|
||||
'--json', 'state,mergeCommit']))
|
||||
if pr['state'] != 'MERGED':
|
||||
raise ValueError('PR merge was not confirmed')
|
||||
merged = pr['mergeCommit']['oid']
|
||||
expected = remote.api(f'repos/{repo}/commits/{head}')['commit']['tree']['sha']
|
||||
actual = remote.api(f'repos/{repo}/commits/{merged}')['commit']
|
||||
if not actual['verification']['verified'] or actual['tree']['sha'] != expected:
|
||||
raise ValueError('Merged signature or source tree differs from qualified source')
|
||||
return {'state': 'merged', 'url': result['url'], 'head': head, 'merge': merged}
|
||||
|
||||
|
||||
def enforce_current(plan):
|
||||
"""Use shared narrow enforcement after the actual branch head passes its gates."""
|
||||
if plan['profile'] not in enforcement.PROFILES:
|
||||
return {'state': 'unsupported-gates', 'profile': plan['profile']}
|
||||
try:
|
||||
current = enforcement.inspect(plan['repository'], plan['branch'], plan['version'])
|
||||
except ValueError as error:
|
||||
return {'state': 'waiting-enforcement', 'reason': str(error)}
|
||||
enforcement.enforce(current)
|
||||
return {'state': 'enforced', 'head': current['head'], 'changed': current['changed'],
|
||||
'protection_mode': current.get('protection_mode', 'classic')}
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Ubuntu Jammy ships gh 2.4, which lacks API --slurp and modern merge guards.
|
||||
# Pin the official amd64 release and its GitHub release asset SHA-256.
|
||||
version=2.103.0
|
||||
checksum=fdd768eb70f6409f80fcfa9f3a1c0adf0fc98d827ab3ce0bed2ac22762c5d786
|
||||
archive=$(mktemp)
|
||||
binary=$(mktemp)
|
||||
trap 'rm -f "$archive" "$binary"' EXIT
|
||||
curl --fail --silent --show-error --location --retry 3 \
|
||||
"https://github.com/cli/cli/releases/download/v${version}/gh_${version}_linux_amd64.tar.gz" -o "$archive"
|
||||
printf '%s %s\n' "$checksum" "$archive" | sha256sum --check --status
|
||||
tar -xOzf "$archive" "gh_${version}_linux_amd64/bin/gh" > "$binary"
|
||||
sudo install -m 0755 "$binary" /usr/local/bin/gh
|
||||
gh --version
|
||||
gh api --help | grep -q -- --slurp
|
||||
gh pr merge --help | grep -q -- --match-head-commit
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plan remote enrollment and submit signed, repeatable PRs with --apply."""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
import enroll_consumers
|
||||
|
||||
PATHS = ('.circleci/config.yml', '.qqq-ci.yml')
|
||||
|
||||
|
||||
def command(args, cwd=None):
|
||||
"""Use existing credentials and signing configuration without interactive prompts."""
|
||||
env = dict(os.environ, GIT_TERMINAL_PROMPT='0')
|
||||
result = subprocess.run(args, cwd=cwd, env=env, capture_output=True,
|
||||
text=True, encoding='utf-8', timeout=120)
|
||||
if result.returncode:
|
||||
raise ValueError('Command failed: ' + args[0])
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def api(path, missing=False):
|
||||
"""Only an explicit 404 is treated as absence; other API failures stop work."""
|
||||
result = subprocess.run(['gh', 'api', path], capture_output=True,
|
||||
text=True, encoding='utf-8', timeout=60)
|
||||
try:
|
||||
data = json.loads(result.stdout)
|
||||
except ValueError:
|
||||
raise ValueError('Invalid GitHub response') from None
|
||||
if result.returncode:
|
||||
if missing and str(data.get('status')) == '404':
|
||||
return None
|
||||
raise ValueError('GitHub read unavailable: ' + path.split('?')[0])
|
||||
return data
|
||||
|
||||
|
||||
def content(repo, path, head):
|
||||
"""Read a regular declaration at an immutable commit, never at a moving branch."""
|
||||
item = api(f'repos/{repo}/contents/{path}?ref={head}', missing=True)
|
||||
if item is None:
|
||||
return None
|
||||
if item.get('type') != 'file' or item.get('encoding') != 'base64':
|
||||
raise ValueError('Declaration is not a regular file')
|
||||
return base64.b64decode(item['content'])
|
||||
|
||||
|
||||
def mutation(path, payload):
|
||||
"""Use the existing gh identity; never print credentials or response bodies."""
|
||||
result = subprocess.run(['gh', 'api', path, '--method', 'POST', '--input', '-'],
|
||||
input=json.dumps(payload), capture_output=True,
|
||||
text=True, encoding='utf-8', timeout=60)
|
||||
if result.returncode:
|
||||
raise ValueError('GitHub mutation failed; inspect remote state before retrying')
|
||||
try:
|
||||
data = json.loads(result.stdout)
|
||||
except ValueError:
|
||||
raise ValueError('Invalid GitHub mutation response') from None
|
||||
if not isinstance(data, dict) or data.get('errors'):
|
||||
raise ValueError('GitHub mutation did not succeed')
|
||||
return data
|
||||
|
||||
|
||||
def signed_api_commit(plan, ref):
|
||||
"""Resume a base-only branch; append atomically without local Git/SSH keys."""
|
||||
repo, branch, base = plan['repository'], plan['automation_branch'], plan['head']
|
||||
if ref is None:
|
||||
mutation(f'repos/{repo}/git/refs', {'ref': 'refs/heads/' + branch, 'sha': base})
|
||||
elif ref['object']['sha'] != base:
|
||||
return ref['object']['sha']
|
||||
if api(f'repos/{repo}/branches/{plan["branch"]}')['commit']['sha'] != base:
|
||||
raise ValueError('Target branch moved before submission')
|
||||
additions = [{'path': path, 'contents': base64.b64encode(plan['replacements'][path]).decode('ascii')}
|
||||
for path in PATHS if plan['originals'][path] != plan['replacements'][path]]
|
||||
result = mutation('graphql', {
|
||||
'query': 'mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { oid } } }',
|
||||
'variables': {'input': {
|
||||
'branch': {'repositoryNameWithOwner': repo, 'branchName': branch},
|
||||
'expectedHeadOid': base,
|
||||
'message': {'headline': 'ci: enroll in the shared QQQ orb',
|
||||
'body': 'Co-Authored-By: Codex <noreply@openai.com>'},
|
||||
'fileChanges': {'additions': additions}}}})
|
||||
try:
|
||||
head = result['data']['createCommitOnBranch']['commit']['oid']
|
||||
except (KeyError, TypeError):
|
||||
raise ValueError('GitHub commit response is incomplete') from None
|
||||
if not isinstance(head, str) or not re.fullmatch(r'[0-9a-f]{40}', head):
|
||||
raise ValueError('GitHub commit response has no valid SHA')
|
||||
return head
|
||||
|
||||
|
||||
def inspect(target, version):
|
||||
"""Resolve observed repository identity and reuse the shared enrollment planner."""
|
||||
if not isinstance(target, dict) or set(target) != {'repository', 'branch'}:
|
||||
raise ValueError('Each target requires repository and branch')
|
||||
repo, branch = target['repository'], target['branch']
|
||||
if not isinstance(repo, str) or not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repo):
|
||||
raise ValueError('Invalid repository')
|
||||
if branch not in ('main', 'develop'):
|
||||
raise ValueError('Enrollment targets current main/develop only')
|
||||
info = api('repos/' + repo)
|
||||
if info['archived']:
|
||||
raise ValueError('Archived repository')
|
||||
head = api(f'repos/{repo}/branches/{branch}')['commit']['sha']
|
||||
originals = {path: content(repo, path, head) for path in PATHS}
|
||||
if originals[PATHS[0]] is None:
|
||||
raise ValueError('No recognized CircleCI entrypoint')
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / '.circleci').mkdir()
|
||||
for path, data in originals.items():
|
||||
if data is not None:
|
||||
(root / path).write_bytes(data)
|
||||
plan = enroll_consumers.plan({'checkout': str(root), 'default_branch': info['default_branch']}, version)
|
||||
replacements = {str(path.relative_to(root)).replace('\\', '/'): new for path, _, new in plan['files']}
|
||||
digest = hashlib.sha256(head.encode() + b''.join(replacements[path] for path in PATHS)).hexdigest()[:20]
|
||||
return {**target, 'head': head, 'originals': originals, 'replacements': replacements,
|
||||
'profile': plan['profile'], 'version': version,
|
||||
'changed': any(originals[path] != replacements[path] for path in PATHS),
|
||||
'automation_branch': f'automation/qqq-enroll-{branch}-{digest}'}
|
||||
|
||||
|
||||
def verify_submission(plan, head):
|
||||
"""Accept only a signed, one-parent commit with exactly the planned changes."""
|
||||
repo = plan['repository']
|
||||
commit = api(f'repos/{repo}/commits/{head}')
|
||||
expected = {path for path in PATHS if plan['originals'][path] != plan['replacements'][path]}
|
||||
if (not commit['commit']['verification']['verified']
|
||||
or [parent['sha'] for parent in commit['parents']] != [plan['head']]
|
||||
or {item['filename'] for item in commit['files']} != expected):
|
||||
raise ValueError('Existing automation branch differs from the signed plan')
|
||||
for path in PATHS:
|
||||
if content(repo, path, head) != plan['replacements'][path]:
|
||||
raise ValueError('Submitted declaration differs from plan')
|
||||
|
||||
|
||||
def apply(plan):
|
||||
"""Never force push; reuse matching submissions after interrupted executions."""
|
||||
if not plan['changed']:
|
||||
return {'state': 'unchanged'}
|
||||
repo, branch = plan['repository'], plan['automation_branch']
|
||||
if api(f'repos/{repo}/branches/{plan["branch"]}')['commit']['sha'] != plan['head']:
|
||||
raise ValueError('Target branch changed after preflight')
|
||||
ref = api(f'repos/{repo}/git/ref/heads/{branch}', missing=True)
|
||||
head = signed_api_commit(plan, ref)
|
||||
verify_submission(plan, head)
|
||||
pulls = api(f'repos/{repo}/pulls?state=all&head={repo.split("/")[0]}:{branch}&base={plan["branch"]}')
|
||||
if pulls:
|
||||
if len(pulls) != 1 or pulls[0]['head']['sha'] != head:
|
||||
raise ValueError('Ambiguous existing enrollment PR')
|
||||
return {'state': 'existing-pr', 'head': head, 'url': pulls[0]['html_url']}
|
||||
body = ('Uses the shared orb enrollment converter and preserves existing publication policy. '
|
||||
'Only the canonical caller and manifest change. Exact-head hosted checks must pass '
|
||||
'before merge; branch enforcement follows successful qualification.\n\n'
|
||||
'Source base: ' + plan['head'])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / 'body.md'
|
||||
path.write_text(body, encoding='utf-8')
|
||||
url = command(['gh', 'pr', 'create', '--repo', repo, '--base', plan['branch'], '--head', branch,
|
||||
'--title', 'ci: enroll in the shared QQQ orb', '--body-file', str(path)])
|
||||
return {'state': 'submitted', 'head': head, 'url': url}
|
||||
|
||||
|
||||
def main():
|
||||
"""Preflight every explicit target before mutations and save resumable results."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
source = parser.add_mutually_exclusive_group(required=True)
|
||||
source.add_argument('--targets', type=Path)
|
||||
source.add_argument('--organization')
|
||||
parser.add_argument('--orb-version', required=True)
|
||||
parser.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--merge-ready', action='store_true')
|
||||
parser.add_argument('--enforce', action='store_true')
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if (args.merge_ready or args.enforce) and not args.apply:
|
||||
parser.error('--merge-ready and --enforce require --apply')
|
||||
if not re.fullmatch(r'\d+\.\d+(?:\.\d+)?', args.orb_version):
|
||||
raise ValueError('A stable minor track or exact patch is required')
|
||||
findings = []
|
||||
if args.organization:
|
||||
from discover_enrollment import discover
|
||||
plans, findings = discover(args.organization, args.orb_version, inspect, api)
|
||||
else:
|
||||
targets = json.loads(args.targets.read_text(encoding='utf-8'))
|
||||
if not isinstance(targets, list) or not targets:
|
||||
raise ValueError('A nonempty target list is required')
|
||||
plans = [inspect(target, args.orb_version) for target in targets]
|
||||
if len({(p['repository'].lower(), p['branch']) for p in plans}) != len(plans):
|
||||
raise ValueError('Duplicate target')
|
||||
rows = [{key: value for key, value in p.items() if key not in ('originals', 'replacements')} for p in plans]
|
||||
report = findings + rows
|
||||
args.output.write_text(json.dumps(report, indent=2), encoding='utf-8')
|
||||
if args.apply:
|
||||
for plan, row in zip(plans, rows):
|
||||
row['result'] = apply(plan)
|
||||
args.output.write_text(json.dumps(report, indent=2), encoding='utf-8')
|
||||
if args.merge_ready or args.enforce:
|
||||
import finish_enrollment
|
||||
if args.merge_ready:
|
||||
row['qualification'] = finish_enrollment.merge_ready(plan, row['result'])
|
||||
args.output.write_text(json.dumps(report, indent=2), encoding='utf-8')
|
||||
if args.enforce:
|
||||
row['enforcement'] = finish_enrollment.enforce_current(plan)
|
||||
args.output.write_text(json.dumps(report, indent=2), encoding='utf-8')
|
||||
print(json.dumps({'targets': len(plans), 'changed': sum(p['changed'] for p in plans),
|
||||
'unresolved': sum(row['state'] == 'unresolved' for row in findings),
|
||||
'applied': args.apply}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upgrade canonical CircleCI callers in one batch; dry-run unless --write.
|
||||
|
||||
Only the orb pin changes. Repository publication policy and profile parameters
|
||||
are preserved. Git commits, pushes and hosted verification remain separate.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def module(name):
|
||||
spec = importlib.util.spec_from_file_location(name, ROOT / 'src/scripts' / (name + '.py'))
|
||||
result = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(result)
|
||||
return result
|
||||
|
||||
|
||||
standard = module('validate_standard')
|
||||
pipeline = module('generate_pipeline')
|
||||
|
||||
|
||||
def plan(checkout, version):
|
||||
root = Path(checkout).resolve(strict=True)
|
||||
paths = [root / '.circleci/config.yml', root / '.qqq-ci.yml']
|
||||
if any(not path.resolve(strict=True).is_relative_to(root) for path in paths):
|
||||
raise ValueError('CI declarations must remain inside their checkout')
|
||||
originals = [path.read_bytes() for path in paths]
|
||||
caller_raw, manifest_raw = [data.decode('utf-8') for data in originals]
|
||||
manifest = standard.load_yaml(manifest_raw)
|
||||
if not isinstance(manifest, dict):
|
||||
raise ValueError('Existing manifest must be a mapping')
|
||||
previous = manifest.get('orb_version')
|
||||
standard.validate(caller_raw, manifest_raw, previous)
|
||||
pipeline.load_manifest(paths[1], previous)
|
||||
caller = standard.load_yaml(caller_raw)
|
||||
if previous == version:
|
||||
replacements = originals
|
||||
else:
|
||||
manifest['orb_version'] = version
|
||||
caller['orbs']['qqq-orb'] = 'kingsrook/qqq-orb@' + version
|
||||
caller['workflows']['setup']['jobs'][0]['qqq-orb/generate_pipeline']['orb_version'] = version
|
||||
new_caller = yaml.safe_dump(caller, sort_keys=False)
|
||||
new_manifest = yaml.safe_dump(manifest, sort_keys=False)
|
||||
standard.validate(new_caller, new_manifest, version)
|
||||
replacements = [new_caller.encode('utf-8'), new_manifest.encode('utf-8')]
|
||||
return {'checkout': str(root), 'profile': manifest['profile'], 'from': previous, 'to': version,
|
||||
'files': list(zip(paths, originals, replacements))}
|
||||
|
||||
|
||||
def rollout(checkouts, version, write=False):
|
||||
if not re.fullmatch(r'\d+\.\d+(?:\.\d+)?', version):
|
||||
raise ValueError('Fleet upgrades require a stable minor track or exact patch version')
|
||||
# Validate the entire batch before writing any repository.
|
||||
plans = [plan(path, version) for path in checkouts]
|
||||
if len({item['checkout'] for item in plans}) != len(plans):
|
||||
raise ValueError('Each checkout may appear only once')
|
||||
if write:
|
||||
for item in plans:
|
||||
for path, original, replacement in item['files']:
|
||||
if path.read_bytes() != original:
|
||||
raise ValueError('CI declaration changed after preflight; rerun the rollout')
|
||||
for item in plans:
|
||||
for path, original, replacement in item['files']:
|
||||
if replacement != original:
|
||||
path.write_bytes(replacement)
|
||||
return [{**{key: value for key, value in item.items() if key != 'files'},
|
||||
'changed': any(old != new for _, old, new in item['files']), 'written': write}
|
||||
for item in plans]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('checkouts', nargs='+', type=Path)
|
||||
parser.add_argument('--orb-version', required=True)
|
||||
parser.add_argument('--write', action='store_true')
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(rollout(args.checkouts, args.orb_version, args.write), indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (ValueError, OSError) as error:
|
||||
sys.exit('Version rollout rejected: ' + str(error))
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One bounded central reconciliation pass; no executor polling for PR checks."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def run(environment=None, execute=subprocess.run, root=None):
|
||||
env = dict(os.environ if environment is None else environment)
|
||||
if (env.get('CIRCLE_PROJECT_USERNAME') != 'QRun-IO'
|
||||
or env.get('CIRCLE_PROJECT_REPONAME') != 'qqq-orb'
|
||||
or env.get('CIRCLE_BRANCH') != 'main' or env.get('CIRCLE_TAG')):
|
||||
raise ValueError('Reconciliation requires the orb main branch')
|
||||
token = env.get('GH_TOKEN') or env.get('GITHUB_TOKEN')
|
||||
if not token:
|
||||
raise ValueError('Existing GitHub authentication is required')
|
||||
env['GH_TOKEN'] = token
|
||||
root = Path(root) if root else Path(__file__).resolve().parents[1]
|
||||
output = root / 'artifacts/enrollment'
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
args = [sys.executable, str(root / 'tools/reconcile_enrollment.py'),
|
||||
'--organization', 'QRun-IO', '--orb-version', '0.8',
|
||||
'--output', str(output / 'result.json'), '--apply', '--merge-ready', '--enforce']
|
||||
try:
|
||||
result = execute(args, cwd=root, env=env, timeout=1200, check=False)
|
||||
code = result.returncode
|
||||
state = 'completed' if code == 0 else 'failed'
|
||||
except subprocess.TimeoutExpired:
|
||||
code, state = 124, 'timed-out'
|
||||
(output / 'run.json').write_text(json.dumps({'state': state, 'exit_code': code,
|
||||
'report_available': (output / 'result.json').exists(),
|
||||
'note': 'Completion of a pass does not prove all repositories conform; inspect unresolved and waiting entries.'}, indent=2), encoding='utf-8')
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(run())
|
||||
+26
-10
@@ -2,7 +2,7 @@
|
||||
"""Replace audited standard QBit pipelines with the canonical orb caller.
|
||||
|
||||
Only modifies the two CI declarations in a supplied checkout. Git operations and
|
||||
publication remain separate. Unknown jobs/options fail closed for human review.
|
||||
publication remain separate. Unknown jobs/options fail closed for central profile resolution.
|
||||
"""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
@@ -11,7 +11,7 @@ import re
|
||||
import yaml
|
||||
|
||||
|
||||
def canonical_caller(version, release_tag_policy=None):
|
||||
def canonical_caller(version, release_tag_policy=None, publish_feature=False):
|
||||
caller = {
|
||||
"version": 2.1,
|
||||
"setup": True,
|
||||
@@ -28,6 +28,9 @@ def canonical_caller(version, release_tag_policy=None):
|
||||
if release_tag_policy not in filters:
|
||||
raise ValueError("Unknown tag policy")
|
||||
caller["workflows"]["setup"]["jobs"][0]["qqq-orb/generate_pipeline"]["filters"]["tags"] = filters[release_tag_policy]
|
||||
if publish_feature:
|
||||
caller["workflows"]["setup"]["jobs"][0]["qqq-orb/generate_pipeline"]["filters"]["tags"] = {
|
||||
"only": ([filters[release_tag_policy]["only"]] if release_tag_policy != "none" else []) + ["/publish.*/"]}
|
||||
return caller
|
||||
|
||||
|
||||
@@ -43,8 +46,14 @@ def publication_filters(kind, filters):
|
||||
raise ValueError("Unknown publication filters")
|
||||
branches = filters.get("branches")
|
||||
tags = filters.get("tags")
|
||||
if kind in ("snapshot", "release_candidate"):
|
||||
expected = "develop" if kind == "snapshot" else r"/release\/.*/"
|
||||
if kind == "feature":
|
||||
# Preserve the audited literal branch selector; it is not a regex and
|
||||
# contains a character Git forbids, so only publish* tags can select it.
|
||||
if branches != {"only": ["feature/.*"]} or tags != {"only": ["/publish.*/"]}:
|
||||
raise ValueError("Unknown feature publication filters")
|
||||
return False, "none"
|
||||
if kind in ("snapshot", "release_candidate", "hotfix"):
|
||||
expected = {"snapshot": "develop", "release_candidate": r"/release\/.*/", "hotfix": r"/hotfix\/.*/"}[kind]
|
||||
if not isinstance(branches, dict) or set(branches) != {"only"} or single_filter(branches["only"]) != expected or tags not in (None, {"only": []}):
|
||||
raise ValueError("Unknown snapshot/RC filters")
|
||||
return False, "none"
|
||||
@@ -58,7 +67,8 @@ def publication_filters(kind, filters):
|
||||
tag_filter = single_filter(tags["only"])
|
||||
if not isinstance(tag_filter, str):
|
||||
raise ValueError("Unknown release tag filter")
|
||||
policy = {r"/^v\d+\.\d+\.\d+$/": "stable", "/v.*/": "v-prefix"}.get(tag_filter)
|
||||
policy = {r"/^v\d+\.\d+\.\d+$/": "stable", r"/^v[0-9]+\.[0-9]+\.[0-9]+$/": "stable",
|
||||
"/v.*/": "v-prefix"}.get(tag_filter)
|
||||
if policy is None:
|
||||
raise ValueError("Unknown release tag filter")
|
||||
return main, policy
|
||||
@@ -80,6 +90,10 @@ def test_filters(filters):
|
||||
"/(develop|release/.*|integration.*)/": "non-release-no-integration",
|
||||
"/(develop|main|release/.*|integration.*)/": "feature",
|
||||
"/(develop|main|release/.*)/": "non-release-no-main",
|
||||
"/(develop|release/.*|hotfix/.*)/": "non-release-no-hotfix",
|
||||
"/(develop|release/.*|hotfix/.*|integration.*)/": "non-release-no-hotfix-integration",
|
||||
"/(develop|main|release/.*|hotfix/.*)/": "non-release-no-main-hotfix",
|
||||
"/(develop|main|release/.*|hotfix/.*|integration.*)/": "feature-no-hotfix",
|
||||
}.get(pattern.replace(r"\/", "/"))
|
||||
if policy is None:
|
||||
raise ValueError("Unknown test branch filters")
|
||||
@@ -87,8 +101,8 @@ def test_filters(filters):
|
||||
|
||||
|
||||
def infer_manifest(config, version, default_branch):
|
||||
if not re.fullmatch(r"(?:\d+\.\d+\.\d+|dev:[A-Za-z0-9_-]+)", version):
|
||||
raise ValueError("An explicit published orb version is required")
|
||||
if not isinstance(version, str) or not re.fullmatch(r"(?:\d+\.\d+(?:\.\d+)?|dev:[A-Za-z0-9_-]+)", version):
|
||||
raise ValueError("A minor track, exact patch or explicit development orb version is required")
|
||||
if default_branch not in ("main", "develop"):
|
||||
raise ValueError("Unsupported default branch")
|
||||
if set(config) - {"version", "orbs", "commands", "workflows"}:
|
||||
@@ -119,7 +133,7 @@ def infer_manifest(config, version, default_branch):
|
||||
raise ValueError("Custom pre-steps require review")
|
||||
if job.endswith("/mvn_publish"):
|
||||
kind = options.get("branch_type")
|
||||
if kind not in ("snapshot", "release_candidate", "release"):
|
||||
if kind not in ("snapshot", "release_candidate", "release", "hotfix", "feature"):
|
||||
raise ValueError("Custom publication contract requires review")
|
||||
if kind in publications:
|
||||
raise ValueError("Duplicate publication jobs require review")
|
||||
@@ -143,6 +157,8 @@ def infer_manifest(config, version, default_branch):
|
||||
"test_branch_policy": test_policy or "all",
|
||||
"publish": {**{kind: kind in publications
|
||||
for kind in ("snapshot", "release_candidate", "release")},
|
||||
**({"hotfix": True} if "hotfix" in publications else {}),
|
||||
**({"feature": True} if "feature" in publications else {}),
|
||||
"asciidoc": False},
|
||||
}
|
||||
|
||||
@@ -158,9 +174,9 @@ def main():
|
||||
manifest_path = args.checkout / ".qqq-ci.yml"
|
||||
config = yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
||||
if manifest_path.exists():
|
||||
raise ValueError("Already has a manifest; use the version rollout operation")
|
||||
raise ValueError("Already has a manifest; use tools/rollout_version.py")
|
||||
manifest = infer_manifest(config, args.orb_version, args.default_branch)
|
||||
caller = canonical_caller(args.orb_version, manifest["release_tag_policy"])
|
||||
caller = canonical_caller(args.orb_version, manifest["release_tag_policy"], manifest["publish"].get("feature", False))
|
||||
if args.write:
|
||||
for path, data in ((config_path, caller), (manifest_path, manifest)):
|
||||
path.write_bytes(yaml.safe_dump(data, sort_keys=False).encode("utf-8"))
|
||||
|
||||
Reference in new issue
Block a user