diff --git a/.circleci/config.yml b/.circleci/config.yml index 08acef3..f4df765 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,8 @@ version: 2.1 orbs: - orb-tools: circleci/orb-tools@12.1 + # Pinned to an exact version (K22-CI-11); bump deliberately. + orb-tools: circleci/orb-tools@12.5.0 filters: &filters tags: @@ -13,6 +14,31 @@ tag-only: &tag-only tags: only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ +jobs: + test: + docker: + - image: cimg/node:22.12 + resource_class: medium + steps: + - checkout + - run: + name: Install test tools + command: | + sudo apt-get update -qq + sudo apt-get install -y -qq gettext-base shellcheck >/dev/null + YQ_VERSION=v4.44.6 + sudo curl -fsSL -o /usr/local/bin/yq \ + "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" + sudo chmod +x /usr/local/bin/yq + git clone -q --depth 1 --branch v1.11.1 https://github.com/bats-core/bats-core.git /tmp/bats-core + sudo /tmp/bats-core/install.sh /usr/local + - run: + name: Script and template tests + command: make test-scripts test-templates + - run: + name: BATS tests + command: bats tests + workflows: lint-pack: jobs: @@ -23,6 +49,8 @@ workflows: - orb-tools/review: filters: *filters exclude: RC006,RC007,RC010 # RC006/RC007: private repo URL unreachable; RC010: machine executors + - test: + filters: *filters test-deploy: jobs: @@ -38,11 +66,15 @@ workflows: - orb-publishing filters: *tag-only + # develop owns the shared dev:snapshot tag. Feature branches publish only an + # immutable dev: tag so open branches cannot overwrite each other's + # snapshot (issue #26). snapshot: jobs: - orb-tools/pack: filters: *filters - orb-tools/publish: + name: publish-develop-snapshot orb_name: kof22/munitor vcs_type: << pipeline.project.type >> pub_type: dev @@ -55,4 +87,17 @@ workflows: branches: only: - develop + - orb-tools/publish: + name: publish-feature-sha + orb_name: kof22/munitor + vcs_type: << pipeline.project.type >> + pub_type: dev + dev_tags: "dev:${CIRCLE_SHA1}" + requires: + - orb-tools/pack + context: + - orb-publishing + filters: + branches: + only: - /feature\/.*/ diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ce5a448 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# KofTwentyTwo .editorconfig (standards K22-REPO-03, configs/editorconfig). +# Exception: the existing Bash scripts and CircleCI templates use 2-space +# indentation; K22-CODE-SH-02 specifies 3. K22 exception EX-0011. +# Do not mix widths in a file. +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false + +[{Makefile,makefile,*.mk}] +indent_style = tab + +# Shell: shfmt reads these keys (K22-CODE-SH-02), apart from the indent width above. +[*.{sh,bash,bats}] +binary_next_line = true +switch_case_indent = true +space_redirects = true +function_next_line = true diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..1cf56cb --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# Every path has an owner (K22-REPO-08). Add narrower rules below for areas that +# need a specific reviewer once the project has more than one maintainer. +* @KofTwentyTwo diff --git a/CLAUDE.md b/CLAUDE.md index a7c5a6f..fca6bab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ CircleCI orb that generates CI/CD pipeline configurations from a `.munitor.yml` ## Key Commands ```bash -make test # Run all tests (BATS) +make test-scripts # Script unit tests (bats tests for the .bats suites) make pack # Pack orb YAML from src/ make all # lint + pack + test make validate # Validate packed orb with CircleCI CLI diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8152d4d..57cd176 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,14 @@ # Contributing to Munitor +## Workflow + +- **Branches.** Branch `feature/` or `hotfix/` from `develop` and open a pull request back to `develop`. Releases go through `release/X.Y.Z` into `main`; pushing the `vX.Y.Z` tag publishes `kof22/munitor@X.Y.Z`, and every push to `develop` publishes `dev:snapshot`, and every push to `feature/*` publishes only the immutable `dev:` tag. Back-merge `main` into `develop` after a release. +- **Commits.** Conventional Commits (`feat(#N): ...`, `fix: ...`, `docs: ...`), signed (`git commit -S`), with a Developer Certificate of Origin sign-off line (`git commit -s`, which adds `Signed-off-by: Name `). Reference the GitHub issue (`Closes #N`) where there is one. +- **Checks.** Run `make all` (yamllint, shellcheck, pack plus `circleci orb validate`, script and template tests) and `bats tests` (the `.bats` suites) before pushing. CircleCI runs the orb-tools lint, pack and review jobs plus a `test` job (`make test-scripts test-templates` and `bats tests`). `circleci orb validate` still runs only locally. +- **Formatting.** Shell scripts and templates use 2-space indentation (see `.editorconfig`); this deviates from K22-CODE-SH-02 under standards exception EX-0011. +- **Security.** Report vulnerabilities privately as described in [SECURITY.md](SECURITY.md), never in a public issue. +- **AI assistance.** Disclose AI-assisted changes in the pull request description. + ## Module Inventory Munitor organizes CI/CD pipelines into **modules** (called "pipeline types" in `.munitor.yml`). Each module is a self-contained set of templates, jobs, commands, and scripts that handle a specific application stack. diff --git a/README.md b/README.md index 46fcc76..ff46867 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,14 @@ Latin: *munitor* (fortifier, protector) -- KofTwentyTwo's unified CircleCI orb for building, testing, and deploying all repositories. Pipeline changes happen in one place instead of 80+ repos. +| | | +| --- | --- | +| **Tier** | Internal until classified -- listed under *Awaiting classification* in the KofTwentyTwo standards [MAINTAINERS.md](https://github.com/KofTwentyTwo/standards/blob/main/MAINTAINERS.md); Product-tier requirements are being adopted | +| **Standards** | Targets [KofTwentyTwo standards](https://github.com/KofTwentyTwo/standards) 1.0.0 | +| **Orb** | [`kof22/munitor`](https://circleci.com/developer/orbs/orb/kof22/munitor), latest stable `0.3.3` | +| **Security** | [SECURITY.md](SECURITY.md) | +| **Contributing** | [CONTRIBUTING.md](CONTRIBUTING.md) | + ## How It Works Munitor uses CircleCI's [dynamic configuration](https://circleci.com/docs/dynamic-config/) to generate your full CI/CD pipeline at runtime. Each repo has a minimal `.circleci/config.yml` that triggers Munitor, and a `.munitor.yml` that declares what your project needs. Munitor reads your config, selects the right template, and generates a complete CircleCI workflow. @@ -58,6 +66,93 @@ Munitor validates branch names and will fail on non-conforming names. | `sonarcloud` | `SONAR_TOKEN` | SonarCloud analysis | | `nvd` | `NVD_API_KEY` | OWASP dependency check (Java only, optional) | +## Adopting in another app + +Munitor is a public CircleCI orb: any CircleCI organization can use it without forking. This checklist covers what a new consumer needs, for example a Java/Maven web service. A QQQ application should also read the [QQQ adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/docs/migration/adoption-checklist.adoc) (section 7 covers release tooling). + +**Version.** Use `kof22/munitor@0.3.3` (or `@0.3` to float on the 0.3 line) in `.circleci/config.yml`, and the same value as `orb_version` in `.munitor.yml`. Keep the setup orb and the generated pipeline on the same line. + +### 1. Repository files + +- `.circleci/config.yml`: the setup config from [Quick Start](#quick-start), pinned to `kof22/munitor@0.3.3`. +- `.munitor.yml`: pick a [pipeline type](#pipeline-types). A Java service is `java-webapp`. KofTwentyTwo's Website-Backend is a working example; a generalized version: + +```yaml +pipeline: java-webapp +orb_version: "0.3.3" +image_name: my-service +java_version: "21" +docker: + registry: ghcr.io/my-org # the org default is ghcr.io/KofTwentyTwo -- always set this +cd: + repo: my-org/my-gitops-repo # omit to skip GitOps write-back + format: kustomize # default is helm (image.tag in values.yaml) + path: apps/my-service/kustomization.yaml # optional explicit file in a consolidated repo + production_only: true # only main updates the GitOps repo +supplemental_images: # optional extra images built from the same commit and version + - name: migrations + dockerfile: docker/Dockerfile.migrations + cd: + image_name: ghcr.io/my-org/my-service-migrations +health: + path: /health + port: "8000" + db: "true" # start a PostgreSQL sidecar for the health gate + migrations: migrations # run the -migrations image against it first + smoke_paths: [/health] +sonar: + project_key: my-org_my-service # omit to skip SonarCloud +contexts: + registry: ghcr + github: github + sonar: sonarcloud # required only with sonar.project_key + nvd: nvd # optional +org: # organization defaults (see Known limitations) + ci_email: ci@example.com + ci_name: My CI +``` + +- A health endpoint that returns HTTP 200 within 30 seconds of container start (see [Health Check Requirement](#health-check-requirement)). +- A Dockerfile only for `node-webapp` (`docker.use_repo_dockerfile: true`) or for `supplemental_images`; `java-webapp` and `node-api` generate theirs. + +### 2. CircleCI project settings + +See [Prerequisites](#prerequisites): enable dynamic config (setup workflows) and allow uncertified orbs in the consuming organization. + +### 3. Contexts and secrets + +CircleCI organization contexts; the context names are set through `contexts.*`. + +| Context | Variables | Needed for | +| --- | --- | --- | +| registry (e.g. `ghcr`) | `GHCR_USERNAME`, `GHCR_TOKEN` (`write:packages`) | Pushing images and inspecting existing tags; the immutable-tag guard fails closed without registry access | +| github | `GITHUB_TOKEN` with contents write on the app repository and on the GitOps repository | Reserving `vX.Y.Z` tags on `main` (required by `production`), GitHub Releases, GitOps write-back | +| sonar (optional) | `SONAR_TOKEN`; SonarCloud organization from `SONAR_ORG` (default `KofTwentyTwo`) | SonarCloud analysis | +| nvd (optional) | `NVD_API_KEY` | OWASP dependency check; `owasp: false` disables the check | + +### 4. Branch model + +Branch names are enforced: `main`, `develop`, `staging`, `feature/*`, `release/*`, `hotfix/*`. A merge to `main` is a production release: it reserves the next `vX.Y.Z` tag, pushes the image, updates the GitOps repository and creates the GitHub Release. Protect `main` accordingly. + +### 5. Deployment infrastructure (outside Munitor) + +Munitor stops at the GitOps commit. The consumer provides: + +- a GitOps repository whose `kustomization.yaml` (an `images[]` entry named `/`) or Helm `values.yaml` (`image.tag`) Munitor edits; +- Argo CD (or another controller) syncing that repository, with pull access to the registry; +- the runtime database and its backups. KofTwentyTwo's website uses a CloudNativePG cluster with an object-store backup and a `ScheduledBackup`, declared in `KofTwentyTwo/cluster-gitops` (`tenants/kof22/website`). Munitor only starts a throwaway PostgreSQL sidecar for the CI health gate and never touches a real database; +- schema migrations. Either the application migrates at start-up, or the deployment runs the `migrations` supplemental image (the KofTwentyTwo website runs a pre-migration backup image and then the migrations image as init containers). `health.migrations` only exercises that image in CI. + +### 6. Known limitations + +- Organization defaults point at KofTwentyTwo: registry `ghcr.io/KofTwentyTwo`, npm scope `@koftwentytwo`, CI email `munitor-ci@koftwentytwo.com`, SonarCloud organization `KofTwentyTwo`. Set `docker.registry`, the `org.*` keys and `SONAR_ORG` explicitly. The `org` block is read, but the unknown-key check prints a harmless `Unknown key 'org'` warning. +- `production` builds on `main` skip the quality gates, which run on `release/*` and `hotfix/*` pull requests. Do not merge unreviewed branches straight to `main`. +- CircleCI only; there is no GitHub Actions equivalent. Docker jobs use machine executors. + +### 7. Upgrade notes + +`0.3.3` reserves release tags atomically on `main` (needs the github context on `build-and-test`, which generated `production` workflows include), refuses to overwrite a version image tag built from another commit, and checks tag ownership before the GitOps bump. A consumer moving from a Munitor-derived orb with a tag-driven production trigger should compare that model with Munitor's merge-to-`main` model before switching. + ## Pipeline Types ### `node-api` @@ -425,6 +520,7 @@ Production versions on `main` are unique per commit, even when merges land minut - **Atomic reservation** -- `build-and-test` on `main` reserves `vX.Y.Z` by creating the git tag through the GitHub API before the version is baked into artifacts. Tag creation is create-if-absent, so concurrent pipelines serialize on it: if another commit already owns the tag, the build takes the next patch version. A rerun of the same commit reuses its tag. Existing tags are never moved. Requires the GitHub context on `build-and-test` (present in the generated `production` workflows). - **Immutable image tags** -- images carry an `org.opencontainers.image.revision` label. `docker-build-push` refuses to push a version tag (main or supplemental) that already exists for a different commit, and fails closed if the registry cannot be inspected. Environment tags such as `develop` stay mutable. - **Ownership-checked GitOps bump** -- `update-cd-repo` verifies that `vX.Y.Z` points at the pipeline's commit before writing the version to the CD repository. +- **Signed GitOps commits** -- `update-cd-repo` creates the CD repository commit through the GitHub GraphQL API (`createCommitOnBranch`) instead of `git push`. GitHub signs API-created commits, so they show as Verified and pass a CD repository ruleset that requires signed commits, with no bypass actor. The commit is authored by the identity behind `GITHUB_TOKEN`; the source repository and commit are recorded in the commit body. Concurrent bumps retry against the new branch head (`expectedHeadOid`). The `ci_git_email`/`ci_git_name` parameters and `org.ci_email`/`org.ci_name` no longer affect these commits. - **Release targets the built commit** -- `github-release` targets `CIRCLE_SHA1`, not the moving `main` head, and refuses if the tag points elsewhere. `cd_path` is rendered into generated pipelines only when `cd.path` is set, so manifests that pin an older `orb_version` keep validating. @@ -475,7 +571,7 @@ Tags follow the format `vX.Y.Z` (git tag) and `X.Y.Z` (Docker tag). Tags are imm | Field | Type | Default | Pipelines | Description | |-------|------|---------|-----------|-------------| | `pipeline` | string | *required* | all | Pipeline type: `node-api`, `node-webapp`, `java-webapp`, `terraform`, `sdk-distribution`, `validate-cd-repo`, `argocd-apps` | -| `orb_version` | string | *required* | all | Orb version for the generated pipeline (e.g., `1`, `dev:snapshot`) | +| `orb_version` | string | *required* | all | Orb version for the generated pipeline (e.g., `"0.3.3"`, `"0.3"`, `dev:snapshot`) | | `image_name` | string | *required** | node-api, node-webapp, java-webapp | Docker image name (without registry prefix) | | `node_version` | string | `20` | node-api, node-webapp | Node.js major version | | `java_version` | string | `21` | java-webapp | Java major version | @@ -486,6 +582,17 @@ Tags follow the format `vX.Y.Z` (git tag) and `X.Y.Z` (Docker tag). Tags are imm | `cd.path` | string | -- | node-api, node-webapp, java-webapp | Explicit repository-relative `kustomization.yaml` path for consolidated GitOps repositories | | `cd.production_only` | bool | `false` | node-api, node-webapp, java-webapp | Disable CD write-back from develop, staging, and release workflows | | `cd.env.release` | string | `staging` | node-api, node-webapp, java-webapp | CD target environment for release branches | +| `cd.env.develop` / `cd.env.staging` / `cd.env.prod` | string | `develop` / `staging` / `prod` | node-api, node-webapp, java-webapp | CD target environment names for the develop, staging and main workflows | +| `cd.format` | string | `helm` | node-api, node-webapp, java-webapp | `helm` (sets `image.tag` in `values.yaml`, per environment under `environments//` when present) or `kustomize` (sets `images[].newTag`) | +| `server_module` | string | -- | gradle-webapp | Gradle module whose `build/libs` JAR goes into the generated image (multi-module builds) | +| `owasp` | bool | `true` | java-webapp | Run the OWASP dependency check | +| `package_manager` | string | `npm` | node-api, node-webapp | `npm` or `pnpm` | +| `supplemental_images` | list | `[]` | node-api, node-webapp, java-webapp | Extra images (`name`, `dockerfile`, `cd.image_name`) built and tagged with the same version | +| `health.db` | bool | `false` | node-api, node-webapp, java-webapp | Start a PostgreSQL sidecar for the health check | +| `health.migrations` | string | -- | node-api, node-webapp, java-webapp | Supplemental image suffix run against the sidecar before the health check | +| `health.smoke_paths` | list | `[]` | node-api, node-webapp, java-webapp | Extra paths that must return HTTP 200 during the health check | +| `test.coverage.summary_path` | string | `coverage/coverage-summary.json` | node-api, node-webapp | Coverage summary file read by the coverage gate | +| `org.docker_registry` / `org.npm_scope` / `org.ci_email` / `org.ci_name` / `org.orb_slug` | string | KofTwentyTwo values | all | Organization defaults; see [Adopting in another app](#adopting-in-another-app) | | `coverage.min_instruction` | int | `70` | node-api, node-webapp, java-webapp | Minimum coverage percentage | | `e2e` | bool | `false` | node-api, node-webapp, java-webapp | Enable Playwright E2E tests | | `sbom` | bool | `false` | node-api, node-webapp, java-webapp | Enable SBOM generation | @@ -596,3 +703,15 @@ excepted version. Audit errors, unknown chains and expired exceptions fail. A ne upstream package release also fails and requires review. Trivy remains unchanged. This mechanism requires npm audit report version 2 and a package-path lockfile; unsupported reports fail. BATS and Node are required for the audit regression tests. + +## Security + +Report vulnerabilities privately; see [SECURITY.md](SECURITY.md). + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md). + +## License + +[GPL-3.0](LICENSE) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..c622d02 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,56 @@ +# Security Policy + +## Reporting a vulnerability + +**Do not open a public issue for a security problem.** + +Report it privately through GitHub's +[private vulnerability reporting](https://github.com/KofTwentyTwo/Munitor/security/advisories/new) +(the repository's **Security** tab, then **Report a vulnerability**). If that form is +unavailable, contact the maintainer privately through GitHub before disclosing anything. + +Security contact: **James Maes** ([@KofTwentyTwo](https://github.com/KofTwentyTwo)), +maintainer. + +Please include the affected orb version, the `.munitor.yml` pipeline type, how to +reproduce it, and the impact you see. Never include secrets or CircleCI context values. + +## What to expect + +KofTwentyTwo follows coordinated vulnerability disclosure. Timeframes are counted +from the day the report arrives and come from the +[KofTwentyTwo security program](https://github.com/KofTwentyTwo/standards/blob/main/policies/security-program.md): + +| Step | Target | +| --- | --- | +| Acknowledge the report | 7 days | +| Assess it: confirm or reject, with a severity | 14 days | +| Release a fix for a CRITICAL vulnerability | 7 days | +| Release a fix for a HIGH vulnerability | 30 days | +| Release a fix for a MEDIUM vulnerability | 90 days | +| Release a fix for a LOW vulnerability | The next release | +| Publish the advisory (GitHub Security Advisory, CVE where applicable) | When the fix ships, or 90 days after the report, whichever comes first | + +Reporters are credited in the advisory unless they ask not to be. + +## Supported versions + +| Version | Security fixes | +| --- | --- | +| 0.3.x (`kof22/munitor@0.3`) | Yes | +| Older | No; upgrade to the latest release | + +Only the latest minor line receives fixes; a line ends when the next minor version is +published to the CircleCI orb registry. + +## Scope notes + +Munitor runs inside the consumer's CircleCI project with the consumer's contexts +(registry, GitHub, SonarCloud, NVD). Issues that let a generated pipeline leak context +secrets, push an image or GitOps change for the wrong commit, or bypass a security +gate are in scope. + +## Published vulnerabilities + +Fixed vulnerabilities are published as +[GitHub Security Advisories](https://github.com/KofTwentyTwo/Munitor/security/advisories). diff --git a/src/commands/update_cd_repo.yml b/src/commands/update_cd_repo.yml index 8a7c72c..2e9937b 100644 --- a/src/commands/update_cd_repo.yml +++ b/src/commands/update_cd_repo.yml @@ -39,11 +39,14 @@ parameters: ci_git_email: type: string default: "" - description: Git author email for CD repo commits. + description: > + Deprecated and ignored. GitOps commits are created through the GitHub API, + signed by GitHub and authored by the GITHUB_TOKEN identity. ci_git_name: type: string default: "" - description: Git author name for CD repo commits. + description: > + Deprecated and ignored. See ci_git_email. supplemental_images: type: string default: "[]" diff --git a/src/jobs/update_cd_repo.yml b/src/jobs/update_cd_repo.yml index 38c33a3..3a3e7a3 100644 --- a/src/jobs/update_cd_repo.yml +++ b/src/jobs/update_cd_repo.yml @@ -31,11 +31,14 @@ parameters: ci_git_email: type: string default: "" - description: Git author email for CD repo commits. + description: > + Deprecated and ignored. GitOps commits are created through the GitHub API, + signed by GitHub and authored by the GITHUB_TOKEN identity. ci_git_name: type: string default: "" - description: Git author name for CD repo commits. + description: > + Deprecated and ignored. See ci_git_email. supplemental_images: type: string default: "[]" diff --git a/src/scripts/generate_config.sh b/src/scripts/generate_config.sh index 21100df..07e0b65 100755 --- a/src/scripts/generate_config.sh +++ b/src/scripts/generate_config.sh @@ -97,7 +97,7 @@ validate_required_fields "${PIPELINE_TYPE}" "${CONFIG_FILE}" # -------------------------------------------------------------------------- # Warn about unknown top-level keys (catches typos like java_verion) # -------------------------------------------------------------------------- -KNOWN_KEYS="pipeline orb_version image_name server_module java_version node_version sonar docker cd coverage e2e sbom owasp contexts npm node services test terraform sast health kustomize kube_linter_config package_manager yamllint_paths supplemental_images" +KNOWN_KEYS="pipeline orb_version org image_name server_module java_version node_version sonar docker cd coverage e2e sbom owasp contexts npm node services test terraform sast health kustomize kube_linter_config package_manager yamllint_paths supplemental_images" ACTUAL_KEYS=$(yq 'keys | .[]' "${CONFIG_FILE}" 2>/dev/null || true) for key in ${ACTUAL_KEYS}; do if ! echo "${KNOWN_KEYS}" | grep -qw "${key}"; then diff --git a/src/scripts/packed_generate_config.sh b/src/scripts/packed_generate_config.sh index d4a83f3..1bdee91 100755 --- a/src/scripts/packed_generate_config.sh +++ b/src/scripts/packed_generate_config.sh @@ -3249,7 +3249,7 @@ validate_required_fields "${PIPELINE_TYPE}" "${CONFIG_FILE}" # -------------------------------------------------------------------------- # Warn about unknown top-level keys (catches typos like java_verion) # -------------------------------------------------------------------------- -KNOWN_KEYS="pipeline orb_version image_name server_module java_version node_version sonar docker cd coverage e2e sbom owasp contexts npm node services test terraform sast health kustomize kube_linter_config package_manager yamllint_paths supplemental_images" +KNOWN_KEYS="pipeline orb_version org image_name server_module java_version node_version sonar docker cd coverage e2e sbom owasp contexts npm node services test terraform sast health kustomize kube_linter_config package_manager yamllint_paths supplemental_images" ACTUAL_KEYS=$(yq 'keys | .[]' "${CONFIG_FILE}" 2>/dev/null || true) for key in ${ACTUAL_KEYS}; do if ! echo "${KNOWN_KEYS}" | grep -qw "${key}"; then diff --git a/src/scripts/update_cd_repo.sh b/src/scripts/update_cd_repo.sh index a7e68ec..2718814 100755 --- a/src/scripts/update_cd_repo.sh +++ b/src/scripts/update_cd_repo.sh @@ -68,128 +68,159 @@ git clone "https://x-access-token:${GITHUB_TOKEN}@github.com/${REPO}.git" "${CLO cd "${CLONE_DIR}" -if [[ "${FORMAT}" == "kustomize" ]]; then - # --- Kustomize mode --- - if [[ -z "${IMAGE_NAME}" ]]; then - echo "ERROR: cd_image_name is required for Kustomize repos." - exit 1 - fi - - # Explicit paths support consolidated GitOps repositories. Existing dedicated - # repositories retain the overlays/ convention by default. - if [[ -n "${CONFIGURED_PATH}" ]]; then - if [[ "${CONFIGURED_PATH}" == /* || "${CONFIGURED_PATH}" == *".."* || "${CONFIGURED_PATH}" != *kustomization.yaml ]]; then - echo "ERROR: cd.path must be a relative path to a kustomization.yaml file without '..'." +apply_updates() { + if [[ "${FORMAT}" == "kustomize" ]]; then + # --- Kustomize mode --- + if [[ -z "${IMAGE_NAME}" ]]; then + echo "ERROR: cd_image_name is required for Kustomize repos." exit 1 fi - KUST_FILE="${CONFIGURED_PATH}" - elif [[ -n "${ENVIRONMENT}" ]]; then - KUST_FILE="overlays/${ENVIRONMENT}/kustomization.yaml" + + # Explicit paths support consolidated GitOps repositories. Existing dedicated + # repositories retain the overlays/ convention by default. + if [[ -n "${CONFIGURED_PATH}" ]]; then + if [[ "${CONFIGURED_PATH}" == /* || "${CONFIGURED_PATH}" == *".."* || "${CONFIGURED_PATH}" != *kustomization.yaml ]]; then + echo "ERROR: cd.path must be a relative path to a kustomization.yaml file without '..'." + exit 1 + fi + KUST_FILE="${CONFIGURED_PATH}" + elif [[ -n "${ENVIRONMENT}" ]]; then + KUST_FILE="overlays/${ENVIRONMENT}/kustomization.yaml" + else + KUST_FILE="kustomization.yaml" + fi + + if [[ ! -f "${KUST_FILE}" ]]; then + echo "ERROR: Kustomize file '${KUST_FILE}' not found in CD repo." + if [[ -z "${CONFIGURED_PATH}" ]]; then + echo "Available overlays:" + ls overlays/ 2>/dev/null || echo " (no overlays directory)" + fi + exit 1 + fi + + echo "Updating ${KUST_FILE}: ${IMAGE_NAME} -> ${VERSION}" + if ! yq -e ".images[] | select(.name == \"${IMAGE_NAME}\")" "${KUST_FILE}" >/dev/null; then + echo "ERROR: Image '${IMAGE_NAME}' is not declared in ${KUST_FILE}." + exit 1 + fi + yq -i "(.images[] | select(.name == \"${IMAGE_NAME}\")).newTag = \"${VERSION}\"" "${KUST_FILE}" + yq -i "del((.images[] | select(.name == \"${IMAGE_NAME}\")).digest)" "${KUST_FILE}" + git add "${KUST_FILE}" + else - KUST_FILE="kustomization.yaml" - fi + # --- Helm mode (default) --- + echo "Updating ${KEY} -> ${VERSION}" - if [[ ! -f "${KUST_FILE}" ]]; then - echo "ERROR: Kustomize file '${KUST_FILE}' not found in CD repo." - if [[ -z "${CONFIGURED_PATH}" ]]; then - echo "Available overlays:" - ls overlays/ 2>/dev/null || echo " (no overlays directory)" - fi - exit 1 - fi - - echo "Updating ${KUST_FILE}: ${IMAGE_NAME} -> ${VERSION}" - if ! yq -e ".images[] | select(.name == \"${IMAGE_NAME}\")" "${KUST_FILE}" >/dev/null; then - echo "ERROR: Image '${IMAGE_NAME}' is not declared in ${KUST_FILE}." - exit 1 - fi - yq -i "(.images[] | select(.name == \"${IMAGE_NAME}\")).newTag = \"${VERSION}\"" "${KUST_FILE}" - yq -i "del((.images[] | select(.name == \"${IMAGE_NAME}\")).digest)" "${KUST_FILE}" - git add "${KUST_FILE}" - -else - # --- Helm mode (default) --- - echo "Updating ${KEY} -> ${VERSION}" - - # Resolve environment-specific values path when CD_ENVIRONMENT is set - if [[ -n "${ENVIRONMENT}" ]]; then - ENV_VALUES="environments/${ENVIRONMENT}/${VALUES}" - if [[ -f "${ENV_VALUES}" ]]; then - VALUES="${ENV_VALUES}" - echo "Using environment path: ${VALUES}" - else - echo "Environment path '${ENV_VALUES}' not found, falling back to root '${VALUES}'" - fi - fi - - if [[ ! -f "${VALUES}" ]]; then - echo "ERROR: Values file '${VALUES}' not found in CD repo." - exit 1 - fi - - yq -i ".${KEY} = \"${VERSION}\"" "${VALUES}" - git add "${VALUES}" -fi - -# Update supplemental image tags (if any) -SUPP_JSON="${SUPPLEMENTAL_IMAGES_JSON:-[]}" -if [[ "${SUPP_JSON}" != "[]" && "${SUPP_JSON}" != "null" ]]; then - SUPP_COUNT=$(echo "${SUPP_JSON}" | jq 'length') - echo "Updating ${SUPP_COUNT} supplemental image tag(s)..." - - for ((i = 0; i < SUPP_COUNT; i++)); do - SUPP_NAME=$(echo "${SUPP_JSON}" | jq -r ".[$i].name") - - if [[ "${FORMAT}" == "kustomize" ]]; then - SUPP_CD_IMAGE=$(echo "${SUPP_JSON}" | jq -r ".[$i].cd.image_name // \"\"") - if [[ -z "${SUPP_CD_IMAGE}" ]]; then - SUPP_CD_IMAGE="${IMAGE_NAME}-${SUPP_NAME}" - fi - echo " Kustomize: ${SUPP_CD_IMAGE} -> ${VERSION}" - # Upsert: add entry if it doesn't exist, then update - EXISTING=$(yq ".images[] | select(.name == \"${SUPP_CD_IMAGE}\") | .name" "${KUST_FILE}") - if [[ -z "${EXISTING}" ]]; then - echo " (creating new image entry)" - yq -i ".images += [{\"name\": \"${SUPP_CD_IMAGE}\", \"newTag\": \"${VERSION}\"}]" "${KUST_FILE}" + # Resolve environment-specific values path when CD_ENVIRONMENT is set + if [[ -n "${ENVIRONMENT}" ]]; then + ENV_VALUES="environments/${ENVIRONMENT}/${VALUES}" + if [[ -f "${ENV_VALUES}" ]]; then + VALUES="${ENV_VALUES}" + echo "Using environment path: ${VALUES}" else - yq -i "(.images[] | select(.name == \"${SUPP_CD_IMAGE}\")).newTag = \"${VERSION}\"" "${KUST_FILE}" - yq -i "del((.images[] | select(.name == \"${SUPP_CD_IMAGE}\")).digest)" "${KUST_FILE}" - fi - git add "${KUST_FILE}" - else - SUPP_IMAGE_KEY=$(echo "${SUPP_JSON}" | jq -r ".[$i].cd.image_key // \"\"") - if [[ -n "${SUPP_IMAGE_KEY}" ]]; then - echo " Helm: ${SUPP_IMAGE_KEY} -> ${VERSION}" - yq -i ".${SUPP_IMAGE_KEY} = \"${VERSION}\"" "${VALUES}" - git add "${VALUES}" + echo "Environment path '${ENV_VALUES}' not found, falling back to root '${VALUES}'" fi fi - done -fi -# Commit and push -git config user.email "${CI_GIT_EMAIL:-${MUNITOR_CI_EMAIL:-munitor-ci@koftwentytwo.com}}" -git config user.name "${CI_GIT_NAME:-${MUNITOR_CI_NAME:-Munitor CI}}" + if [[ ! -f "${VALUES}" ]]; then + echo "ERROR: Values file '${VALUES}' not found in CD repo." + exit 1 + fi -if git diff --cached --quiet; then - echo "No changes to commit (version already up to date)." - exit 0 -fi + yq -i ".${KEY} = \"${VERSION}\"" "${VALUES}" + git add "${VALUES}" + fi -git commit -m "chore: update ${IMAGE_NAME:-image} to ${VERSION}" -m "Source: ${CIRCLE_PROJECT_REPONAME:-unknown}@${CIRCLE_SHA1:-unknown}" + # Update supplemental image tags (if any) + SUPP_JSON="${SUPPLEMENTAL_IMAGES_JSON:-[]}" + if [[ "${SUPP_JSON}" != "[]" && "${SUPP_JSON}" != "null" ]]; then + SUPP_COUNT=$(echo "${SUPP_JSON}" | jq 'length') + echo "Updating ${SUPP_COUNT} supplemental image tag(s)..." + + for ((i = 0; i < SUPP_COUNT; i++)); do + SUPP_NAME=$(echo "${SUPP_JSON}" | jq -r ".[$i].name") + + if [[ "${FORMAT}" == "kustomize" ]]; then + SUPP_CD_IMAGE=$(echo "${SUPP_JSON}" | jq -r ".[$i].cd.image_name // \"\"") + if [[ -z "${SUPP_CD_IMAGE}" ]]; then + SUPP_CD_IMAGE="${IMAGE_NAME}-${SUPP_NAME}" + fi + echo " Kustomize: ${SUPP_CD_IMAGE} -> ${VERSION}" + # Upsert: add entry if it doesn't exist, then update + EXISTING=$(yq ".images[] | select(.name == \"${SUPP_CD_IMAGE}\") | .name" "${KUST_FILE}") + if [[ -z "${EXISTING}" ]]; then + echo " (creating new image entry)" + yq -i ".images += [{\"name\": \"${SUPP_CD_IMAGE}\", \"newTag\": \"${VERSION}\"}]" "${KUST_FILE}" + else + yq -i "(.images[] | select(.name == \"${SUPP_CD_IMAGE}\")).newTag = \"${VERSION}\"" "${KUST_FILE}" + yq -i "del((.images[] | select(.name == \"${SUPP_CD_IMAGE}\")).digest)" "${KUST_FILE}" + fi + git add "${KUST_FILE}" + else + SUPP_IMAGE_KEY=$(echo "${SUPP_JSON}" | jq -r ".[$i].cd.image_key // \"\"") + if [[ -n "${SUPP_IMAGE_KEY}" ]]; then + echo " Helm: ${SUPP_IMAGE_KEY} -> ${VERSION}" + yq -i ".${SUPP_IMAGE_KEY} = \"${VERSION}\"" "${VALUES}" + git add "${VALUES}" + fi + fi + done + fi +} + +# Commit through the GitHub GraphQL API (createCommitOnBranch) rather than pushing local commits. +# GitHub signs API-created commits, so they show as Verified and satisfy a +# "require signed commits" ruleset on the GitOps branch without a bypass actor. +# The commit author is the GITHUB_TOKEN identity; the source build is recorded +# in the commit body. expectedHeadOid makes concurrent updates fail cleanly and +# retry against the new head instead of overwriting each other. +BRANCH="$(git branch --show-current)" +HEADLINE="chore: update ${IMAGE_NAME:-image} to ${VERSION}" +MESSAGE_BODY="Source: ${CIRCLE_PROJECT_REPONAME:-unknown}@${CIRCLE_SHA1:-unknown}" + +commit_via_api() { + local head_oid additions payload response file + head_oid="$(git rev-parse HEAD)" + additions='[]' + while IFS= read -r file; do + base64 -w0 "${file}" > /tmp/cd-file.b64 + additions="$(jq --arg path "${file}" --rawfile contents /tmp/cd-file.b64 '. + [{path: $path, contents: $contents}]' <<< "${additions}")" + done < <(git diff --cached --name-only) + payload="$(jq -n --arg repo "${REPO}" --arg branch "${BRANCH}" --arg oid "${head_oid}" --arg headline "${HEADLINE}" --arg body "${MESSAGE_BODY}" --argjson additions "${additions}" '{query: "mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { oid url } } }", + variables: {input: {branch: {repositoryNameWithOwner: $repo, branchName: $branch}, + expectedHeadOid: $oid, message: {headline: $headline, body: $body}, + fileChanges: {additions: $additions}}}}')" + if ! response="$(curl -sS -X POST -H "Authorization: bearer ${GITHUB_TOKEN}" -H "Content-Type: application/json" https://api.github.com/graphql --data @- <<< "${payload}")"; then + echo "GraphQL request failed." + return 1 + fi + if [[ "$(jq -r '.data.createCommitOnBranch.commit.oid // empty' <<< "${response}")" == "" ]]; then + echo "GraphQL commit rejected: $(jq -c '.errors // .' <<< "${response}")" + return 1 + fi + echo "Signed commit: $(jq -r '.data.createCommitOnBranch.commit.url' <<< "${response}")" +} -# Independent pipelines may update a shared GitOps repository concurrently. for attempt in 1 2 3; do - if git push origin HEAD; then + apply_updates + + if git diff --cached --quiet; then + echo "No changes to commit (version already up to date)." + exit 0 + fi + + if commit_via_api; then echo "CD repo updated successfully." exit 0 fi if [[ "${attempt}" == "3" ]]; then - echo "ERROR: Could not push the GitOps update after ${attempt} attempts." + echo "ERROR: Could not commit the GitOps update after ${attempt} attempts." exit 1 fi - echo "Push raced with another GitOps update; rebasing before retry ${attempt}/3." - git pull --rebase origin "$(git branch --show-current)" + echo "Commit raced with another GitOps update; refreshing before retry ${attempt}/3." + git fetch -q origin "${BRANCH}" + git reset -q --hard "origin/${BRANCH}" done diff --git a/tests/test_scripts.sh b/tests/test_scripts.sh index 3ce611c..6db9f13 100755 --- a/tests/test_scripts.sh +++ b/tests/test_scripts.sh @@ -328,12 +328,21 @@ else fail "should support and validate an explicit Kustomize path" fi -echo -n " TEST: retries concurrent shared-repository pushes... " +echo -n " TEST: retries concurrent shared-repository commits against the new head... " if grep -q 'for attempt in 1 2 3' "${SRC_SCRIPTS}/update_cd_repo.sh" && - grep -q 'git pull --rebase' "${SRC_SCRIPTS}/update_cd_repo.sh"; then + grep -q 'expectedHeadOid' "${SRC_SCRIPTS}/update_cd_repo.sh" && + grep -q 'git reset -q --hard "origin/${BRANCH}"' "${SRC_SCRIPTS}/update_cd_repo.sh"; then pass else - fail "should rebase and retry shared-repository push races" + fail "should retry commit races against the refreshed branch head" +fi + +echo -n " TEST: commits through the signed GraphQL API, never git push... " +if grep -q 'createCommitOnBranch' "${SRC_SCRIPTS}/update_cd_repo.sh" && + ! grep -q 'git push' "${SRC_SCRIPTS}/update_cd_repo.sh"; then + pass +else + fail "should create GitOps commits with createCommitOnBranch so GitHub signs them" fi echo -n " TEST: deletes digest field when setting newTag (kustomize)... "