From c8fd974bfdcf03a7d83885c3cbef4c3e61ad416c Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 19 Oct 2026 10:22:29 -0500 Subject: [PATCH 1/3] perf(ci): bound Maven caches and remove duplicate setup Keep restore and save keys stable across release version rewrites, avoid inherited cross-version caches, skip unused OWASP setup, upload test results once and pack the orb once before gated publication. Co-Authored-By: Codex Signed-off-by: James Maes --- .circleci/config.yml | 22 ++--- Makefile | 2 +- docs/PLAN-lean-java-ci.md | 41 ++++++++++ docs/SESSION-STATE.md | 14 ++++ docs/TODO.md | 2 + src/commands/restore_maven_cache.yml | 11 ++- src/commands/save_maven_cache.yml | 9 ++- src/jobs/mvn_build_and_test.yml | 24 +++++- src/jobs/security_scan.yml | 6 +- src/scripts/maven_cache_key.sh | 19 +++++ src/scripts/maven_workspace.sh | 46 +++++++++++ src/scripts/packed_generate_config.sh | 77 +++--------------- src/scripts/templates/java-webapp.yml.tpl | 51 ++---------- .../partials/java-webapp-deploy.yml.tpl | 26 +----- tests/lean_java_workflow.bats | 81 +++++++++++++++++++ tests/maven_cache_key.bats | 47 +++++++++++ tests/maven_workspace.bats | 38 +++++++++ tests/own_ci_regression.cjs | 49 +++++++++++ tests/security_scan_owasp.bats | 45 +++++++++++ tests/test_generate_config.sh | 24 +++--- tests/test_own_ci.sh | 5 ++ tests/test_security_scan_owasp.sh | 3 + tests/test_targets.bats | 11 +++ 23 files changed, 487 insertions(+), 166 deletions(-) create mode 100644 docs/PLAN-lean-java-ci.md create mode 100644 src/scripts/maven_cache_key.sh create mode 100644 src/scripts/maven_workspace.sh create mode 100644 tests/lean_java_workflow.bats create mode 100644 tests/maven_cache_key.bats create mode 100644 tests/maven_workspace.bats create mode 100644 tests/own_ci_regression.cjs create mode 100644 tests/security_scan_owasp.bats create mode 100644 tests/test_own_ci.sh create mode 100644 tests/test_security_scan_owasp.sh create mode 100644 tests/test_targets.bats diff --git a/.circleci/config.yml b/.circleci/config.yml index f4df765..1c48adf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,27 +52,21 @@ workflows: - test: filters: *filters - test-deploy: - jobs: - - orb-tools/pack: - filters: *tag-only - orb-tools/publish: orb_name: kof22/munitor vcs_type: << pipeline.project.type >> pub_type: production requires: + - orb-tools/lint - orb-tools/pack + - orb-tools/review + - test context: - orb-publishing filters: *tag-only - # develop owns the shared dev:snapshot tag. Feature branches publish only an - # immutable dev: tag so open branches cannot overwrite each other's - # snapshot (issue #26). - snapshot: - jobs: - - orb-tools/pack: - filters: *filters + # All publishers consume the same packed workspace after all checks pass. + # Only develop updates dev:snapshot; feature branches publish dev:. - orb-tools/publish: name: publish-develop-snapshot orb_name: kof22/munitor @@ -80,7 +74,10 @@ workflows: pub_type: dev dev_tags: "dev:${CIRCLE_SHA1},dev:snapshot" requires: + - orb-tools/lint - orb-tools/pack + - orb-tools/review + - test context: - orb-publishing filters: @@ -94,7 +91,10 @@ workflows: pub_type: dev dev_tags: "dev:${CIRCLE_SHA1}" requires: + - orb-tools/lint - orb-tools/pack + - orb-tools/review + - test context: - orb-publishing filters: diff --git a/Makefile b/Makefile index b22b845..717fe18 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ lint: test-scripts: @echo "Running script tests..." @for f in tests/test_*.sh; do \ - if [ -f "$$f" ]; then \ + if [ -f "$$f" ] && [ "$$f" != "tests/test_generate_config.sh" ]; then \ echo " Running $$f"; \ bash "$$f" || exit 1; \ fi; \ diff --git a/docs/PLAN-lean-java-ci.md b/docs/PLAN-lean-java-ci.md new file mode 100644 index 0000000..ecc0ec6 --- /dev/null +++ b/docs/PLAN-lean-java-ci.md @@ -0,0 +1,41 @@ +# Fast Java CI + +Backend ticket work is paused while this performance change is delivered. + +## Measured baseline + +Website-Backend PR311, CircleCI jobs6314–6318 on2026-10-19: + +- Build/test job:30m15s. Compilation49s; tests14m24s; Java installation4m27s. +- Maven cache48GiB: restore5m32s, save4m19s. +- Quality, coverage and security each restore the same48GiB again (5–6min). +- Coverage computation itself takes5.4s. Quality tools take57s combined. +- OWASP is disabled in this consumer, but security still installs Java and restores Maven. +- Test results are uploaded by both the test command and its enclosing job:777 results appear twice. + +Evidence: coordinator cache/cicd-baseline.json, cicd-test-timings.json and cicd-github-friction.md. + +## First implementation + +1. Replace the old broad Maven fallback with an exact v2 key based on checked-out dependency definitions. Keep that key stable across release-version rewrites and invalidate for nested POM or Maven configuration changes. Never restore the48GiB v1 cache. +2. Skip Java/Maven/NVD setup when OWASP is disabled; preserve Trivy. +3. Upload Maven test results once. +4. Pack the orb once per pipeline; publish only after its existing checks pass, preserving branch/tag behavior and required check names. +5. Run template tests only in their dedicated Make target, rather than also discovering them in the generic script loop. +6. Run Checkstyle, SpotBugs, PMD and coverage on the existing build worker for generated PR/develop/staging/release workflows. Keep the standalone jobs available for custom consumers and preserve production behavior. Generated downstream jobs depend on the completed build gate. +7. Render each immutable test fixture once per test run; retain all277 assertions. + +Maven caches are dependency reuse, not artifact handoff. A real three-module experiment failed SpotBugs with current target outputs and an older installed-module cache. Preserve locally installed version directories and ancestor metadata in a workspace archive, then overlay it after dependency-cache restore. The experiment passes for release versions, snapshots and classifier dependencies. Maven Resolver identifies local installs with empty repository IDs in `_remote.repositories`; this implementation depends on that Maven3 format. Do not replace this archive with the entire local repository. + +Cold dependency downloads on changed POMs are deliberate: no cross-version fallback accumulation. Do not delete caches, artifacts, tags or change branch protections. + +## Verification and rollout + +- Focused behavioral/config regressions, then required make all and BATS suites once the combined patch is complete. +- Publish immutable feature orb; run a backend feature canary with both pins matching that SHA. +- Compare real cache size and build/check durations against the baseline. No claimed speedup before measurement; do not merge the consumer until tests and all existing security gates pass. +- Release the proven orb and adopt its stable version through normal PRs. Track the already-running backend release separately; do not resume paused ticket work. + +## Follow-up decisions after measurement + +If tests dominate after cache repair, inspect measured slow fixtures and isolate/parallelize only tests proven independent. Exact-source artifact promotion is a separate future change requiring verified source identity; do not skip merged-source validation on the strength of an earlier PR run. Avoid a new orchestration framework or speculative configuration options. diff --git a/docs/SESSION-STATE.md b/docs/SESSION-STATE.md index ebe63c9..558e45b 100644 --- a/docs/SESSION-STATE.md +++ b/docs/SESSION-STATE.md @@ -1,3 +1,17 @@ +## Fast Java CI (2026-10-19) + +Active user priority: fast, agile, lightweight CI/CD; backend ticket goal paused. +Measured48GiB Maven cache restored four times and saved once per backend PR. +Work on feature/lean-java-ci: stable exact dependency cache keys with new namespace, +conditional OWASP setup, single test-result upload, and one orb pack per pipeline. +The first patch passed make all and15BATS. Further simplification combines quality +and coverage with the build, renders fixtures once, and carries installed reactor +artifacts in a small workspace archive. A real multi-module experiment proved this +artifact handoff necessary, including snapshot/classifier coverage. Expanded make +all and20BATS pass, including all277template assertions. Real consumer timing and +stable publication/adoption remain pending. No deployed speedup claimed. +See docs/PLAN-lean-java-ci.md. + ## Supplemental runtime freshness (2026-10-19) Backend PR311 image scan6307 failed because db-backup reused a cached apk-upgrade layer (libexpat2.8.5-r0, fixed2.9.0-r0). Supplemental builds now use --pull --no-cache in both PR scan-only and release publishing paths. Trivy and immutable-tag guards remain unchanged. Targeted tests cover both Dockerfile forms, fresh-build flags, scan-before-push ordering and scan failure propagation. Validation pending; no release or consumer adoption yet. diff --git a/docs/TODO.md b/docs/TODO.md index 9ab64e4..d359b8f 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -2,6 +2,8 @@ ## Pending +- [ ] Validate and publish fast Java CI patch, canary actual cache size/timings, then adopt stable orb (docs/PLAN-lean-java-ci.md). + - [ ] Validate and release supplemental-image freshness fix, then adopt it in backend PR311 and verify all supplemental Trivy scans. - [ ] Release JVM runtime cache fix and adopt the stable orb in Website-Backend; verify fresh image Trivy before release. diff --git a/src/commands/restore_maven_cache.yml b/src/commands/restore_maven_cache.yml index 9544088..15fd02b 100644 --- a/src/commands/restore_maven_cache.yml +++ b/src/commands/restore_maven_cache.yml @@ -1,7 +1,12 @@ description: > - Restore the Maven local repository cache based on pom.xml checksum. + Restore only the cache for the checked-out Maven dependency definitions. steps: + - run: + name: Identify Maven dependency cache + command: <> - restore_cache: keys: - - maven-v1-{{ checksum "pom.xml" }} - - maven-v1- + - maven-v2-{{ checksum ".munitor-maven-cache-key" }} + - run: + name: Restore this build's installed Maven artifacts + command: <> diff --git a/src/commands/save_maven_cache.yml b/src/commands/save_maven_cache.yml index 418c356..035224b 100644 --- a/src/commands/save_maven_cache.yml +++ b/src/commands/save_maven_cache.yml @@ -1,7 +1,12 @@ description: > - Save the Maven local repository to cache for future builds. + Save the Maven repository under the same immutable key used at restore time. steps: - save_cache: - key: maven-v1-{{ checksum "pom.xml" }} + key: maven-v2-{{ checksum ".munitor-maven-cache-key" }} paths: - ~/.m2/repository + - run: + name: Preserve installed Maven artifacts for this workspace + environment: + MAVEN_WORKSPACE_ACTION: save + command: <> diff --git a/src/jobs/mvn_build_and_test.yml b/src/jobs/mvn_build_and_test.yml index cfc7d80..d9ab322 100644 --- a/src/jobs/mvn_build_and_test.yml +++ b/src/jobs/mvn_build_and_test.yml @@ -3,6 +3,14 @@ description: > version, builds, tests, saves cache, and persists the workspace for downstream jobs. parameters: + quality_checks: + type: boolean + default: false + description: Run quality and coverage checks on the build worker before sharing artifacts. + min_instruction: + type: string + default: "70" + description: Minimum instruction coverage when quality_checks is enabled. java_version: type: string default: "21" @@ -29,8 +37,20 @@ steps: - mvn_set_version - mvn_build - mvn_test - - store_test_results: - path: target/surefire-reports + - when: + condition: << parameters.quality_checks >> + steps: + - mvn_checkstyle + - mvn_spotbugs + - mvn_pmd + - mvn_jacoco: + min_instruction: << parameters.min_instruction >> + - store_artifacts: + path: target/checkstyle-result.xml + destination: checkstyle-result.xml + - store_artifacts: + path: target/site/jacoco + destination: coverage-report - run: name: Persist version metadata command: <> diff --git a/src/jobs/security_scan.yml b/src/jobs/security_scan.yml index 66a5acd..5097d05 100644 --- a/src/jobs/security_scan.yml +++ b/src/jobs/security_scan.yml @@ -23,12 +23,12 @@ executor: steps: - attach_workspace: at: . - - install_java: - version: << parameters.java_version >> - - restore_maven_cache - when: condition: << parameters.owasp >> steps: + - install_java: + version: << parameters.java_version >> + - restore_maven_cache - restore_cache: keys: - nvd-cache-v1-{{ .Branch }} diff --git a/src/scripts/maven_cache_key.sh b/src/scripts/maven_cache_key.sh new file mode 100644 index 0000000..38e34a9 --- /dev/null +++ b/src/scripts/maven_cache_key.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +MUNITOR_HELPERS="${MUNITOR_HELPERS:-$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)/munitor_helpers.sh}" +# shellcheck source=munitor_helpers.sh +if [[ -f "${MUNITOR_HELPERS}" ]]; then + source "${MUNITOR_HELPERS}" +elif ! type munitor_header &>/dev/null; then + munitor_header() { echo "=== Munitor: ${1:-unknown} ==="; } +fi +munitor_header "maven_cache_key" + +# Hash the checked-out dependency definitions, not release-version rewrites. +# The same index travels with the workspace to downstream Maven jobs. +git ls-files -s -- pom.xml '**/pom.xml' '.mvn/*' > .munitor-maven-cache-key +if [[ ! -s .munitor-maven-cache-key ]]; then + echo "ERROR: Maven cache requires tracked dependency definitions." >&2 + exit 1 +fi diff --git a/src/scripts/maven_workspace.sh b/src/scripts/maven_workspace.sh new file mode 100644 index 0000000..05956ab --- /dev/null +++ b/src/scripts/maven_workspace.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +MUNITOR_HELPERS="${MUNITOR_HELPERS:-$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)/munitor_helpers.sh}" +# shellcheck source=munitor_helpers.sh +if [[ -f "${MUNITOR_HELPERS}" ]]; then + source "${MUNITOR_HELPERS}" +elif ! type munitor_header &>/dev/null; then + munitor_header() { echo "=== Munitor: ${1:-unknown} ==="; } +fi +munitor_header "maven_workspace" + +archive="${PWD}/.munitor-maven-installed.tgz" +repository="${HOME}/.m2/repository" +case "${MAVEN_WORKSPACE_ACTION:-restore}" in + save) + # Maven marks local installs with an empty repository ID (artifact>=). + # Carry those build outputs in the workspace, separately from cached dependencies. + files=$(mktemp) + trap 'rm -f "${files}"' EXIT + cd "${repository}" + while IFS= read -r -d '' tracking; do + if grep -qE '^[^#].*>=[[:space:]]*$' "${tracking}"; then + directory=$(dirname "${tracking}") + printf '%s\0' "${directory}" >> "${files}" + while [[ "${directory}" != . ]]; do + directory=$(dirname "${directory}") + if [[ -f "${directory}/maven-metadata-local.xml" ]]; then + printf '%s\0' "${directory}/maven-metadata-local.xml" >> "${files}" + fi + done + fi + done < <(find . -name _remote.repositories -type f -print0) + tar -czf "${archive}" --null -T "${files}" + ;; + restore) + if [[ -f "${archive}" ]]; then + mkdir -p "${repository}" + tar -xzf "${archive}" -C "${repository}" + fi + ;; + *) + echo "ERROR: Unknown Maven workspace action." >&2 + exit 1 + ;; +esac diff --git a/src/scripts/packed_generate_config.sh b/src/scripts/packed_generate_config.sh index 60cc71d..29121b2 100755 --- a/src/scripts/packed_generate_config.sh +++ b/src/scripts/packed_generate_config.sh @@ -358,6 +358,8 @@ workflows: jobs: - munitor/mvn_build_and_test: name: build-and-test + quality_checks: true + min_instruction: "${MUNITOR_COVERAGE_MIN}" java_version: "${MUNITOR_JAVA_VERSION}" context: - ${MUNITOR_CONTEXT_GITHUB} @@ -395,27 +397,6 @@ workflows: only: - /feature\/.*/ - /hotfix\/.*/ - - munitor/mvn_code_quality: - name: code-quality - java_version: "${MUNITOR_JAVA_VERSION}" - requires: - - build-and-test - filters: - branches: - only: - - /feature\/.*/ - - /hotfix\/.*/ - - munitor/mvn_coverage: - name: coverage - java_version: "${MUNITOR_JAVA_VERSION}" - min_instruction: "${MUNITOR_COVERAGE_MIN}" - requires: - - build-and-test - filters: - branches: - only: - - /feature\/.*/ - - /hotfix\/.*/ - munitor/security_scan: name: security-scan java_version: "${MUNITOR_JAVA_VERSION}" @@ -453,6 +434,8 @@ workflows: jobs: - munitor/mvn_build_and_test: name: build-and-test + quality_checks: true + min_instruction: "${MUNITOR_COVERAGE_MIN}" java_version: "${MUNITOR_JAVA_VERSION}" context: - ${MUNITOR_CONTEXT_GITHUB} @@ -473,25 +456,6 @@ workflows: branches: only: - /release\/.*/ - - munitor/mvn_code_quality: - name: code-quality - java_version: "${MUNITOR_JAVA_VERSION}" - requires: - - build-and-test - filters: - branches: - only: - - /release\/.*/ - - munitor/mvn_coverage: - name: coverage - java_version: "${MUNITOR_JAVA_VERSION}" - min_instruction: "${MUNITOR_COVERAGE_MIN}" - requires: - - build-and-test - filters: - branches: - only: - - /release\/.*/ - munitor/security_scan: name: security-scan java_version: "${MUNITOR_JAVA_VERSION}" @@ -526,7 +490,6 @@ workflows: sonar_project_key: ${MUNITOR_SONAR_PROJECT_KEY} requires: - build-and-test - - coverage context: - ${MUNITOR_CONTEXT_SONAR} filters: @@ -547,8 +510,7 @@ workflows: supplemental_images: '${MUNITOR_SUPPLEMENTAL_IMAGES_JSON}' ##ENDIF_SUPPLEMENTAL## requires: - - code-quality - - coverage + - build-and-test - secrets-scan - sast-scan - security-scan @@ -571,8 +533,7 @@ workflows: context: - ${MUNITOR_CONTEXT_GITHUB} requires: - - code-quality - - coverage + - build-and-test - secrets-scan - sast-scan - security-scan @@ -2572,6 +2533,8 @@ cat > /tmp/munitor/templates/partials/java-webapp-deploy.yml.tpl << 'MUNITOR_PAR jobs: - munitor/mvn_build_and_test: name: build-and-test + quality_checks: true + min_instruction: "${MUNITOR_COVERAGE_MIN}" java_version: "${MUNITOR_JAVA_VERSION}" context: - ${MUNITOR_CONTEXT_GITHUB} @@ -2589,23 +2552,6 @@ cat > /tmp/munitor/templates/partials/java-webapp-deploy.yml.tpl << 'MUNITOR_PAR filters: branches: only: __BRANCH_FILTER__ - - munitor/mvn_code_quality: - name: code-quality - java_version: "${MUNITOR_JAVA_VERSION}" - requires: - - build-and-test - filters: - branches: - only: __BRANCH_FILTER__ - - munitor/mvn_coverage: - name: coverage - java_version: "${MUNITOR_JAVA_VERSION}" - min_instruction: "${MUNITOR_COVERAGE_MIN}" - requires: - - build-and-test - filters: - branches: - only: __BRANCH_FILTER__ - munitor/security_scan: name: security-scan java_version: "${MUNITOR_JAVA_VERSION}" @@ -2638,7 +2584,6 @@ cat > /tmp/munitor/templates/partials/java-webapp-deploy.yml.tpl << 'MUNITOR_PAR sonar_project_key: ${MUNITOR_SONAR_PROJECT_KEY} requires: - build-and-test - - coverage context: - ${MUNITOR_CONTEXT_SONAR} filters: @@ -2656,8 +2601,7 @@ cat > /tmp/munitor/templates/partials/java-webapp-deploy.yml.tpl << 'MUNITOR_PAR supplemental_images: '${MUNITOR_SUPPLEMENTAL_IMAGES_JSON}' ##ENDIF_SUPPLEMENTAL## requires: - - code-quality - - coverage + - build-and-test - secrets-scan - sast-scan - security-scan @@ -2679,8 +2623,7 @@ cat > /tmp/munitor/templates/partials/java-webapp-deploy.yml.tpl << 'MUNITOR_PAR context: - ${MUNITOR_CONTEXT_GITHUB} requires: - - code-quality - - coverage + - build-and-test - secrets-scan - sast-scan - security-scan diff --git a/src/scripts/templates/java-webapp.yml.tpl b/src/scripts/templates/java-webapp.yml.tpl index 34665d7..4e3ee53 100644 --- a/src/scripts/templates/java-webapp.yml.tpl +++ b/src/scripts/templates/java-webapp.yml.tpl @@ -8,6 +8,8 @@ workflows: jobs: - munitor/mvn_build_and_test: name: build-and-test + quality_checks: true + min_instruction: "${MUNITOR_COVERAGE_MIN}" java_version: "${MUNITOR_JAVA_VERSION}" context: - ${MUNITOR_CONTEXT_GITHUB} @@ -45,27 +47,6 @@ workflows: only: - /feature\/.*/ - /hotfix\/.*/ - - munitor/mvn_code_quality: - name: code-quality - java_version: "${MUNITOR_JAVA_VERSION}" - requires: - - build-and-test - filters: - branches: - only: - - /feature\/.*/ - - /hotfix\/.*/ - - munitor/mvn_coverage: - name: coverage - java_version: "${MUNITOR_JAVA_VERSION}" - min_instruction: "${MUNITOR_COVERAGE_MIN}" - requires: - - build-and-test - filters: - branches: - only: - - /feature\/.*/ - - /hotfix\/.*/ - munitor/security_scan: name: security-scan java_version: "${MUNITOR_JAVA_VERSION}" @@ -103,6 +84,8 @@ workflows: jobs: - munitor/mvn_build_and_test: name: build-and-test + quality_checks: true + min_instruction: "${MUNITOR_COVERAGE_MIN}" java_version: "${MUNITOR_JAVA_VERSION}" context: - ${MUNITOR_CONTEXT_GITHUB} @@ -123,25 +106,6 @@ workflows: branches: only: - /release\/.*/ - - munitor/mvn_code_quality: - name: code-quality - java_version: "${MUNITOR_JAVA_VERSION}" - requires: - - build-and-test - filters: - branches: - only: - - /release\/.*/ - - munitor/mvn_coverage: - name: coverage - java_version: "${MUNITOR_JAVA_VERSION}" - min_instruction: "${MUNITOR_COVERAGE_MIN}" - requires: - - build-and-test - filters: - branches: - only: - - /release\/.*/ - munitor/security_scan: name: security-scan java_version: "${MUNITOR_JAVA_VERSION}" @@ -176,7 +140,6 @@ workflows: sonar_project_key: ${MUNITOR_SONAR_PROJECT_KEY} requires: - build-and-test - - coverage context: - ${MUNITOR_CONTEXT_SONAR} filters: @@ -197,8 +160,7 @@ workflows: supplemental_images: '${MUNITOR_SUPPLEMENTAL_IMAGES_JSON}' ##ENDIF_SUPPLEMENTAL## requires: - - code-quality - - coverage + - build-and-test - secrets-scan - sast-scan - security-scan @@ -221,8 +183,7 @@ workflows: context: - ${MUNITOR_CONTEXT_GITHUB} requires: - - code-quality - - coverage + - build-and-test - secrets-scan - sast-scan - security-scan diff --git a/src/scripts/templates/partials/java-webapp-deploy.yml.tpl b/src/scripts/templates/partials/java-webapp-deploy.yml.tpl index 1010d10..3afd420 100644 --- a/src/scripts/templates/partials/java-webapp-deploy.yml.tpl +++ b/src/scripts/templates/partials/java-webapp-deploy.yml.tpl @@ -2,6 +2,8 @@ jobs: - munitor/mvn_build_and_test: name: build-and-test + quality_checks: true + min_instruction: "${MUNITOR_COVERAGE_MIN}" java_version: "${MUNITOR_JAVA_VERSION}" context: - ${MUNITOR_CONTEXT_GITHUB} @@ -19,23 +21,6 @@ filters: branches: only: __BRANCH_FILTER__ - - munitor/mvn_code_quality: - name: code-quality - java_version: "${MUNITOR_JAVA_VERSION}" - requires: - - build-and-test - filters: - branches: - only: __BRANCH_FILTER__ - - munitor/mvn_coverage: - name: coverage - java_version: "${MUNITOR_JAVA_VERSION}" - min_instruction: "${MUNITOR_COVERAGE_MIN}" - requires: - - build-and-test - filters: - branches: - only: __BRANCH_FILTER__ - munitor/security_scan: name: security-scan java_version: "${MUNITOR_JAVA_VERSION}" @@ -68,7 +53,6 @@ sonar_project_key: ${MUNITOR_SONAR_PROJECT_KEY} requires: - build-and-test - - coverage context: - ${MUNITOR_CONTEXT_SONAR} filters: @@ -86,8 +70,7 @@ supplemental_images: '${MUNITOR_SUPPLEMENTAL_IMAGES_JSON}' ##ENDIF_SUPPLEMENTAL## requires: - - code-quality - - coverage + - build-and-test - secrets-scan - sast-scan - security-scan @@ -109,8 +92,7 @@ context: - ${MUNITOR_CONTEXT_GITHUB} requires: - - code-quality - - coverage + - build-and-test - secrets-scan - sast-scan - security-scan diff --git a/tests/lean_java_workflow.bats b/tests/lean_java_workflow.bats new file mode 100644 index 0000000..cb3ce5b --- /dev/null +++ b/tests/lean_java_workflow.bats @@ -0,0 +1,81 @@ +#!/usr/bin/env bats + +setup() { + ROOT="${BATS_TEST_DIRNAME}/.." + CONFIG="${BATS_TEST_TMPDIR}/rendered.yml" +} + +check_workflows() { + local expected="$1" + yq -o=json '.' "$CONFIG" | EXPECTED_COVERAGE="$expected" node -e ' + const assert = require("node:assert/strict"); + const config = JSON.parse(require("node:fs").readFileSync(0,"utf8")); + for (const [workflow, definition] of Object.entries(config.workflows)) { + const jobs = definition.jobs.map(entry => { + const [type, options] = Object.entries(entry)[0]; + assert.ok(!["munitor/mvn_code_quality", "munitor/mvn_coverage"].includes(type)); + return {type, ...options}; + }); + const names = jobs.map(job => job.name); + assert.equal(new Set(names).size, names.length, workflow); + const build = jobs.find(job => job.name === "build-and-test"); + assert.ok(build, workflow); + const expectedFilter = { + "pr-checks": ["/feature\\/.*/", "/hotfix\\/.*/"], + develop: "develop", staging: "staging", + "release-candidate": ["/release\\/.*/"], production: "main" + }[workflow]; + assert.deepEqual(build.filters.branches.only, expectedFilter, workflow); + if (workflow === "production") { + assert.equal(build.quality_checks, undefined); + assert.equal(build.min_instruction, undefined); + } else { + assert.equal(build.quality_checks, true, workflow); + assert.equal(String(build.min_instruction), process.env.EXPECTED_COVERAGE, workflow); + for (const gate of ["secrets-scan", "sast-scan", "security-scan"]) assert.ok(names.includes(gate), workflow); + } + for (const job of jobs) { + assert.deepEqual(job.filters, build.filters, `${workflow}/${job.name} filters`); + const dependencies = job.requires || []; + assert.equal(new Set(dependencies).size, dependencies.length, job.name); + for (const name of dependencies) assert.ok(names.includes(name), `${workflow}/${job.name}: ${name}`); + if (["image-scan", "security-scan", "e2e-tests", "sonar-scan"].includes(job.name)) assert.ok(dependencies.includes("build-and-test")); + if (["docker-build-push", "sbom"].includes(job.name)) { + assert.ok(dependencies.includes("build-and-test")); + if (workflow !== "production") for (const gate of ["secrets-scan", "sast-scan", "security-scan", "e2e-tests", "sonar-scan"].filter(name => names.includes(name))) assert.ok(dependencies.includes(gate), `${workflow}/${job.name}/${gate}`); + } + if (["update-cd-repo", "github-release"].includes(job.name)) assert.ok(dependencies.includes("docker-build-push")); + } + if (workflow === "pr-checks") assert.ok(names.includes("image-scan")); + } + for (const workflow of ["pr-checks", "develop", "staging", "release-candidate", "production"]) assert.ok(config.workflows[workflow]); + ' +} + +@test "full Java workflow folds quality into build and preserves optional gates" { + MUNITOR_CONFIG="${ROOT}/tests/fixtures/java-webapp.munitor.yml" MUNITOR_OUTPUT_FILE="$CONFIG" bash "${ROOT}/src/scripts/generate_config.sh" + check_workflows 80 + [ "$(yq '.workflows.develop.jobs[] | .[] | select(.name == "sonar-scan") | .name' "$CONFIG")" = "sonar-scan" ] +} + +@test "minimal Java workflow retains default coverage and production behavior" { + MUNITOR_CONFIG="${ROOT}/tests/fixtures/java-webapp-minimal.munitor.yml" MUNITOR_OUTPUT_FILE="$CONFIG" bash "${ROOT}/src/scripts/generate_config.sh" + check_workflows 70 +} + +@test "quality steps finish before workspace persistence and forward coverage threshold" { + yq -o=json '.' "${ROOT}/src/jobs/mvn_build_and_test.yml" | node -e ' + const assert = require("node:assert/strict"); + const job = JSON.parse(require("node:fs").readFileSync(0,"utf8")); + assert.equal(job.parameters.quality_checks.default, false); + assert.equal(job.parameters.min_instruction.default, "70"); + const index = name => job.steps.findIndex(step => step === name || typeof step === "object" && name in step); + assert.ok(index("mvn_test") < index("when")); + assert.ok(index("when") < index("persist_to_workspace")); + const block = job.steps[index("when")].when; + assert.equal(block.condition, "<< parameters.quality_checks >>"); + const commands = block.steps.map(step => typeof step === "string" ? step : Object.keys(step)[0]); + assert.deepEqual(commands.slice(0,4), ["mvn_checkstyle", "mvn_spotbugs", "mvn_pmd", "mvn_jacoco"]); + assert.equal(block.steps[3].mvn_jacoco.min_instruction, "<< parameters.min_instruction >>"); + ' +} diff --git a/tests/maven_cache_key.bats b/tests/maven_cache_key.bats new file mode 100644 index 0000000..20eed02 --- /dev/null +++ b/tests/maven_cache_key.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../src/scripts/maven_cache_key.sh" + cd "$BATS_TEST_TMPDIR" + git init -q + printf '1-SNAPSHOT\n' > pom.xml + git add pom.xml +} + +@test "release rewrites retain the restored cache identity" { + bash "$SCRIPT" + cp .munitor-maven-cache-key before + printf '1.2.3\n' > pom.xml + bash "$SCRIPT" + cmp before .munitor-maven-cache-key +} + +@test "nested checked-out dependency changes invalidate the cache" { + mkdir module + printf '\n' > module/pom.xml + git add module/pom.xml + bash "$SCRIPT" + cp .munitor-maven-cache-key before + printf 'changed\n' > module/pom.xml + git add module/pom.xml + bash "$SCRIPT" + ! cmp -s before .munitor-maven-cache-key +} + +@test "source changes do not invalidate dependency cache" { + bash "$SCRIPT" + cp .munitor-maven-cache-key before + printf 'class Example {}\n' > Example.java + git add Example.java + bash "$SCRIPT" + cmp before .munitor-maven-cache-key +} + +@test "restore has no broad fallback and save uses the same key" { + command_root="${BATS_TEST_DIRNAME}/../src/commands" + restore=$(sed -n '/maven-v2-/p' "$command_root/restore_maven_cache.yml") + save=$(sed -n '/maven-v2-/p' "$command_root/save_maven_cache.yml") + [[ "$restore" == *'maven-v2-{{ checksum ".munitor-maven-cache-key" }}'* ]] + [[ "$save" == *'maven-v2-{{ checksum ".munitor-maven-cache-key" }}'* ]] + ! grep -q 'maven-v1\|maven-v2-$' "$command_root/restore_maven_cache.yml" +} diff --git a/tests/maven_workspace.bats b/tests/maven_workspace.bats new file mode 100644 index 0000000..a14d5ee --- /dev/null +++ b/tests/maven_workspace.bats @@ -0,0 +1,38 @@ +#!/usr/bin/env bats + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../src/scripts/maven_workspace.sh" + export HOME="${BATS_TEST_TMPDIR}/home" + cd "$BATS_TEST_TMPDIR" +} + +@test "workspace overlays current installed artifacts and snapshot metadata without copying remote dependencies" { + repo="$HOME/.m2/repository" + mkdir -p "$repo/example/library/2-SNAPSHOT" "$repo/vendor/dependency/1" + printf 'library-2-SNAPSHOT.jar>=\r\nlibrary-2-SNAPSHOT.pom>=\r\nlibrary-2-SNAPSHOT-tests.jar>=\r\n' > "$repo/example/library/2-SNAPSHOT/_remote.repositories" + for extension in jar pom; do + printf 'current build\n' > "$repo/example/library/2-SNAPSHOT/library-2-SNAPSHOT.$extension" + done + printf 'current test classes\n' > "$repo/example/library/2-SNAPSHOT/library-2-SNAPSHOT-tests.jar" + printf 'snapshot metadata\n' > "$repo/example/library/maven-metadata-local.xml" + printf 'plugin prefix metadata\n' > "$repo/example/maven-metadata-local.xml" + printf 'dependency-1.jar>central=\n' > "$repo/vendor/dependency/1/_remote.repositories" + printf 'remote dependency\n' > "$repo/vendor/dependency/1/dependency-1.jar" + MAVEN_WORKSPACE_ACTION=save bash "$SCRIPT" + run tar -tzf .munitor-maven-installed.tgz + [ "$status" -eq 0 ] + [[ "$output" != *vendor* ]] + export HOME="${BATS_TEST_TMPDIR}/downstream" + bash "$SCRIPT" + restored="$HOME/.m2/repository/example/library" + cmp "$repo/example/library/2-SNAPSHOT/library-2-SNAPSHOT.jar" "$restored/2-SNAPSHOT/library-2-SNAPSHOT.jar" + cmp "$repo/example/library/maven-metadata-local.xml" "$restored/maven-metadata-local.xml" + cmp "$repo/example/maven-metadata-local.xml" "$restored/../maven-metadata-local.xml" + [ -f "$restored/2-SNAPSHOT/library-2-SNAPSHOT-tests.jar" ] + [ ! -e "$HOME/.m2/repository/vendor" ] +} + +@test "initial build without a workspace archive needs no overlay" { + run bash "$SCRIPT" + [ "$status" -eq 0 ] +} diff --git a/tests/own_ci_regression.cjs b/tests/own_ci_regression.cjs new file mode 100644 index 0000000..61f52f3 --- /dev/null +++ b/tests/own_ci_regression.cjs @@ -0,0 +1,49 @@ +// Public CI config only: no network, credentials or publication. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const config = JSON.parse(fs.readFileSync(0, 'utf8')); +const workflows = Object.values(config.workflows); +assert.equal(workflows.length, 1, 'all publishers must share the checked pack workspace'); +const jobs = workflows.flatMap(workflow => workflow.jobs).map(job => { + const [type, settings] = Object.entries(job)[0]; + return { type, ...settings, name: settings.name || type }; +}); +assert.equal(jobs.filter(job => job.type === 'orb-tools/pack').length, 1, 'pack once per pipeline'); +const gates = ['orb-tools/lint', 'orb-tools/pack', 'orb-tools/review', 'test']; +for (const gate of gates) assert.equal(jobs.filter(job => job.name === gate).length, 1, `retain required context ${gate}`); +const publishers = jobs.filter(job => job.type === 'orb-tools/publish'); +assert.equal(publishers.length, 3); +for (const job of publishers) { + assert.deepEqual([...job.requires].sort(), [...gates].sort(), `${job.name} must wait for every gate`); + assert.equal(job.orb_name, 'kof22/munitor'); + assert.deepEqual(job.context, ['orb-publishing']); +} +function matches(pattern, value) { + return pattern.startsWith('/') && pattern.endsWith('/') + ? new RegExp(`^(?:${pattern.slice(1, -1)})$`).test(value) : pattern === value; +} +function runs(job, kind, ref) { + const filter = job.filters?.[kind]; + if (!filter) return kind === 'branches'; // CircleCI ignores tags unless opted in. + const list = value => Array.isArray(value) ? value : [value]; + return (!filter.only || list(filter.only).some(pattern => matches(pattern, ref))) + && (!filter.ignore || !list(filter.ignore).some(pattern => matches(pattern, ref))); +} +const cases = [ + ['branches', 'develop', ['publish-develop-snapshot']], + ['branches', 'feature/ci-speed', ['publish-feature-sha']], + ['branches', 'main', []], ['branches', 'hotfix/1.2.3', []], + ['branches', 'release/1.2.3', []], ['branches', 'fix/ci-speed', []], + ['tags', 'v1.2.3', ['orb-tools/publish']], + ['tags', 'v1.2.3-RC.1', []], ['tags', 'v1.2', []], ['tags', 'other', []], +]; +for (const [kind, ref, expected] of cases) { + assert.deepEqual(publishers.filter(job => runs(job, kind, ref)).map(job => job.name).sort(), expected.sort(), `${kind}:${ref}`); + for (const gate of gates) assert.ok(runs(jobs.find(job => job.name === gate), kind, ref), `${gate} must run for ${ref}`); +} +assert.equal(jobs.find(job => job.name === 'publish-develop-snapshot').dev_tags, 'dev:${CIRCLE_SHA1},dev:snapshot'); +assert.equal(jobs.find(job => job.name === 'publish-feature-sha').dev_tags, 'dev:${CIRCLE_SHA1}'); +assert.equal(jobs.find(job => job.name === 'orb-tools/publish').pub_type, 'production'); +assert.equal(jobs.find(job => job.name === 'publish-feature-sha').pub_type, 'dev'); +assert.equal(jobs.find(job => job.name === 'publish-develop-snapshot').pub_type, 'dev'); +console.log('PASS: one pack, unchanged required contexts, all publish gates and 10 branch/tag cases'); diff --git a/tests/security_scan_owasp.bats b/tests/security_scan_owasp.bats new file mode 100644 index 0000000..731bd8c --- /dev/null +++ b/tests/security_scan_owasp.bats @@ -0,0 +1,45 @@ +#!/usr/bin/env bats + +setup() { + export JOB="${BATS_TEST_DIRNAME}/../src/jobs/security_scan.yml" +} + +# Resolve only this job's compile-time OWASP condition, rejecting unknown ones. +resolved_steps() { + yq -o=json '.' "$JOB" | jq --argjson enabled "$1" ' + def expand: + .[] | if type == "object" and has("when") then + if .when.condition != "<< parameters.owasp >>" then error("unknown condition") + elif $enabled then .when.steps | expand else empty end + else . end; + [.steps | expand]' +} + +@test "OWASP disabled skips Java Maven and NVD but retains Trivy and its artifact" { + run resolved_steps false + [ "$status" -eq 0 ] + echo "$output" | jq -e ' + map(if type == "string" then . else keys[0] end) == + ["attach_workspace", "restore_trivy_cache", "run_trivy", "save_trivy_cache", "store_artifacts"]' >/dev/null + echo "$output" | jq -e '.[-1].store_artifacts.path == "/tmp/trivy-results.json"' >/dev/null +} + +@test "OWASP enabled retains ordered Java Maven NVD scan and report workflow" { + run resolved_steps true + [ "$status" -eq 0 ] + echo "$output" | jq -e ' + map(if type == "string" then . else keys[0] end) == + ["attach_workspace", "install_java", "restore_maven_cache", "restore_cache", + "mvn_owasp", "save_cache", "restore_trivy_cache", "run_trivy", "save_trivy_cache", + "store_artifacts", "store_artifacts"]' >/dev/null + echo "$output" | jq -e ' + .[1].install_java.version == "<< parameters.java_version >>" and + .[3].restore_cache.keys == ["nvd-cache-v1-{{ .Branch }}", "nvd-cache-v1-"] and + .[5].save_cache.paths == ["~/.dependency-check"] and + .[9].store_artifacts.path == "target/dependency-check-report.html" and + .[10].store_artifacts.path == "/tmp/trivy-results.json"' >/dev/null +} + +@test "OWASP remains enabled by default" { + [ "$(yq '.parameters.owasp.default' "$JOB")" = true ] +} diff --git a/tests/test_generate_config.sh b/tests/test_generate_config.sh index 79ce8e9..4fa758c 100755 --- a/tests/test_generate_config.sh +++ b/tests/test_generate_config.sh @@ -8,8 +8,8 @@ PASS=0 FAIL=0 GENERATE_SCRIPT="${PROJECT_DIR}/src/scripts/generate_config.sh" -GENERATED_CONFIG=$(mktemp) -trap 'rm -f "${GENERATED_CONFIG}"' EXIT +GENERATED_DIR=$(mktemp -d) +trap 'rm -rf "${GENERATED_DIR}"' EXIT # -------------------------------------------------------------------------- # Helper: render a fixture by calling the real generate_config.sh script @@ -17,11 +17,15 @@ trap 'rm -f "${GENERATED_CONFIG}"' EXIT # -------------------------------------------------------------------------- render_fixture() { local fixture="$1" + local generated_config="${GENERATED_DIR}/$(basename "${fixture}")" - # Run the real script; capture stderr for diagnostics, return generated YAML - MUNITOR_CONFIG="${fixture}" MUNITOR_OUTPUT_FILE="${GENERATED_CONFIG}" \ - bash "${GENERATE_SCRIPT}" >/dev/null 2>&1 - cat "${GENERATED_CONFIG}" + # Fixtures are immutable and uniquely named: render once, assert many times. + if [[ ! -f "${generated_config}" ]]; then + MUNITOR_CONFIG="${fixture}" MUNITOR_OUTPUT_FILE="${generated_config}.tmp" \ + bash "${GENERATE_SCRIPT}" >/dev/null 2>&1 || return 1 + mv "${generated_config}.tmp" "${generated_config}" + fi + cat "${generated_config}" } run_test() { @@ -672,17 +676,17 @@ run_negative_context_test "java-webapp: production has no security-scan" \ "security_scan" # java-webapp: release-candidate has quality gates -run_context_test "java-webapp: release-candidate has code-quality" \ +run_context_test "java-webapp: release-candidate enables quality checks" \ "${FIXTURES_DIR}/java-webapp.munitor.yml" \ "release-candidate:" \ "production:" \ - "code-quality" + "quality_checks: true" -run_context_test "java-webapp: release-candidate has coverage" \ +run_context_test "java-webapp: release-candidate forwards coverage threshold" \ "${FIXTURES_DIR}/java-webapp.munitor.yml" \ "release-candidate:" \ "production:" \ - "mvn_coverage" + 'min_instruction: "80"' run_context_test "java-webapp: release-candidate has security-scan" \ "${FIXTURES_DIR}/java-webapp.munitor.yml" \ diff --git a/tests/test_own_ci.sh b/tests/test_own_ci.sh new file mode 100644 index 0000000..3a7053d --- /dev/null +++ b/tests/test_own_ci.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Protect required CI contexts and publication gates when consolidating workflows. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +yq -o=json '.' "${SCRIPT_DIR}/../.circleci/config.yml" | node "${SCRIPT_DIR}/own_ci_regression.cjs" diff --git a/tests/test_security_scan_owasp.sh b/tests/test_security_scan_owasp.sh new file mode 100644 index 0000000..271e8fa --- /dev/null +++ b/tests/test_security_scan_owasp.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +bats "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/security_scan_owasp.bats" diff --git a/tests/test_targets.bats b/tests/test_targets.bats new file mode 100644 index 0000000..4aec352 --- /dev/null +++ b/tests/test_targets.bats @@ -0,0 +1,11 @@ +#!/usr/bin/env bats + +@test "combined script and template targets run each suite once" { + mkdir -p "$BATS_TEST_TMPDIR/tests" + printf 'echo SCRIPT_EXECUTED\n' > "$BATS_TEST_TMPDIR/tests/test_example.sh" + printf 'echo TEMPLATE_EXECUTED\n' > "$BATS_TEST_TMPDIR/tests/test_generate_config.sh" + run make --no-print-directory -f "$BATS_TEST_DIRNAME/../Makefile" -C "$BATS_TEST_TMPDIR" test-scripts test-templates + [ "$status" -eq 0 ] + [ "$(echo "$output" | grep -c '^SCRIPT_EXECUTED$')" -eq 1 ] + [ "$(echo "$output" | grep -c '^TEMPLATE_EXECUTED$')" -eq 1 ] +} -- 2.54.0 From d7c278b34406f8f366b1bf4f9ac5e3cd81f660f7 Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 19 Oct 2026 10:46:38 -0500 Subject: [PATCH 2/3] fix(ci): preserve required Java PR check name Give the differently parameterized production build a distinct name so CircleCI retains build-and-test for pull requests. Co-Authored-By: Codex Signed-off-by: James Maes --- src/scripts/packed_generate_config.sh | 4 ++-- src/scripts/templates/java-webapp.yml.tpl | 4 ++-- tests/lean_java_workflow.bats | 9 ++++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/scripts/packed_generate_config.sh b/src/scripts/packed_generate_config.sh index 29121b2..678b4cb 100755 --- a/src/scripts/packed_generate_config.sh +++ b/src/scripts/packed_generate_config.sh @@ -588,7 +588,7 @@ workflows: production: jobs: - munitor/mvn_build_and_test: - name: build-and-test + name: production-build-and-test java_version: "${MUNITOR_JAVA_VERSION}" context: - ${MUNITOR_CONTEXT_GITHUB} @@ -608,7 +608,7 @@ workflows: supplemental_images: '${MUNITOR_SUPPLEMENTAL_IMAGES_JSON}' ##ENDIF_SUPPLEMENTAL## requires: - - build-and-test + - production-build-and-test context: - ${MUNITOR_CONTEXT_REGISTRY} filters: diff --git a/src/scripts/templates/java-webapp.yml.tpl b/src/scripts/templates/java-webapp.yml.tpl index 4e3ee53..9799488 100644 --- a/src/scripts/templates/java-webapp.yml.tpl +++ b/src/scripts/templates/java-webapp.yml.tpl @@ -238,7 +238,7 @@ workflows: production: jobs: - munitor/mvn_build_and_test: - name: build-and-test + name: production-build-and-test java_version: "${MUNITOR_JAVA_VERSION}" context: - ${MUNITOR_CONTEXT_GITHUB} @@ -258,7 +258,7 @@ workflows: supplemental_images: '${MUNITOR_SUPPLEMENTAL_IMAGES_JSON}' ##ENDIF_SUPPLEMENTAL## requires: - - build-and-test + - production-build-and-test context: - ${MUNITOR_CONTEXT_REGISTRY} filters: diff --git a/tests/lean_java_workflow.bats b/tests/lean_java_workflow.bats index cb3ce5b..776c499 100644 --- a/tests/lean_java_workflow.bats +++ b/tests/lean_java_workflow.bats @@ -18,8 +18,11 @@ check_workflows() { }); const names = jobs.map(job => job.name); assert.equal(new Set(names).size, names.length, workflow); - const build = jobs.find(job => job.name === "build-and-test"); + const buildName = workflow === "production" ? "production-build-and-test" : "build-and-test"; + const build = jobs.find(job => job.name === buildName); assert.ok(build, workflow); + assert.equal(jobs.filter(job => job.type === "munitor/mvn_build_and_test").length, 1); + assert.ok(!names.includes(workflow === "production" ? "build-and-test" : "production-build-and-test")); const expectedFilter = { "pr-checks": ["/feature\\/.*/", "/hotfix\\/.*/"], develop: "develop", staging: "staging", @@ -39,9 +42,9 @@ check_workflows() { const dependencies = job.requires || []; assert.equal(new Set(dependencies).size, dependencies.length, job.name); for (const name of dependencies) assert.ok(names.includes(name), `${workflow}/${job.name}: ${name}`); - if (["image-scan", "security-scan", "e2e-tests", "sonar-scan"].includes(job.name)) assert.ok(dependencies.includes("build-and-test")); + if (["image-scan", "security-scan", "e2e-tests", "sonar-scan"].includes(job.name)) assert.ok(dependencies.includes(buildName)); if (["docker-build-push", "sbom"].includes(job.name)) { - assert.ok(dependencies.includes("build-and-test")); + assert.ok(dependencies.includes(buildName)); if (workflow !== "production") for (const gate of ["secrets-scan", "sast-scan", "security-scan", "e2e-tests", "sonar-scan"].filter(name => names.includes(name))) assert.ok(dependencies.includes(gate), `${workflow}/${job.name}/${gate}`); } if (["update-cd-repo", "github-release"].includes(job.name)) assert.ok(dependencies.includes("docker-build-push")); -- 2.54.0 From 77cdf931c2413494f7609c7e91d9670c28ed6784 Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 19 Oct 2026 11:02:15 -0500 Subject: [PATCH 3/3] fix(ci): carry refreshed snapshot dependencies with build artifacts Overlay mutable external SNAPSHOT dependencies as well as locally installed reactor outputs so downstream jobs do not read older snapshots from the immutable cache. Co-Authored-By: Codex Signed-off-by: James Maes --- docs/PLAN-lean-java-ci.md | 2 +- src/scripts/maven_workspace.sh | 5 +++-- tests/maven_workspace.bats | 8 ++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/PLAN-lean-java-ci.md b/docs/PLAN-lean-java-ci.md index ecc0ec6..50c6250 100644 --- a/docs/PLAN-lean-java-ci.md +++ b/docs/PLAN-lean-java-ci.md @@ -25,7 +25,7 @@ Evidence: coordinator cache/cicd-baseline.json, cicd-test-timings.json and cicd- 6. Run Checkstyle, SpotBugs, PMD and coverage on the existing build worker for generated PR/develop/staging/release workflows. Keep the standalone jobs available for custom consumers and preserve production behavior. Generated downstream jobs depend on the completed build gate. 7. Render each immutable test fixture once per test run; retain all277 assertions. -Maven caches are dependency reuse, not artifact handoff. A real three-module experiment failed SpotBugs with current target outputs and an older installed-module cache. Preserve locally installed version directories and ancestor metadata in a workspace archive, then overlay it after dependency-cache restore. The experiment passes for release versions, snapshots and classifier dependencies. Maven Resolver identifies local installs with empty repository IDs in `_remote.repositories`; this implementation depends on that Maven3 format. Do not replace this archive with the entire local repository. +Maven caches are dependency reuse, not artifact handoff. A real three-module experiment failed SpotBugs with current target outputs and an older installed-module cache. Preserve locally installed version directories, mutable SNAPSHOT dependencies and ancestor metadata in a workspace archive, then overlay it after dependency-cache restore. The experiment passes for release versions, snapshots and classifier dependencies. Maven Resolver identifies local installs with empty repository IDs in `_remote.repositories`; this implementation depends on that Maven3 format. Do not replace this archive with the entire local repository. Cold dependency downloads on changed POMs are deliberate: no cross-version fallback accumulation. Do not delete caches, artifacts, tags or change branch protections. diff --git a/src/scripts/maven_workspace.sh b/src/scripts/maven_workspace.sh index 05956ab..6a00135 100644 --- a/src/scripts/maven_workspace.sh +++ b/src/scripts/maven_workspace.sh @@ -15,12 +15,13 @@ repository="${HOME}/.m2/repository" case "${MAVEN_WORKSPACE_ACTION:-restore}" in save) # Maven marks local installs with an empty repository ID (artifact>=). - # Carry those build outputs in the workspace, separately from cached dependencies. + # Carry build outputs and mutable SNAPSHOT dependencies separately from the cache. files=$(mktemp) trap 'rm -f "${files}"' EXIT cd "${repository}" while IFS= read -r -d '' tracking; do - if grep -qE '^[^#].*>=[[:space:]]*$' "${tracking}"; then + if [[ "${tracking}" == */*-SNAPSHOT/_remote.repositories ]] || + grep -qE '^[^#].*>=[[:space:]]*$' "${tracking}"; then directory=$(dirname "${tracking}") printf '%s\0' "${directory}" >> "${files}" while [[ "${directory}" != . ]]; do diff --git a/tests/maven_workspace.bats b/tests/maven_workspace.bats index a14d5ee..293ea22 100644 --- a/tests/maven_workspace.bats +++ b/tests/maven_workspace.bats @@ -18,10 +18,13 @@ setup() { printf 'plugin prefix metadata\n' > "$repo/example/maven-metadata-local.xml" printf 'dependency-1.jar>central=\n' > "$repo/vendor/dependency/1/_remote.repositories" printf 'remote dependency\n' > "$repo/vendor/dependency/1/dependency-1.jar" + mkdir -p "$repo/vendor/mutable/2-SNAPSHOT" + printf 'mutable-2-SNAPSHOT.jar>central=\n' > "$repo/vendor/mutable/2-SNAPSHOT/_remote.repositories" + printf 'updated snapshot\n' > "$repo/vendor/mutable/2-SNAPSHOT/mutable-2-SNAPSHOT.jar" MAVEN_WORKSPACE_ACTION=save bash "$SCRIPT" run tar -tzf .munitor-maven-installed.tgz [ "$status" -eq 0 ] - [[ "$output" != *vendor* ]] + [[ "$output" != *vendor/dependency* ]] export HOME="${BATS_TEST_TMPDIR}/downstream" bash "$SCRIPT" restored="$HOME/.m2/repository/example/library" @@ -29,7 +32,8 @@ setup() { cmp "$repo/example/library/maven-metadata-local.xml" "$restored/maven-metadata-local.xml" cmp "$repo/example/maven-metadata-local.xml" "$restored/../maven-metadata-local.xml" [ -f "$restored/2-SNAPSHOT/library-2-SNAPSHOT-tests.jar" ] - [ ! -e "$HOME/.m2/repository/vendor" ] + [ ! -e "$HOME/.m2/repository/vendor/dependency" ] + cmp "$repo/vendor/mutable/2-SNAPSHOT/mutable-2-SNAPSHOT.jar" "$HOME/.m2/repository/vendor/mutable/2-SNAPSHOT/mutable-2-SNAPSHOT.jar" } @test "initial build without a workspace archive needs no overlay" { -- 2.54.0