From c338642d8aa936dec507cb4c80c4ac3fff0aaa50 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Fri, 7 Aug 2026 14:43:36 -0400 Subject: [PATCH 1/7] perf(ci): let the build cache reuse core-web across PR and trunk (#36947) core-web's `is_pr` profile was auto-activated by -Dgithub.event.name=pull_request, which the artifact build passed. Three of its four properties (git.origin.branch, nx.affected.options, pretty.quick.options) were byte-identical to the defaults above it, so its only live effect was skip.validate=false: run eslint + prettier inside the artifact build, on PRs and nowhere else. That made the PR effective pom differ from trunk's, so a content-addressed build cache can never reuse this module across the two -- for checks that gate nothing the build consumes. Both executions sit at generate-resources; the expensive `nx run-many -t build` is at compile and does not depend on them. Lint and format now run under -Pvalidate in the Frontend Unit Tests job, which already builds the whole workspace, so they cost almost nothing there and leave the serial prefix that gates every test job. Verified: after this change the PR-shaped, trunk-shaped and bare builds all hash dotcms-core-web to fd879a30...; adding -Dskip.validate=false yields 2b0c0b9f..., confirming that property alone was the divergence. Also makes format-test's base explicit. It ran `nx format:check` with no --base, falling back to nx.json defaultBase "main" -- a LOCAL branch that only exists because a job passed require-main=true to prepare-runner. The test phase never does, so relocating the check would have broken it. origin/main is always present after a fetch-depth:0 checkout. Trade-off: lint/format failures now surface in the Frontend Unit Tests job rather than the earlier build job. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014a2iJy9JXRBSVdKBbmoZ2S --- .github/test-matrix.yml | 5 +++- .github/workflows/cicd_comp_build-phase.yml | 4 ++- core-web/pom.xml | 30 ++++++++++----------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/test-matrix.yml b/.github/test-matrix.yml index d9fe80aa8168..07f815deee5e 100644 --- a/.github/test-matrix.yml +++ b/.github/test-matrix.yml @@ -46,7 +46,10 @@ test_types: needs_node: true suites: - name: "Frontend Unit Tests" - maven_args: "test -pl :dotcms-core-web" + # -Pvalidate runs eslint + prettier (generate-resources). They used to run inside the + # Initial Artifact Build on PRs only, which kept core-web out of the build cache. + # This job already builds the whole workspace, so they cost almost nothing here. + maven_args: "test -pl :dotcms-core-web -Pvalidate" stage_name: "Frontend Tests" # === MULTI-SUITE TESTS === diff --git a/.github/workflows/cicd_comp_build-phase.yml b/.github/workflows/cicd_comp_build-phase.yml index 978df5121b78..b90b48f2dfa9 100644 --- a/.github/workflows/cicd_comp_build-phase.yml +++ b/.github/workflows/cicd_comp_build-phase.yml @@ -109,7 +109,9 @@ jobs: - uses: ./.github/actions/core-cicd/maven-job with: stage-name: "Initial Artifact Build" - maven-args: "clean install ${{ env.VALIDATE_PROFILE }} -Dprod=true -DskipTests=true -Dgithub.event.name=${{ github.event_name }}" + # No -Dgithub.event.name: its only effect was activating core-web's is_pr profile, + # which made the PR effective pom differ from trunk's for that module. + maven-args: "clean install ${{ env.VALIDATE_PROFILE }} -Dprod=true -DskipTests=true" generate-artifacts: true require-main: ${{ inputs.version == '1.0.0-SNAPSHOT' }} github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/core-web/pom.xml b/core-web/pom.xml index 452184a4ec03..258c50f32660 100644 --- a/core-web/pom.xml +++ b/core-web/pom.xml @@ -177,6 +177,11 @@ exec nx format:check + + --base=${git.origin.branch} @@ -387,22 +392,15 @@ - - is_pr - - - - github.event.name - pull_request - - - - origin/main - --base=${git.origin.branch} --head=HEAD - --branch=${git.origin.branch} - false - - + format From 7730b17c48c2c87681fc6fd0f6eedc94abce8f33 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Fri, 7 Aug 2026 16:47:20 -0400 Subject: [PATCH 2/7] Update core-web/pom.xml Co-authored-by: Steve Freudenthaler <31257998+sfreudenthaler@users.noreply.github.com> --- core-web/pom.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/core-web/pom.xml b/core-web/pom.xml index 258c50f32660..f1dbf19dc0a0 100644 --- a/core-web/pom.xml +++ b/core-web/pom.xml @@ -392,15 +392,6 @@ - format From e17b7d9e7932e445f84e0fee518a5848d7176b82 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Fri, 7 Aug 2026 16:56:02 -0400 Subject: [PATCH 3/7] perf(ci): back the Maven build cache with a shared S3 bucket (#36947) (#36961) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes part of #36947. **Stacked on #36960** — review that one first; this PR's diff is against it. ## What Gives the Apache Maven Build Cache Extension somewhere to keep its results: a shared S3 bucket on OVH object storage. The extension was **already installed and enabled** (`.mvn/extensions.xml`, v1.2.0) — but local-only, so on a fresh CI runner it cached nothing. The only missing piece was shared storage. Scoped to the **Initial Artifact Build**, the serial prefix that gates every test job. Test phases are deliberately *not* cached: a memoised green run would mean "we did not run", and this suite has measured flakes plus a hang that has burned a 122m job timeout. ## Transport The extension speaks HTTP `PUT`/`GET`/`HEAD`; S3 needs SigV4. Rather than add a Maven S3 wagon — the available ones aren't maintained (`seahen` 1.3.3 is 2021, `gkatzioura` 2.3 is 2019, both AWS SDK v1) — [`aws-sigv4-proxy`](https://github.com/awslabs/aws-sigv4-proxy) runs as a container and signs on the way out, and Maven talks to `127.0.0.1`. **No new Maven dependency.** ## Security A build cache untrusted code can write is a supply-chain vector: a poisoned entry is replayed as a build output on a trusted ref. This is [CVE-2025-36852](https://www.cve.org/CVERecord?id=CVE-2025-36852) ("CREEP"), which killed Nx's `@nx/s3-cache` and its siblings. **The control is the credential, not the client** — a job holding a writable key can bypass Maven entirely with one `aws s3 cp`, so `remote.save.enabled` is defence in depth, not the boundary. | Ref | Key | Writes | |---|---|---| | PR | `..._ACCESS_KEY_RO` (GetObject only) | no | | merge queue / trunk | `..._ACCESS_KEY` | yes | | fork PR | none — builds uncached | no | Plus: - `remote.save.final=true` — an existing entry is never overwritten. - The action **asserts** its read-only key is read-only (one `PUT`, expects `403`) instead of assuming it. A writable "read-only" key looks identical to a correct setup until abused. - Writing builds record a `provenance.json` beside each entry (ref, sha, run id, actor), first-writer-wins — nothing else in a bucket says which ref produced a hash. ## The subtle one: `alwaysRunPlugins` Load-bearing, not tuning. On a cache hit the extension skips cached plugin executions **including `install:install`** — measured: 1 jar in `~/.m2/repository` after a cold build, **0 after a hit**. This job exists to publish that repository as the `maven-repo` artifact ~25 test jobs consume. Same story for `docker-maven-plugin:build`, which writes the `docker-build.tar` the next step uploads. Both would have looked perfectly green on the cold populate run and broken everything on the first *warm* one. ## Verification Against MinIO before any of this was wired: | Check | Result | |---|---| | `PUT` / `GET` / `HEAD` through the proxy | `200` / `200` / `200` | | Missing key | `404` (a `403` reads as a hard error, not a miss) | | 10 MB body round-trip | byte-identical | | Build with an **empty** local cache | `Found cached build, restoring … by checksum` | | Remote unreachable | build exits `0`, logs an error, rebuilds | And separately, because `cicd_comp_build-phase.yml` fails a PR on a dirty tree while `openapi.yaml` is a tracked file generated at compile: built `:dotcms-core --am` twice — 12 modules restored including `dotcms-core`, `openapi.yaml` md5 identical, `git status` unchanged. ## Releases are not affected Release, LTS, nightly, manual-deploy and CLI-release workflows don't pass the secrets, and **no workflow in this repo uses `secrets: inherit`** (verified). The action sees empty credentials and exports an empty `BUILD_CACHE_ARGS`; those pipelines build from scratch exactly as today. A release is the build where "we didn't actually compile this" is least acceptable and the saving is worth least. ## Turning it off | Scope | How | |---|---| | One PR | label `CI: no build cache` | | Everything, now | repo/org variable `BUILD_CACHE_DISABLED=true` | | Local build | `-Dmaven.build.cache.enabled=false` | | Force rebuild, still publish | `-Dmaven.build.cache.skipCache=true` | ## Expected effect, honestly Ceiling is the ~7.0m of Maven time inside a 14.2m build job, against a 74–103m PR wall clock. Real but modest — the twin-tail shard rebalance (#36943) is still the bigger lever. `dotcms-core-web` only participates once #36960 lands. **The first merge-queue run is the canary**: it's what proves the OVH SigV4 handshake and the region derived from the endpoint host. If either is wrong the cache disables itself with a warning rather than failing the build. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_014a2iJy9JXRBSVdKBbmoZ2S --------- Co-authored-by: Claude Opus 5 (1M context) --- .../core-cicd/build-cache-remote/action.yml | 180 ++++++++++++++++++ .../actions/core-cicd/maven-job/action.yml | 3 +- .github/workflows/cicd_1-pr.yml | 8 + .github/workflows/cicd_2-merge-queue.yml | 8 + .github/workflows/cicd_3-trunk.yml | 6 + .github/workflows/cicd_comp_build-phase.yml | 71 +++++++ .mvn/maven-build-cache-config.xml | 7 + 7 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 .github/actions/core-cicd/build-cache-remote/action.yml diff --git a/.github/actions/core-cicd/build-cache-remote/action.yml b/.github/actions/core-cicd/build-cache-remote/action.yml new file mode 100644 index 000000000000..ad1f485f7daa --- /dev/null +++ b/.github/actions/core-cicd/build-cache-remote/action.yml @@ -0,0 +1,180 @@ +name: 'Remote Build Cache' +description: | + Points the Apache Maven Build Cache Extension at the S3 build-cache bucket. + + The extension speaks plain HTTP PUT/GET/HEAD, and S3 needs SigV4, so an + aws-sigv4-proxy container signs on the way out. Maven talks to 127.0.0.1. + + Exports BUILD_CACHE_ARGS for the Maven job to append. If the bucket secrets + are absent (forks, or the feature is off) it exports an empty string and the + build runs exactly as it does today. + +inputs: + enabled: + description: | + Set false to build without the cache. The caller wires this to a global + kill switch (BUILD_CACHE_DISABLED) and a per-PR label, so a suspected bad + entry can be bypassed without editing a workflow. + required: false + default: 'true' + bucket: + description: 'S3 bucket name for the build cache' + required: false + default: '' + endpoint: + description: 'S3 endpoint URL, e.g. https://s3.gra.io.cloud.ovh.net' + required: false + default: '' + access-key: + description: 'S3 access key. Use the READ-ONLY key on untrusted refs.' + required: false + default: '' + secret-key: + description: 'S3 secret key' + required: false + default: '' + region: + description: 'Signing region. Derived from the endpoint host when empty.' + required: false + default: '' + save: + description: | + Whether this job may push to the shared cache. Only trusted refs + (trunk, merge queue) should ever set this true — a writable cache + reached from untrusted code is a supply-chain vector (CVE-2025-36852). + Defence in depth only: the real control is handing PR jobs a + GetObject-only key. + required: false + default: 'false' + prefix: + description: 'Key prefix inside the bucket' + required: false + default: 'maven-build-cache' + port: + description: 'Localhost port for the signing proxy' + required: false + default: '8079' + +outputs: + enabled: + description: 'true when the remote cache was wired up' + value: ${{ steps.setup.outputs.enabled }} + +runs: + using: 'composite' + steps: + - id: setup + name: Start build-cache signing proxy + shell: bash + env: + ENABLED: ${{ inputs.enabled }} + BUCKET: ${{ inputs.bucket }} + ENDPOINT: ${{ inputs.endpoint }} + AWS_ACCESS_KEY_ID: ${{ inputs.access-key }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.secret-key }} + REGION_IN: ${{ inputs.region }} + SAVE: ${{ inputs.save }} + PREFIX: ${{ inputs.prefix }} + PORT: ${{ inputs.port }} + run: | + set -euo pipefail + echo "BUILD_CACHE_ARGS=" >> "$GITHUB_ENV" + echo "enabled=false" >> "$GITHUB_OUTPUT" + + if [[ "$ENABLED" != "true" ]]; then + echo "Remote build cache turned off (BUILD_CACHE_DISABLED variable or PR label)." + exit 0 + fi + if [[ -z "$BUCKET" || -z "$ENDPOINT" || -z "$AWS_ACCESS_KEY_ID" || -z "$AWS_SECRET_ACCESS_KEY" ]]; then + echo "Build-cache secrets not available; remote cache disabled." + exit 0 + fi + if [[ "${RUNNER_OS}" != "Linux" ]]; then + echo "Remote cache only wired for Linux runners; skipping on ${RUNNER_OS}." + exit 0 + fi + + HOST="${ENDPOINT#*://}"; HOST="${HOST%%/*}" + SCHEME="${ENDPOINT%%://*}"; [[ "$SCHEME" == "$ENDPOINT" ]] && SCHEME=https + + # OVH endpoints look like s3..io.cloud.ovh.net + REGION="$REGION_IN" + if [[ -z "$REGION" ]]; then + REGION=$(echo "$HOST" | cut -d. -f2) + echo "Derived signing region '${REGION}' from ${HOST}" + fi + if [[ -z "$REGION" ]]; then + echo "::warning::Could not derive a signing region from '${HOST}'; remote cache disabled." + exit 0 + fi + + docker run -d --name build-cache-sigv4 \ + -p "127.0.0.1:${PORT}:8080" \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + public.ecr.aws/aws-observability/aws-sigv4-proxy:latest \ + --name s3 --region "$REGION" \ + --host "$HOST" --sign-host "$HOST" \ + --upstream-url-scheme "$SCHEME" \ + --log-failed-requests + + # A HEAD on a key that does not exist must answer 404, not 403 — the + # extension reads 403 as a hard error rather than a cache miss. + for i in $(seq 1 20); do + CODE=$(curl -s -o /dev/null -w '%{http_code}' -I \ + "http://127.0.0.1:${PORT}/${BUCKET}/${PREFIX}/.probe" || echo 000) + case "$CODE" in + 404|200) echo "Signing proxy ready (probe HTTP ${CODE})"; break ;; + 403) echo "::warning::Bucket rejected the probe (403) — bad key or region. Remote cache disabled." + docker rm -f build-cache-sigv4 >/dev/null 2>&1 || true + exit 0 ;; + esac + sleep 1 + done + if [[ "${CODE:-000}" != "404" && "${CODE:-000}" != "200" ]]; then + echo "::warning::Signing proxy did not become ready (last HTTP ${CODE:-000}). Remote cache disabled." + docker logs build-cache-sigv4 2>&1 | tail -20 || true + docker rm -f build-cache-sigv4 >/dev/null 2>&1 || true + exit 0 + fi + + # This job is not supposed to be able to write. That is the whole security + # boundary — client-side flags are not, since a job holding a writable key + # can bypass Maven entirely. Prove the key is really read-only instead of + # assuming it, because a silently-writable "read-only" key looks identical + # to a correct setup until someone abuses it. + if [[ "$SAVE" != "true" ]]; then + WCODE=$(curl -s -o /dev/null -w '%{http_code}' -X PUT --data-binary '' \ + "http://127.0.0.1:${PORT}/${BUCKET}/${PREFIX}/_probe/readonly-check" || echo 000) + if [[ "$WCODE" == "403" ]]; then + echo "Verified: this job's credentials cannot write to the cache." + elif [[ "$WCODE" == "200" || "$WCODE" == "204" ]]; then + echo "::warning title=Build cache key is not read-only::A job that should only read \ + from the build cache was able to WRITE to it (HTTP ${WCODE}). Untrusted code can poison \ + entries that trusted builds later replay (CVE-2025-36852). Re-issue this key with \ + GetObject-only permissions." + else + echo "Read-only write-probe returned HTTP ${WCODE}; could not confirm either way." + fi + fi + + # alwaysRunPlugins is NOT optional. On a cache hit the extension skips + # every cached plugin execution, including install:install — measured: + # the module's jar then never lands in ~/.m2/repository, and this job + # exists to publish that repository as the `maven-repo` artifact ~25 test + # jobs consume. Same reasoning for docker-maven-plugin: a hit on + # dotcms-core would otherwise skip the execution that writes + # dotCMS/target/docker-build.tar, which the next step uploads. + # (Named goals, not globs — FINAL_ARGS is expanded unquoted in maven-job.) + ALWAYS_RUN="maven-install-plugin:install,docker-maven-plugin:build" + + # remote.save.final: never let a later build overwrite an existing entry. + ARGS="-Dmaven.build.cache.remote.enabled=true" + ARGS="$ARGS -Dmaven.build.cache.remote.url=http://127.0.0.1:${PORT}/${BUCKET}/${PREFIX}" + ARGS="$ARGS -Dmaven.build.cache.remote.save.enabled=${SAVE}" + ARGS="$ARGS -Dmaven.build.cache.remote.save.final=true" + ARGS="$ARGS -Dmaven.build.cache.lazyRestore=true" + ARGS="$ARGS -Dmaven.build.cache.alwaysRunPlugins=${ALWAYS_RUN}" + echo "BUILD_CACHE_ARGS=$ARGS" >> "$GITHUB_ENV" + echo "BUILD_CACHE_SAVE=${SAVE}" >> "$GITHUB_ENV" + echo "enabled=true" >> "$GITHUB_OUTPUT" + echo "Remote build cache enabled (save=${SAVE})." diff --git a/.github/actions/core-cicd/maven-job/action.yml b/.github/actions/core-cicd/maven-job/action.yml index 1a8cbf80bd5c..4b64f0e38980 100644 --- a/.github/actions/core-cicd/maven-job/action.yml +++ b/.github/actions/core-cicd/maven-job/action.yml @@ -307,7 +307,8 @@ runs: echo "Defaulting dotcms.core.compiler.release=${JAVA_MAJOR} to match java-version" fi - FINAL_ARGS=$(echo "$DEFAULT_ARGS $COMPILER_ARGS $MAVEN_ARGS" | tr ' ' '\n' | awk '!seen[$0]++' | tr '\n' ' ') + # Set by the build-cache-remote action when the S3 cache is wired up; empty otherwise. + FINAL_ARGS=$(echo "$DEFAULT_ARGS $COMPILER_ARGS ${BUILD_CACHE_ARGS:-} $MAVEN_ARGS" | tr ' ' '\n' | awk '!seen[$0]++' | tr '\n' ' ') if [[ "${{ runner.os }}" == "Windows" && "${{ inputs.native }}" == "true" ]]; then echo "Building Maven with args $FINAL_ARGS" diff --git a/.github/workflows/cicd_1-pr.yml b/.github/workflows/cicd_1-pr.yml index 3a4a57bc264a..8ba392e770a5 100644 --- a/.github/workflows/cicd_1-pr.yml +++ b/.github/workflows/cicd_1-pr.yml @@ -66,6 +66,14 @@ jobs: with: core-build: true run-pr-checks: true + secrets: + # READ-ONLY key: PR code is untrusted, and a build cache an attacker can + # write is replayed as a build output on trunk (CVE-2025-36852). Fork PRs + # receive no secrets at all and fall back to a normal uncached build. + build-cache-access-key: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_ACCESS_KEY_RO }} + build-cache-secret-key: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_SECRET_KEY_RO }} + build-cache-endpoint: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_ENDPOINT }} + build-cache-bucket: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_NAME }} permissions: contents: read packages: write diff --git a/.github/workflows/cicd_2-merge-queue.yml b/.github/workflows/cicd_2-merge-queue.yml index 0af9b08b50c1..4cb5de844bde 100644 --- a/.github/workflows/cicd_2-merge-queue.yml +++ b/.github/workflows/cicd_2-merge-queue.yml @@ -17,6 +17,14 @@ jobs: needs: [ initialize ] if: needs.initialize.outputs.found_artifacts == 'false' uses: ./.github/workflows/cicd_comp_build-phase.yml + secrets: + # Read-write: the merge queue is a trusted ref and is the primary cache + # populator — its builds are PR-shaped, so what it writes is what the next + # merge-queue build can reuse. + build-cache-access-key: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_ACCESS_KEY }} + build-cache-secret-key: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_SECRET_KEY }} + build-cache-endpoint: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_ENDPOINT }} + build-cache-bucket: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_NAME }} permissions: contents: read packages: write diff --git a/.github/workflows/cicd_3-trunk.yml b/.github/workflows/cicd_3-trunk.yml index 70c4c8668c84..92b2e6be6186 100644 --- a/.github/workflows/cicd_3-trunk.yml +++ b/.github/workflows/cicd_3-trunk.yml @@ -70,6 +70,12 @@ jobs: java-version: ${{ github.event.inputs.java-version || '' }} maven-compiler-release: ${{ github.event.inputs.maven-compiler-release || '' }} artifact-suffix: ${{ github.event.inputs.artifact-suffix || '' }} + secrets: + # Read-write: trunk is trusted. + build-cache-access-key: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_ACCESS_KEY }} + build-cache-secret-key: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_SECRET_KEY }} + build-cache-endpoint: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_ENDPOINT }} + build-cache-bucket: ${{ secrets.OVH_S3_BUILD_CACHE_BUCKET_NAME }} permissions: contents: read packages: write diff --git a/.github/workflows/cicd_comp_build-phase.yml b/.github/workflows/cicd_comp_build-phase.yml index b90b48f2dfa9..ef922019f7bf 100644 --- a/.github/workflows/cicd_comp_build-phase.yml +++ b/.github/workflows/cicd_comp_build-phase.yml @@ -58,6 +58,20 @@ on: required: false type: string default: '' + secrets: + # Remote build cache (S3). Absent secrets simply disable the cache, so any + # caller that does not pass these -- every release workflow -- builds from + # scratch exactly as it does today. + # Untrusted refs must be given the READ-ONLY key pair; see the + # build-cache-remote action for why. + build-cache-access-key: + required: false + build-cache-secret-key: + required: false + build-cache-endpoint: + required: false + build-cache-bucket: + required: false jobs: # Initial JDK Build @@ -105,6 +119,23 @@ jobs: echo "VALIDATE_PROFILE=" >> "$GITHUB_ENV" fi + # Wire the Maven build cache to the shared S3 bucket. Only the merge queue + # and trunk may write; every other ref reads. Exports BUILD_CACHE_ARGS. + # + # Two ways to turn it off without editing this file: + # - repo/org variable BUILD_CACHE_DISABLED=true (global kill switch) + # - PR label "CI : No Build Cache" (one pull request) + - name: Set up remote build cache + id: build-cache + uses: ./.github/actions/core-cicd/build-cache-remote + with: + enabled: "${{ vars.BUILD_CACHE_DISABLED != 'true' && !contains(github.event.pull_request.labels.*.name, 'CI : No Build Cache') }}" + bucket: ${{ secrets.build-cache-bucket }} + endpoint: ${{ secrets.build-cache-endpoint }} + access-key: ${{ secrets.build-cache-access-key }} + secret-key: ${{ secrets.build-cache-secret-key }} + save: ${{ github.event_name == 'merge_group' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }} + # Run the Maven build job - uses: ./.github/actions/core-cicd/maven-job with: @@ -122,6 +153,46 @@ jobs: maven-compiler-release: ${{ inputs.maven-compiler-release }} artifact-suffix: ${{ inputs.artifact-suffix }} + # Record who produced each cache entry. Nothing in an S3 bucket says which + # ref wrote a given hash, which is half of what makes a shared build cache + # a supply-chain risk (CVE-2025-36852). Written first-writer-wins, so an + # entry keeps the identity of the build that actually created it. + - name: Record build-cache provenance + if: env.BUILD_CACHE_SAVE == 'true' && success() + continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.build-cache-access-key }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.build-cache-secret-key }} + ENDPOINT: ${{ secrets.build-cache-endpoint }} + BUCKET: ${{ secrets.build-cache-bucket }} + run: | + set -uo pipefail + root="$HOME/.m2/build-cache" + [[ -d "$root" ]] || exit 0 + cat > /tmp/provenance.json </local' directory means this run built that module itself. + while IFS= read -r dir; do + rel="${dir#"$root"/}" + key="maven-build-cache/${rel%/local}/provenance.json" + aws --endpoint-url "$ENDPOINT" s3api head-object \ + --bucket "$BUCKET" --key "$key" >/dev/null 2>&1 && continue + aws --endpoint-url "$ENDPOINT" s3 cp /tmp/provenance.json \ + "s3://$BUCKET/$key" --only-show-errors && count=$((count+1)) + done < <(find "$root" -mindepth 5 -maxdepth 5 -type d -name local) + echo "Recorded provenance for $count cache entries." + # Check for unauthorized changes to the working directory (only for PR checks) - name: Check for changes to source during build if: inputs.run-pr-checks diff --git a/.mvn/maven-build-cache-config.xml b/.mvn/maven-build-cache-config.xml index f500f6dadf68..b74108c0037d 100644 --- a/.mvn/maven-build-cache-config.xml +++ b/.mvn/maven-build-cache-config.xml @@ -34,6 +34,13 @@ https://maven.apache.org/extensions/maven-build-cache-extension/maven-build-cach 3 + + From cf1d1d50a75aa6bd2a5b5d74e2411f4b21ae2c72 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Fri, 7 Aug 2026 17:41:04 -0400 Subject: [PATCH 4/7] ci: cache the Maven wrapper distribution (#36947) Every job runs ./mvnw, and on a cold runner mvnw downloads the ~40MB Maven distribution from Maven Central. Nothing cached it: maven-job caches ~/.m2/repository, installs, the pnpm store and Sonar, but not ~/.m2/wrapper. So a single run fetched the same zip roughly 26 times. Maven Central eventually answers 429. That is what killed this PR's own run 31218010178 -- Postman Container died after 455ms, before any dotCMS code ran: IOException: Server returned HTTP response code: 429 for URL: .../apache-maven/3.9.2/apache-maven-3.9.2-bin.zip at org.apache.maven.wrapper.MavenWrapperMain.main and fail-fast then cancelled 24 other jobs, so one transient upstream rate limit presents as a wholesale pipeline failure with no attributable cause. The key hashes .mvn/wrapper/maven-wrapper.properties, so it changes only when the Maven version does and is otherwise a permanent hit. Uses the combined actions/cache rather than the restore/save split the other caches use: this content is an immutable versioned download, so there is no risk of persisting a polluted cache, and letting any job save means the first one to run on a cold key repairs it for the rest of the matrix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014a2iJy9JXRBSVdKBbmoZ2S --- .github/actions/core-cicd/maven-job/action.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/actions/core-cicd/maven-job/action.yml b/.github/actions/core-cicd/maven-job/action.yml index 4b64f0e38980..2bbd9655fb12 100644 --- a/.github/actions/core-cicd/maven-job/action.yml +++ b/.github/actions/core-cicd/maven-job/action.yml @@ -168,6 +168,24 @@ runs: echo "${DOTCMS_LICENSE_KEY}" > ${DOTCMS_LICENSE_PATH}/license.dat echo "DOTCMS_LICENSE_FILE=${DOTCMS_LICENSE_PATH}/license.dat" >> "$GITHUB_ENV" + # The Maven distribution itself, which mvnw downloads from Maven Central on a + # cold runner. Without this every job in a run fetches the same ~40MB zip -- + # around 26 downloads per run -- and Maven Central eventually answers 429, + # killing the job before any dotCMS code runs (and, under fail-fast, taking + # the rest of the matrix with it). + # + # Unlike the caches below this uses the combined action rather than the + # restore/save split: the content is an immutable versioned download, so + # there is no risk of persisting a polluted cache, and letting every job save + # means the first one to run on a cold key repairs it for the rest. + - id: cache-maven-wrapper + name: Cache Maven Wrapper Distribution + uses: actions/cache@v4 + with: + path: ~/.m2/wrapper + # Changes only when the Maven version does, so this key is ~always a hit. + key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }} + - id: restore-cache-maven name: Restore Maven Repository Cache if: ${{ inputs.artifacts-from == '' }} From 0a356d95d67c89da85e04d70bdefa83dab4b48ca Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Fri, 7 Aug 2026 17:54:28 -0400 Subject: [PATCH 5/7] ci: pin actions/cache to a commit SHA (#36947) Semgrep flagged the new Cache Maven Wrapper Distribution step as a blocking finding (github-actions-mutable-action-tag): `v4` is a mutable tag the action owner can silently repoint, which is how the trivy-action and kics-github-action compromises worked. A cache action runs in every job in the pipeline and sees the build output, so it is a meaningful position to hold. Pinned to 0057852bfaa89a56745cba8c7296529d2fc39830, the commit v4 currently resolves to (also tagged v4.3.0), with the version in a trailing comment so the next bump is a deliberate edit. Only the new step is pinned. The six pre-existing actions/cache/restore@v4 and actions/cache/save@v4 uses in this file predate the scan and are unflagged; pinning them is worth doing but belongs in its own change. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014a2iJy9JXRBSVdKBbmoZ2S --- .github/actions/core-cicd/maven-job/action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/core-cicd/maven-job/action.yml b/.github/actions/core-cicd/maven-job/action.yml index 2bbd9655fb12..61452b19e960 100644 --- a/.github/actions/core-cicd/maven-job/action.yml +++ b/.github/actions/core-cicd/maven-job/action.yml @@ -180,7 +180,10 @@ runs: # means the first one to run on a cold key repairs it for the rest. - id: cache-maven-wrapper name: Cache Maven Wrapper Distribution - uses: actions/cache@v4 + # Pinned to a commit SHA, not the mutable v4 tag: a tag can be repointed by + # the action owner, which is how the trivy-action and kics-github-action + # compromises worked. Bump deliberately when updating. + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ~/.m2/wrapper # Changes only when the Maven version does, so this key is ~always a hit. From b0adf077e31e116b77d05a244181f79166bccce1 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Fri, 7 Aug 2026 18:05:14 -0400 Subject: [PATCH 6/7] ci: pin the sigv4 proxy image by digest (#36947) The build-cache action ran public.ecr.aws/aws-observability/aws-sigv4-proxy:latest, a mutable tag on a third-party image. That container is handed AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY and proxies every cache read and write, so whoever controls that tag is one push away from a credential-stealing position inside the build -- a strictly worse exposure than the actions/cache tag Semgrep flagged, and unflagged only because Semgrep's rule covers `uses:` and not `docker run`. Pinned to sha256:6cd48ff272e30b6c3c01c02eac42dc3376bc3efa6ed3377bccd484e1c1b389df, resolved three ways (docker buildx imagetools, the local RepoDigests of the pulled image, and the digest recorded during the MinIO smoke test) so the digest being pinned is the exact artifact validated end-to-end against the OVH bucket. It is an OCI image index covering linux/amd64 and linux/arm64, so pinning the index rather than a per-platform manifest keeps CI (amd64) and local development (arm64) on the same reference. Verified it still pulls after removing the local copy. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014a2iJy9JXRBSVdKBbmoZ2S --- .github/actions/core-cicd/build-cache-remote/action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/core-cicd/build-cache-remote/action.yml b/.github/actions/core-cicd/build-cache-remote/action.yml index ad1f485f7daa..3e5aca5270d4 100644 --- a/.github/actions/core-cicd/build-cache-remote/action.yml +++ b/.github/actions/core-cicd/build-cache-remote/action.yml @@ -108,10 +108,15 @@ runs: exit 0 fi + # Pinned by digest, not :latest. This container is handed the bucket + # credentials and proxies every cache read and write, so a repointed tag + # would be a credential-stealing position. The digest below is the one + # validated end-to-end against the OVH bucket; it is an OCI index + # covering linux/amd64 and linux/arm64. Bump deliberately. docker run -d --name build-cache-sigv4 \ -p "127.0.0.1:${PORT}:8080" \ -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ - public.ecr.aws/aws-observability/aws-sigv4-proxy:latest \ + public.ecr.aws/aws-observability/aws-sigv4-proxy@sha256:6cd48ff272e30b6c3c01c02eac42dc3376bc3efa6ed3377bccd484e1c1b389df \ --name s3 --region "$REGION" \ --host "$HOST" --sign-host "$HOST" \ --upstream-url-scheme "$SCHEME" \ From 7aeb1a6026306670e453a5b855bf4b7338e333c3 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Fri, 7 Aug 2026 20:33:51 -0400 Subject: [PATCH 7/7] ci: add "CI : No Fail Fast" label to keep the whole matrix running (#36947) fail-fast is right by default -- one broken suite should not burn 25 runners -- but it destroys evidence exactly when you need it. A single failure cancels the other ~24 jobs, and in the checks UI a cancelled job is indistinguishable from a failed one, so a lone flake presents as a wholesale pipeline failure. This PR's own run 31218010178 showed 24 red jobs from one transient Maven Central 429, and run 31222593504 showed 8 more from one leaked persona in PersonaAPITest. Labelling a PR "CI : No Fail Fast" now runs every suite to completion, so you can tell "this change broke one suite" from "this change broke twenty" in a single run instead of fixing, re-running, and discovering the next casualty. Costs runner time, so it stays opt-in per PR rather than becoming the default. merge_group events carry no pull_request, so contains() is false there and the queue keeps bailing early, which is what you want when the tree is at stake. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014a2iJy9JXRBSVdKBbmoZ2S --- .github/workflows/cicd_comp_test-phase.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd_comp_test-phase.yml b/.github/workflows/cicd_comp_test-phase.yml index 68c1dfd4bcac..932d48665d1b 100644 --- a/.github/workflows/cicd_comp_test-phase.yml +++ b/.github/workflows/cicd_comp_test-phase.yml @@ -232,7 +232,17 @@ jobs: # a comma-list subset like '1,2', or 'all') run every suite/phase to completion so # failures are attributable per phase. Derived from "phase off" rather than an # exact-match list so comma-list subsets are covered too. - fail-fast: ${{ inputs.opensearch_phase == '' || inputs.opensearch_phase == 'none' || inputs.opensearch_phase == '0' }} + # + # The label escape hatch exists because fail-fast destroys evidence: one failing + # suite cancels the other ~24, and a cancelled job is indistinguishable from a + # failed one at a glance. When you are chasing a flake, or want to know whether a + # change broke one suite or twenty, label the PR "CI : No Fail Fast" and every + # suite runs to completion. Costs runner time, so it is opt-in per PR. + # On merge_group there is no pull_request in the event, contains() is false, and + # fast-fail stays on -- the queue should still bail early. + fail-fast: >- + ${{ (inputs.opensearch_phase == '' || inputs.opensearch_phase == 'none' || inputs.opensearch_phase == '0') + && !contains(github.event.pull_request.labels.*.name, 'CI : No Fail Fast') }} matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix) }} steps: