Copa hardened fixes

.github/scripts/generate-cve-report.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# Render docs/known-cves.md from per-image CVE data collected by the release workflow.
+# Usage: generate-cve-report.sh <data-dir> <timestamp>
+#   <data-dir>/<key>.meta.json      {image,variant,arch,plain_digest,hardened_digest}
+#   <data-dir>/<key>.plain.json     full Trivy JSON of the plain image
+#   <data-dir>/<key>.hardened.json  full Trivy JSON of the hardened image (may be absent)
+set -euo pipefail
+
+data_dir="${1:?usage: generate-cve-report.sh <data-dir> <timestamp>}"
+timestamp="${2:?usage: generate-cve-report.sh <data-dir> <timestamp>}"
+
+short_digest() { case "$1" in sha256:*) printf '%s' "${1#sha256:}" | cut -c1-12 ;; *) printf '%s' "$1" ;; esac; }
+sev_rank() { case "$1" in CRITICAL) echo 0;; HIGH) echo 1;; MEDIUM) echo 2;; LOW) echo 3;; *) echo 4;; esac; }
+
+cat <<EOF
+# Known CVEs & hardening report
+
+_Generated ${timestamp}._
+
+Per published **stable release image** and architecture: the plain / Copa-hardened digests
+and every known CVE with its status. CVE data is a full Trivy scan (all severities,
+OS + library packages, unfixable CVEs included). **Development / rolling tags (\`*-dev\`) are
+not covered** — they are plain-only and never Copa-patched.
+
+**Status legend:** ✅ \`fixed\` = Copa patched it (old → new version) · ⚠️ \`residual\` = still
+present in the hardened image (\`no fix\` = no upstream fix available). The **Image digest**
+column in the CVE table is a 12-char pointer — the **plain** digest for \`fixed\` rows, the
+**hardened** digest for \`residual\` rows (full digests in the table below).
+
+## Image digests
+
+| Image | Arch | Plain digest | Hardened digest |
+|-------|------|--------------|-----------------|
+EOF
+
+metas=$(find "$data_dir" -maxdepth 1 -name '*.meta.json' | LC_ALL=C sort)
+
+for meta in $metas; do
+    image=$(jq -r '.image' "$meta"); arch=$(jq -r '.arch' "$meta")
+    pd=$(jq -r '.plain_digest' "$meta"); hd=$(jq -r '.hardened_digest' "$meta")
+    if [ "$pd" = "unpublished" ]; then pd_disp="_not published this run_"; else pd_disp="\`${pd}\`"; fi
+    if [ "$hd" = "unpublished" ]; then hd_disp="_not published this run_"; else hd_disp="\`${hd}\`"; fi
+    printf '| `%s` | %s | %s | %s |\n' "$image" "$arch" "$pd_disp" "$hd_disp"
+done
+
+cat <<EOF
+
+## CVEs
+
+| Image | Arch | Image digest | CVE | Severity | Package | Status |
+|-------|------|--------------|-----|----------|---------|--------|
+EOF
+
+for meta in $metas; do
+    key=$(basename "$meta" .meta.json)
+    image=$(jq -r '.image' "$meta"); arch=$(jq -r '.arch' "$meta")
+    pd=$(jq -r '.plain_digest' "$meta"); hd=$(jq -r '.hardened_digest' "$meta")
+    plain_json="${data_dir}/${key}.plain.json"; hardened_json="${data_dir}/${key}.hardened.json"
+    pd_short=$(short_digest "$pd"); hd_short=$(short_digest "$hd")
+
+    rows=""
+    if [ -f "$hardened_json" ]; then
+        # residual = every vuln still in the hardened image
+        # Capture jq output into a variable first (under set -e) rather than reading
+        # it via process substitution: `while … done < <(jq …)` hides a jq failure
+        # from set -e/pipefail entirely -- the loop just reads zero lines and the
+        # script still exits 0, silently producing an incomplete report on malformed
+        # Trivy JSON. A plain command-substitution assignment aborts loudly instead.
+        residual_tsv="$(jq -r '[.Results[]?.Vulnerabilities[]?] | unique_by([.VulnerabilityID,.PkgName]) | .[] | [.VulnerabilityID,.Severity,.PkgName,.InstalledVersion,(.FixedVersion//"")] | @tsv' "$hardened_json")"
+        while IFS=$'\t' read -r id sev pkg _installed fixed; do
+            [ -z "$id" ] && continue
+            if [ -n "$fixed" ]; then fv="fix ${fixed} available"; else fv="no fix"; fi
+            rows+="$(sev_rank "$sev")1"$'\t'"| \`${image}\` | ${arch} | \`${hd_short}\` | ${id} | ${sev} | ${pkg} | ⚠️ residual · ${fv} |"$'\n'
+        done <<< "$residual_tsv"
+        # fixed = in plain, absent from hardened -- keyed on VulnerabilityID+PkgName
+        # (NUL-joined) with EXACT array membership (index), not id-only `inside`.
+        # `inside`/`contains` on string arrays is recursive substring matching (e.g.
+        # ["CVE-2024-1"] | inside(["CVE-2024-12345"]) is true), and keying on id
+        # alone conflates a CVE across packages: if a CVE affects both pkgA and
+        # pkgB and Copa fixes only pkgA, the id stays in the hardened id set (via
+        # pkgB) and the pkgA row would incorrectly be excluded from "fixed" (and
+        # isn't residual for pkgA either) -- it would vanish from both tables.
+        fixed_tsv="$(jq -r --slurpfile h "$hardened_json" '
+            ([$h[0].Results[]?.Vulnerabilities[]? | .VulnerabilityID + "\u0000" + .PkgName] | unique) as $hkeys
+            | [.Results[]?.Vulnerabilities[]?]
+            | map(select( (.VulnerabilityID + "\u0000" + .PkgName) as $key | ($hkeys | index($key)) == null ))
+            | unique_by([.VulnerabilityID,.PkgName])
+            | .[] | [.VulnerabilityID,.Severity,.PkgName,.InstalledVersion,(.FixedVersion//"?")] | @tsv
+        ' "$plain_json")"
+        while IFS=$'\t' read -r id sev pkg old new; do
+            [ -z "$id" ] && continue
+            rows+="$(sev_rank "$sev")0"$'\t'"| \`${image}\` | ${arch} | \`${pd_short}\` | ${id} | ${sev} | ${pkg} | ✅ fixed · ${old} → ${new} |"$'\n'
+        done <<< "$fixed_tsv"
+    else
+        # hardened image was not produced (gate failed) — list the plain CVEs as unpatched
+        unpatched_tsv="$(jq -r '[.Results[]?.Vulnerabilities[]?] | unique_by([.VulnerabilityID,.PkgName]) | .[] | [.VulnerabilityID,.Severity,.PkgName,.InstalledVersion,(.FixedVersion//"")] | @tsv' "$plain_json")"
+        while IFS=$'\t' read -r id sev pkg _installed _fixed; do
+            [ -z "$id" ] && continue
+            rows+="$(sev_rank "$sev")1"$'\t'"| \`${image}\` | ${arch} | \`${pd_short}\` | ${id} | ${sev} | ${pkg} | ⚠️ unpatched · hardened not produced |"$'\n'
+        done <<< "$unpatched_tsv"
+    fi
+    # sort this image's rows by severity then fixed-before-residual, drop the sort key
+    [ -n "$rows" ] && printf '%s' "$rows" | LC_ALL=C sort -t$'\t' -k1,1 | cut -f2-
+done

.github/scripts/scan-patch-gate.sh

@@ -6,6 +6,7 @@
 # still ships and other variants continue. Genuine infra errors abort (set -e).
 set -euo pipefail
 
+HERE="$(cd "$(dirname "$0")" && pwd)"   # for sibling helpers (with-retry.sh)
 variant="${1:?usage: scan-patch-gate.sh <variant>}"
 : "${IMAGE_NAME:?}"; : "${ARCH_TAG:?}"; : "${GATE_SEVERITY:?}"
 STATE_DIR="${STATE_DIR:-.docker-state}"
@@ -40,7 +41,7 @@ fail_gate() { # <reason> -- record + skip hardened, but let plain ship
 }
 
 echo "Scanning plain image ${PLAIN_IMAGE} for OS vulnerabilities"
-trivy image --pkg-types os --ignore-unfixed --format json -o "$report" "${PLAIN_IMAGE}" \
+"${HERE}/with-retry.sh" trivy image --pkg-types os --ignore-unfixed --format json -o "$report" "${PLAIN_IMAGE}" \
     || fail_gate "Trivy scan of plain image failed"
 
 jq empty "$report" 2>/dev/null || fail_gate "Trivy report of plain image is not valid JSON"
@@ -73,7 +74,7 @@ if [ "$GATE_SEVERITY" != "NONE" ]; then
     REPORT_JSON="${REPORT_DIR}/${TAG}-hardened_${IMAGE_HASH}.json"
     REPORT_TXT="${REPORT_DIR}/${TAG}-hardened_${IMAGE_HASH}.txt"
 
-    trivy image --pkg-types os --ignore-unfixed --severity "$GATE_SEVERITY" \
+    "${HERE}/with-retry.sh" trivy image --pkg-types os --ignore-unfixed --severity "$GATE_SEVERITY" \
         --format json -o "${REPORT_JSON}" "${HARDENED_IMAGE}" \
         || fail_gate "Trivy gate scan failed"
 
@@ -100,7 +101,7 @@ fi
 
 # Gate passed (or disabled): generate the SBOM first, then publish the markers atomically.
 HARDENED_SBOM="${SBOM_DIR}/${BASE_TAG}-${VERSION}-hardened-${ARCH_TAG}.spdx.json"
-trivy image --format spdx-json -o "${HARDENED_SBOM}" "${HARDENED_IMAGE}" \
+"${HERE}/with-retry.sh" trivy image --format spdx-json -o "${HARDENED_SBOM}" "${HARDENED_IMAGE}" \
     || fail_gate "hardened SBOM generation failed"
 
 while IFS= read -r plain_tag; do

.github/scripts/tests/fixtures/cve/v5.2-amd64.hardened.json

@@ -0,0 +1,3 @@
+{"Results":[{"Vulnerabilities":[
+{"VulnerabilityID":"CVE-2025-6020","Severity":"HIGH","PkgName":"libpam0g","InstalledVersion":"1.5.3-7","FixedVersion":""}
+]}]}

.github/scripts/tests/fixtures/cve/v5.2-amd64.meta.json

@@ -0,0 +1 @@
+{"image":"php8.5-v5.2","variant":"default","arch":"amd64","plain_digest":"sha256:1f3a9c4e2b7d05a8c1e6f4b9d3072a5e8c1b6f0d4a7e2c9b5083f1d6a4c7e2b90","hardened_digest":"sha256:8ad4c17b93e0a5d2f4681c9b0e3a7d5c2b8f1069a4e7c3b05d9f28a1c6e4b0f37"}

.github/scripts/tests/fixtures/cve/v5.2-amd64.plain.json

@@ -0,0 +1,4 @@
+{"Results":[{"Vulnerabilities":[
+{"VulnerabilityID":"CVE-2024-45491","Severity":"HIGH","PkgName":"libexpat1","InstalledVersion":"2.6.2-1","FixedVersion":"2.6.2-2+deb13u1"},
+{"VulnerabilityID":"CVE-2025-6020","Severity":"HIGH","PkgName":"libpam0g","InstalledVersion":"1.5.3-7","FixedVersion":""}
+]}]}

.github/scripts/tests/fixtures/cve/v5.2-debug-amd64.meta.json

@@ -0,0 +1 @@
+{"image":"php8.5-debug-v5.2","variant":"debug","arch":"amd64","plain_digest":"sha256:44bec0a1d2e3f405162738495a6b7c8d9e0f1a2b3c4d5e6f7081920a3b4c5d6e7","hardened_digest":"unpublished"}

.github/scripts/tests/fixtures/cve/v5.2-debug-amd64.plain.json

@@ -0,0 +1,3 @@
+{"Results":[{"Vulnerabilities":[
+{"VulnerabilityID":"CVE-2024-7883","Severity":"MEDIUM","PkgName":"libxml2","InstalledVersion":"2.12.7+dfsg-3","FixedVersion":""}
+]}]}

.github/scripts/tests/fixtures/cve/v5.2-multipkg-amd64.hardened.json

@@ -0,0 +1,3 @@
+{"Results":[{"Vulnerabilities":[
+{"VulnerabilityID":"CVE-2025-9999","Severity":"HIGH","PkgName":"pkgb","InstalledVersion":"2.0","FixedVersion":""}
+]}]}

.github/scripts/tests/fixtures/cve/v5.2-multipkg-amd64.meta.json

@@ -0,0 +1 @@
+{"image":"php8.5-multipkg-v5.2","variant":"default","arch":"amd64","plain_digest":"sha256:dd81d549a32e1c53156f2fc83da5814f0637389fef78ed6e2d27f8f3384fe742","hardened_digest":"sha256:d07739baf7cd22301c30836a09ee577cf7a81b0c2904c8d78ed81779127ba4bf"}

.github/scripts/tests/fixtures/cve/v5.2-multipkg-amd64.plain.json

@@ -0,0 +1,4 @@
+{"Results":[{"Vulnerabilities":[
+{"VulnerabilityID":"CVE-2025-9999","Severity":"HIGH","PkgName":"pkga","InstalledVersion":"1.0","FixedVersion":"1.1"},
+{"VulnerabilityID":"CVE-2025-9999","Severity":"HIGH","PkgName":"pkgb","InstalledVersion":"2.0","FixedVersion":""}
+]}]}

.github/scripts/tests/run.sh

@@ -134,7 +134,10 @@ setup_variant() { # <dir> <variant>
         "ghcr.io/pimcore/pimcore:php8.5-$2-v5.1-amd64" > "$d/plain_tags.txt"
 }
 run_gate() { # runs scan-patch-gate.sh in <dir> with env already exported
-    ( cd "$1" && IMAGE_NAME=pimcore/pimcore ARCH_TAG=amd64 \
+    # RETRY_MAX=1: the with-retry.sh wrapper around trivy runs the command exactly
+    # once here (no sleeps, behaviour identical to the unwrapped call). with-retry's
+    # own retry/backoff is covered by its dedicated tests below.
+    ( cd "$1" && IMAGE_NAME=pimcore/pimcore ARCH_TAG=amd64 RETRY_MAX=1 \
         "${ROOT}/.github/scripts/scan-patch-gate.sh" "$2" ) 2>&1
 }
 
@@ -286,5 +289,54 @@ oras_attaches="$(grep -c 'attach' "$logS" 2>/dev/null || true)"
 assert_contains "$(cat "$logS")" "attach --artifact-type application/spdx+json docker.io/pimcore/pimcore:php8.5-v5.2 sboms/php8.5-v5.2-amd64.spdx.json" "S amd64 SBOM attached to the LOGICAL tag (docker.io-qualified)"
 assert_contains "$(cat "$logS")" "attach --artifact-type application/spdx+json docker.io/pimcore/pimcore:php8.5-v5.2 sboms/php8.5-v5.2-arm64.spdx.json" "S arm64 SBOM attached to the LOGICAL tag (docker.io-qualified)"
 
+echo "== with-retry.sh =="
+WR="${ROOT}/.github/scripts/with-retry.sh"
+
+# 1. success on the first try -> exit 0, command run once
+cf1="$(mktemp)"; tmpdirs+=("$cf1"); echo 0 > "$cf1"
+CF="$cf1" RETRY_DELAY=0 "$WR" bash -c 'echo $(( $(cat "$CF") + 1 )) > "$CF"' ; rc=$?
+[ "$rc" = 0 ] && echo "  ok: success -> exit 0" || { echo "  FAIL: rc $rc"; fail=1; }
+[ "$(cat "$cf1")" = 1 ] && echo "  ok: ran exactly once" || { echo "  FAIL: ran $(cat "$cf1")x"; fail=1; }
+
+# 2. fails twice then succeeds (max 3) -> overall success, 3 attempts
+cf2="$(mktemp)"; tmpdirs+=("$cf2"); echo 0 > "$cf2"
+CF="$cf2" RETRY_DELAY=0 RETRY_MAX=3 "$WR" bash -c 'n=$(( $(cat "$CF") + 1 )); echo $n > "$CF"; [ "$n" -ge 3 ]'; rc=$?
+[ "$rc" = 0 ] && echo "  ok: retries then succeeds -> exit 0" || { echo "  FAIL: rc $rc"; fail=1; }
+[ "$(cat "$cf2")" = 3 ] && echo "  ok: took 3 attempts" || { echo "  FAIL: took $(cat "$cf2")"; fail=1; }
+
+# 3. always fails -> returns the command's last exit code after RETRY_MAX attempts
+cf3="$(mktemp)"; tmpdirs+=("$cf3"); echo 0 > "$cf3"
+CF="$cf3" RETRY_DELAY=0 RETRY_MAX=2 "$WR" bash -c 'echo $(( $(cat "$CF") + 1 )) > "$CF"; exit 7'; rc=$?
+[ "$rc" = 7 ] && echo "  ok: returns last exit code (7)" || { echo "  FAIL: rc $rc (want 7)"; fail=1; }
+[ "$(cat "$cf3")" = 2 ] && echo "  ok: tried RETRY_MAX (2) times" || { echo "  FAIL: tried $(cat "$cf3")x"; fail=1; }
+
+# 4. no command given -> usage error (exit 2), not a silent success
+"$WR" >/dev/null 2>&1; rc=$?
+[ "$rc" = 2 ] && echo "  ok: no-args -> exit 2 (usage)" || { echo "  FAIL: no-args rc $rc (want 2)"; fail=1; }
+
+echo "== generate-cve-report.sh =="
+CVE_FIX="${ROOT}/.github/scripts/tests/fixtures/cve"
+CVE_OUT="$(bash "${ROOT}/.github/scripts/generate-cve-report.sh" "$CVE_FIX" "2026-07-16 02:41 UTC" 2>/tmp/cve-err)"; CVE_RC=$?
+assert_eq "$CVE_RC" "0" "generate-cve-report exits 0"
+# digests table: full digest for a published image
+assert_contains "$CVE_OUT" "sha256:1f3a9c4e2b7d05a8c1e6f4b9d3072a5e8c1b6f0d4a7e2c9b5083f1d6a4c7e2b90" "full plain digest in digests table"
+assert_contains "$CVE_OUT" "sha256:8ad4c17b93e0a5d2f4681c9b0e3a7d5c2b8f1069a4e7c3b05d9f28a1c6e4b0f37" "full hardened digest in digests table"
+assert_contains "$CVE_OUT" "not published this run" "unpublished hardened shown in digests table"
+# fixed row: short PLAIN digest pointer + old->new + fixed status
+assert_contains "$CVE_OUT" "| \`1f3a9c4e2b7d\` | CVE-2024-45491 | HIGH | libexpat1 | ✅ fixed · 2.6.2-1 → 2.6.2-2+deb13u1 |" "fixed row rendered with plain short digest and version bump"
+# residual row: short HARDENED digest pointer + no fix
+assert_contains "$CVE_OUT" "| \`8ad4c17b93e0\` | CVE-2025-6020 | HIGH | libpam0g | ⚠️ residual · no fix |" "residual row rendered with hardened short digest"
+# unpublished-hardened image: residual rows use 'unpublished' pointer + unpatched status
+# hardened not produced: plain IS still published, so the row points at the PLAIN short digest
+assert_contains "$CVE_OUT" "| \`44bec0a1d2e3\` | CVE-2024-7883 | MEDIUM | libxml2 | ⚠️ unpatched · hardened not produced |" "unpublished-hardened variant lists plain CVEs as unpatched (plain digest pointer)"
+assert_contains "$CVE_OUT" "Development / rolling tags" "header notes dev exclusion"
+# Regression: one CVE id affecting two packages, fixed on one (pkga) and residual on
+# the other (pkgb) -- the fixed-set query must key on id+package (exact), not id
+# alone, or the pkga row silently vanishes from BOTH tables (see fixture
+# v5.2-multipkg-amd64: CVE-2025-9999 present in plain for pkga+pkgb, hardened only
+# still has pkgb).
+assert_contains "$CVE_OUT" "| \`dd81d549a32e\` | CVE-2025-9999 | HIGH | pkga | ✅ fixed · 1.0 → 1.1 |" "same CVE id fixed on one package (id+package keying)"
+assert_contains "$CVE_OUT" "| \`d07739baf7cd\` | CVE-2025-9999 | HIGH | pkgb | ⚠️ residual · no fix |" "same CVE id residual on a different package (id+package keying)"
+
 echo; [ "$fail" = "0" ] && echo "ALL TESTS PASSED" || echo "TESTS FAILED"
 exit "$fail"

.github/scripts/with-retry.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Run a command with retries + exponential backoff, for transient CI flakiness
+# (e.g. Trivy DB/artifact downloads 404ing from a mirror). Retries on ANY non-zero
+# exit. Config via env: RETRY_MAX (default 3), RETRY_DELAY (first backoff seconds,
+# default 10; doubles each attempt). Returns the command's last exit code if all
+# attempts fail. Deliberately omits `set -e`: the retry loop owns all failure
+# handling, so a failing attempt must fall through to the backoff, never abort.
+set -uo pipefail
+
+[ "$#" -gt 0 ] || { echo "usage: with-retry.sh <cmd> [args...]" >&2; exit 2; }
+
+max="${RETRY_MAX:-3}"
+delay="${RETRY_DELAY:-10}"
+attempt=1
+
+while true; do
+    "$@" && exit 0
+    rc=$?
+    if [ "$attempt" -ge "$max" ]; then
+        echo "::warning::command failed after ${attempt} attempt(s) (exit ${rc}): $*" >&2
+        exit "$rc"
+    fi
+    echo "attempt ${attempt}/${max} failed (exit ${rc}); retrying in ${delay}s: $*" >&2
+    sleep "$delay"
+    attempt=$((attempt + 1))
+    delay=$((delay * 2))
+done