diff --git a/hydra-gates/scripts/lib/check_contract_coverage.py b/hydra-gates/scripts/lib/check_contract_coverage.py index a944153..a0dfb64 100644 --- a/hydra-gates/scripts/lib/check_contract_coverage.py +++ b/hydra-gates/scripts/lib/check_contract_coverage.py @@ -71,7 +71,7 @@ EXIT_PASS = 0 EXIT_FAIL = 1 EXIT_ERROR = 2 -EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip structural +EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip `na` (.github#268) EXIT_NOT_APPLICABLE = 4 # subject matter absent entirely -> runner _skip na # A routed name: 'controller#method' (snake_case controller, camelCase method, diff --git a/hydra-gates/scripts/lib/check_e2e_coverage.py b/hydra-gates/scripts/lib/check_e2e_coverage.py index 8d1fd73..1bb950a 100644 --- a/hydra-gates/scripts/lib/check_e2e_coverage.py +++ b/hydra-gates/scripts/lib/check_e2e_coverage.py @@ -1150,7 +1150,7 @@ def changed_spec_files(base_ref: str, app_dir: Path) -> set[str]: # that inspected everything and liked it. PASS and "empty scope" were the same # 0, so `--require-full-coverage` — whose entire job is to notice gates that did # not run — could not see this one. (.github#242) -EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip structural +EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip `na` (.github#268) EXIT_NOT_APPLICABLE = 4 # subject matter absent entirely -> runner _skip na diff --git a/hydra-gates/scripts/lib/check_store_and_settings_surface.py b/hydra-gates/scripts/lib/check_store_and_settings_surface.py index 14ca9ba..d6f923d 100755 --- a/hydra-gates/scripts/lib/check_store_and_settings_surface.py +++ b/hydra-gates/scripts/lib/check_store_and_settings_surface.py @@ -34,7 +34,7 @@ EXIT_PASS = 0 EXIT_FAIL = 1 EXIT_ERROR = 2 -EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip structural +EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip `na` (.github#268) EXIT_NOT_APPLICABLE = 4 # subject matter absent entirely -> runner _skip na # ADR-077 vocabulary: the canonical glyph for each concept this gate names. diff --git a/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh b/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh index eee65f2..e8f0fac 100755 --- a/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh +++ b/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh @@ -24,15 +24,40 @@ # gate-19 5 findings as the runner invoked it -> 412 over the full tree # gate-25 PASS as the runner invoked it -> 32 over the full tree # -# THREE ARMS, and all three are needed: +# AND WHAT #268 CORRECTED +# ----------------------- +# #258 filed the empty-scope case as `structural`, which COUNTS AGAINST +# --require-full-coverage. So any PR that happened to touch no spec and no +# manifest exited 98 for a gate that had nothing to judge — measured as 4 runs +# across 3 repos (doriath x2, larpingapp, softwarecatalog) blocked on nothing. +# +# The category was the bug, not the skip. The runner's own definitions: +# +# na subject matter absent from this repo OR THIS DIFF. Nothing in +# the repository is missing and no change the author could make +# would put a spec file into a diff that does not touch one. +# structural the subject matter EXISTS and nothing produced the gate's +# input — a gap the repo CAN close (the axe-report case). +# +# An empty ADR-020 diff scope is the first. Gates 4/6/7/28 already called the +# identical situation `na`. What #258 bought survives the reclassification +# because it lives in the RENDERING, not the accounting: the verdict is +# `NOT APPLICABLE`, which is not `PASS`. +# +# FOUR ARMS, and all four are needed: # # ARM 1 a planted TRUE POSITIVE is still caught in full-tree mode. # Widening a checker until it catches nothing is not a fix, so this # arm runs FIRST and everything else is meaningless without it. -# ARM 2 an empty scope produces a visible SKIP, and --require-full-coverage -# FAILS the run (exit 98). +# ARM 2 an empty scope is VISIBLE and is never PASS (#242/#240), and it does +# NOT fail --require-full-coverage (#268). # ARM 3 a genuinely non-empty scope with nothing wrong still PASSes, so the # fix has not simply turned every gate into a permanent skip. +# ARM 4 ANTI-WIDENING. A genuinely `structural` gap — the same tree, the +# same flags, plus --axe-enabled and no axe report — must STILL exit +# 98. Without this arm, ARM 2 could be satisfied by neutering +# --require-full-coverage altogether, and `na` would become the hole +# that the whole coverage accounting exists to prevent. set -u @@ -93,7 +118,11 @@ _run() { # _run [runner args...] return $? } -_verdict() { grep -oE "^\[gate-$2\] [^:]+: [A-Z]+( \([a-z]+\))?" "$1" | head -1 | sed 's/^[^:]*: //'; } +# NOTE the `( [A-Z]+)*`: the verdict word is not always one token. "NOT +# APPLICABLE" parsed as "NOT" under the original single-token pattern, so every +# arm comparing against a two-word verdict failed on the string rather than on +# the behaviour it meant to test. +_verdict() { grep -oE "^\[gate-$2\] [^:]+: [A-Z]+( [A-Z]+)*( \([a-z]+\))?" "$1" | head -1 | sed 's/^[^:]*: //'; } # --------------------------------------------------------------------------- # ARM 1 — the planted true positives are still caught, full-tree. @@ -120,7 +149,8 @@ for _g in 62 63; do done # --------------------------------------------------------------------------- -# ARM 2 — an empty scope is a visible SKIP, and it fails --require-full-coverage. +# ARM 2 — an empty scope is VISIBLE and never PASS (#242/#240), and it does not +# fail --require-full-coverage (#268). # --------------------------------------------------------------------------- _scoped="${_tmp}/scoped.txt" _run "${_scoped}" --scope-to-diff --base HEAD~1 --require-full-coverage @@ -129,34 +159,86 @@ _scoped_rc=$? for _g in 19 25 62 63; do _v="$(_verdict "${_scoped}" "${_g}")" case "${_v}" in - "SKIPPED (structural)") - _ok "gate-${_g} reports SKIPPED (structural) over an empty scope" + "NOT APPLICABLE") + _ok "gate-${_g} reports NOT APPLICABLE over an empty diff scope" ;; PASS) _bad "gate-${_g} reported PASS over a scope it never opened — this is the #242 defect" ;; + "SKIPPED (structural)"|"SKIPPED (wiring)") + _bad "gate-${_g} reported '${_v}' over an empty diff scope — this is the #268 regression: an empty ADR-020 scope counts against coverage and exits 98" + ;; *) - _bad "gate-${_g} empty-scope verdict is '${_v}' — expected SKIPPED (structural)" + _bad "gate-${_g} empty-scope verdict is '${_v}' — expected NOT APPLICABLE" ;; esac done +# The exit code is the whole point of #268: this run has no findings and no +# real coverage gap, so --require-full-coverage must let it through. if [ "${_scoped_rc}" -eq 98 ]; then - _ok "--require-full-coverage failed the run over the empty scopes (exit 98)" + _bad "--require-full-coverage exited 98 over an empty diff scope — the #268 regression: a PR that touches no spec and no manifest is blocked for a gate that had nothing to judge" +elif [ "${_scoped_rc}" -eq 0 ]; then + _ok "--require-full-coverage let an empty diff scope through (exit 0)" else - _bad "--require-full-coverage exited ${_scoped_rc}, expected 98 — the skips are not being counted" + _bad "empty-scope run exited ${_scoped_rc}, expected 0 — unexpected verdict" fi -# The skip must carry a REASON. A bare "SKIPPED" is how a gate disappears -# quietly, which is the failure this whole accounting exists to stop. +# ...and it must not be counted as a coverage gap in the summary either. The +# exit code alone would still pass if the four were listed as DID NOT RUN while +# some other gate happened to hold the run open. +if grep -q 'GATES THAT DID NOT RUN' "${_scoped}"; then + _bad "the empty-scope run reported a coverage gap — expected none; DID-NOT-RUN list: $(sed -n '/GATES THAT DID NOT RUN/,$p' "${_scoped}" | grep -oE 'gate-[0-9]+' | tr '\n' ' ')" +else + _ok "the empty-scope run reports NO coverage gap at all" +fi + +# The declaration must carry a REASON naming the diff-scoping rule. A bare +# "NOT APPLICABLE" is how a gate disappears quietly, which is the failure this +# whole accounting exists to stop. for _g in 19 25 62 63; do - if grep -qE "^\[gate-${_g}\][^:]*: SKIPPED \(structural\) — .+UNVERIFIED" "${_scoped}"; then - _ok "gate-${_g}'s skip states what it left unverified" + if grep -qE "^\[gate-${_g}\][^:]*: NOT APPLICABLE — .+ADR-020" "${_scoped}"; then + _ok "gate-${_g} states WHY it was not applicable, and names ADR-020" else - _bad "gate-${_g}'s skip has no reason naming what went unverified" + _bad "gate-${_g}'s NOT APPLICABLE line has no reason naming ADR-020 diff scoping" fi done +# --------------------------------------------------------------------------- +# ARM 4 — ANTI-WIDENING. `na` must not have become a hole. +# +# ARM 2 asserts that --require-full-coverage does NOT fire over an empty diff +# scope. On its own that assertion is satisfiable by breaking +# --require-full-coverage outright, which would re-open .github#169 — the +# accounting hole this whole mechanism was built to close. +# +# So: THE SAME TREE AND THE SAME FLAGS AS ARM 2, plus --axe-enabled and no +# tests/axe/report.json. That is a GENUINELY structural gap — the input was +# expected, the repo could produce it, and it did not arrive — and it must +# still exit 98. +# +# It has to be this tree specifically. `_FAILED` is evaluated BEFORE the +# coverage branch, so any gate with a real finding pre-empts exit 98 and the +# arm would measure nothing. (Measured while writing this: run it after ARM 3's +# manifest commit and gates 22/53 fail on an unresolvable ajv, the run exits 2, +# and the assertion reads as a widening regression that is not there.) +# --------------------------------------------------------------------------- +_axe="${_tmp}/axe.txt" +_run "${_axe}" --scope-to-diff --base HEAD~1 --require-full-coverage --axe-enabled +_axe_rc=$? + +if grep -qE "^\[gate-33\][^:]*: SKIPPED \(structural\)" "${_axe}"; then + _ok "a genuinely structural gap is still categorised structural (gate-33, axe report expected and absent)" +else + _bad "gate-33 did not report a structural skip with --axe-enabled and no report — got: $(_verdict "${_axe}" 33)" +fi + +if [ "${_axe_rc}" -eq 98 ]; then + _ok "--require-full-coverage STILL fails a genuinely structural gap (exit 98) — \`na\` did not become a hole" +else + _bad "--require-full-coverage exited ${_axe_rc} over a real structural gap, expected 98 — the #268 fix has widened into .github#169" +fi + # --------------------------------------------------------------------------- # ARM 3 — a NON-empty scope with nothing wrong still passes. # @@ -182,6 +264,51 @@ for _g in 62 63; do fi done +# --------------------------------------------------------------------------- +# ARM 5 — THE INVERSE INVARIANT. A gate must not report "nothing to judge" +# when its subject matter IS sitting in the diff. +# +# ARM 3 proves a clean in-scope manifest still PASSes. That is necessary but +# not sufficient: a gate that had been neutered to always-`na` would fail ARM 3 +# loudly, but a gate that merely stopped ENFORCING would sail through it. So +# plant a REAL ADR-079 violation in the manifest the diff touches — a +# type:settings page claiming the reserved platform name — and require a FAIL. +# +# Together with ARM 2 this pins both directions: +# subject absent from the diff -> na, does not fail the run +# subject present in the diff -> a real verdict, and violations still FAIL +# --------------------------------------------------------------------------- +( + cd "${_app}" || exit 1 + printf '{"name":"fx","menu":[],"pages":[{"id":"settings","type":"settings","title":"Settings"}]}\n' \ + > src/manifest.json + git add src/manifest.json + git -c user.email=t@t -c user.name=t commit -qm "feat: a settings page claiming the reserved name" +) >/dev/null 2>&1 + +_violation="${_tmp}/violation.txt" +_run "${_violation}" --scope-to-diff --base HEAD~1 --require-full-coverage +_violation_rc=$? + +_v="$(_verdict "${_violation}" 63)" +case "${_v}" in + FAIL) + _ok "gate-63 FAILs a real ADR-079 violation sitting in the diff — the subject was judged, not declared away" + ;; + "NOT APPLICABLE") + _bad "gate-63 declared NOT APPLICABLE over a manifest THE DIFF TOUCHED and which carries a real ADR-079 violation — \`na\` is swallowing a present subject" + ;; + *) + _bad "gate-63 returned '${_v}' for a planted ADR-079 violation in an in-scope manifest — expected FAIL" + ;; +esac + +if [ "${_violation_rc}" -ne 0 ] && [ "${_violation_rc}" -ne 98 ]; then + _ok "the run exits non-zero on the planted violation (exit ${_violation_rc} = finding count, not a coverage verdict)" +else + _bad "the run exited ${_violation_rc} with a planted ADR-079 violation in scope — a finding must fail the run on its own merits" +fi + echo if [ "${_failures}" -eq 0 ]; then echo "test_gate_empty_scope_never_passes.sh: ALL PASS" diff --git a/hydra-gates/scripts/run-hydra-gates.sh b/hydra-gates/scripts/run-hydra-gates.sh index f3ef706..031bf88 100755 --- a/hydra-gates/scripts/run-hydra-gates.sh +++ b/hydra-gates/scripts/run-hydra-gates.sh @@ -150,6 +150,56 @@ fi mkdir -p "${HYDRA_GATE_LOG_DIR}" 2>/dev/null || true echo "[hydra-gates] findings logs: ${HYDRA_GATE_LOG_DIR}" +# --------------------------------------------------------------------------- +# WHICH PACKAGE PRODUCED THIS VERDICT (.github#268) +# +# The fleet consumes this package at `@main`, UNPINNED. So two runs minutes +# apart can be two different programs, and nothing in the output said so. +# +# That is not hypothetical. On 2026-08-08 doriath's `development` tip was gated +# green at 14:21:47Z; a classification change landed on main at 14:21:55Z — +# EIGHT SECONDS later — and the next PR run was red. The diff between "green" +# and "red" was the gates, not the code, and there was no way to see that from +# either log. +# +# It matters most for the STRICT-SUBSET merge rule ("the PR's failing gate +# names are a subset of the base branch's, so the PR introduced nothing new"). +# That rule compares a live measurement against a stored one. If the two were +# produced by different packages it is comparing two different programs, and +# the conclusion does not follow. +# +# This line does not FIX that — it makes it CHECKABLE. A comparison can now +# state which package produced each side, and refuse to conclude when they +# differ. The policy question (pin consumers? re-run the baseline with the +# PR's package? record the package in the baseline artifact?) is deliberately +# left open in #268; emitting the identity is the prerequisite for any of them. +# +# Resolution order, most to least trustworthy. The last resort prints UNKNOWN +# rather than nothing: a silent omission is indistinguishable from an older +# package that never had this line, and that ambiguity is the bug. +# --------------------------------------------------------------------------- +_pkg_sha="${HYDRA_GATES_PKG_SHA:-}" +_pkg_origin="caller (HYDRA_GATES_PKG_SHA)" +if [ -z "${_pkg_sha}" ]; then + # The package's own checkout — set when consumed as a git clone or submodule. + # `git -C` on the SCRIPT dir, never the app under test: resolving this + # against the repo being gated would report the APP's sha as the gates' sha, + # which is worse than reporting nothing. + _pkg_sha="$(git -C "${SCRIPT_DIR}" rev-parse HEAD 2>/dev/null || true)" + _pkg_origin="git checkout at ${SCRIPT_DIR}" +fi +if [ -z "${_pkg_sha}" ] && [ -f "${SCRIPT_DIR}/../VERSION" ]; then + _pkg_sha="$(tr -d '[:space:]' < "${SCRIPT_DIR}/../VERSION" 2>/dev/null || true)" + _pkg_origin="VERSION file" +fi +if [ -n "${_pkg_sha}" ]; then + echo "[hydra-gates] gate package: ${_pkg_sha} (${_pkg_origin})" +else + echo "[hydra-gates] gate package: UNKNOWN — this run cannot say which version of the gates produced its verdicts." + echo "[hydra-gates] Comparing it against another run (e.g. a strict-subset baseline check) compares two possibly-different programs." + echo "[hydra-gates] Set HYDRA_GATES_PKG_SHA to make the comparison sound." +fi + SCOPE_TO_DIFF=0 BASE_REF="origin/development" APP_DIR="" @@ -1136,6 +1186,43 @@ _optout_text() { # `structural` and `wiring` both count against coverage and both fail a run # started with --require-full-coverage. Only `na` does not. # +# AN EMPTY ADR-020 DIFF SCOPE IS `na`, NEVER `structural` (.github#268) +# --------------------------------------------------------------------- +# The two are easy to confuse and the difference is whether ANYTHING IN THIS +# REPOSITORY COULD HAVE MADE THE GATE RUN. +# +# structural something in this repo SHOULD have produced the input and did +# not, and the repo can be changed so that it does. An app with +# src/ that ships no axe report can ship one. A composer.json +# with no `license` field can declare one. The gap is REAL and +# it is ACTIONABLE HERE — which is exactly why it fails the run. +# +# na the input is absent from THIS DIFF. ADR-020 scoping excluded +# it, which is the entire purpose of ADR-020. Nothing in the +# repository is missing, nothing is broken, and NO CHANGE THE +# AUTHOR COULD MAKE would let this gate inspect a file the diff +# does not contain — short of manufacturing the gate's input, +# which is the false green this package exists to prevent. +# +# #258 correctly stopped gates 19/25/62/63 printing PASS over an unopened +# scope, but filed the empty-scope case as `structural`. Under +# --require-full-coverage that exited 98 on any PR that happened to touch no +# spec and no manifest: 4 runs across 3 repos blocked on nothing, purely as a +# function of which files the diff contained. Gates 4, 6, 7 and 28 already +# called the identical situation `na` ("0 lib/Controller PHP file(s) in this +# diff"), and the summary header has always read "subject matter absent from +# this repo OR THIS DIFF" — so `na` is both the correct category and the one +# the rest of this file was already using. +# +# What #258 bought is UNCHANGED by that reclassification, because it lives in +# the rendering and not in the accounting: an empty scope prints +# `NOT APPLICABLE`, which is not `PASS`. The invariant to hold when editing +# this file is therefore: +# +# * an unopened scope must never render as PASS (#242/#240 — _skip, any category) +# * an unopened scope must never fail the run (#268 — category `na`) +# * a gap the repo COULD close must still fail it (#169 — category structural/wiring) +# # The category is validated, and an unrecognised one is a HARD FAILURE rather # than a default. A typo that silently resolved to `na` would be a lever for # making any gate's absence stop counting — which is precisely the accounting @@ -2447,11 +2534,13 @@ if [ -d openspec/specs ] || [ -d tests/e2e ]; then if [ "${_e2e_fail}" -eq 0 ]; then _pass 19 "e2e-coverage" elif [ "${_e2e_fail}" -eq 3 ]; then - # EMPTY SCOPE. Specs exist; the diff selected none of them. That is - # not a pass — it is a gate that inspected nothing, and it must be - # visible to --require-full-coverage. + # EMPTY SCOPE. Specs exist; the diff selected none of them. Not a + # pass — the gate inspected nothing and says so out loud (#242). + # `na`, NOT structural (#268): nothing in this repository is + # missing and no change the author could make would put a spec + # file into a diff that does not touch one. See _skip's header. _e2e_ran=0 - _skip 19 "e2e-coverage" structural "the diff against '${BASE_REF}' touched NO spec file, so no scenario was inspected; @e2e traceability (ADR-020) is UNVERIFIED by this run. See ${_e2e_log}." + _skip 19 "e2e-coverage" na "the diff against '${BASE_REF}' touched NO spec file, so no scenario was inspected. Diff-scoped out under ADR-020, exactly as gates 4/6/7 are for the same diff — not a gap: the specs in this repo are unchanged from the base branch, so this PR introduces no scenario whose @e2e traceability could be missing. This gate runs on the next PR that touches a spec. See ${_e2e_log}." elif [ "${_e2e_fail}" -eq 4 ]; then _e2e_ran=0 _skip 19 "e2e-coverage" na "no openspec/specs/*/spec.md in this repository — there is no declared scenario for an e2e test to trace back to." @@ -2892,8 +2981,9 @@ if [ -f appinfo/routes.php ]; then if [ "${_cc_fail}" -eq 0 ]; then _pass 25 "contract-coverage" elif [ "${_cc_fail}" -eq 3 ]; then + # EMPTY SCOPE — `na`, not structural (#268). See _skip's header. _cc_ran=0 - _skip 25 "contract-coverage" structural "the diff against '${BASE_REF}' changed NO file, so no endpoint was inspected; wire-contract coverage is UNVERIFIED by this run. See ${_cc_log}." + _skip 25 "contract-coverage" na "the diff against '${BASE_REF}' touched NO lib/Controller file, so no endpoint was inspected. Diff-scoped out under ADR-020, exactly as gates 6/7 are for the same diff — not a gap: this PR exposes no new endpoint whose wire contract could be untested. This gate runs on the next PR that touches a controller. See ${_cc_log}." elif [ "${_cc_fail}" -eq 4 ]; then _cc_ran=0 _skip 25 "contract-coverage" na "no appinfo/routes.php — this app exposes no routed endpoint whose wire contract could be tested." @@ -5648,7 +5738,11 @@ set +e if [ "${_sp_rc}" -eq 0 ]; then _pass 62 "store-plane" elif [ "${_sp_rc}" -eq 3 ]; then - _skip 62 "store-plane" structural "the diff against '${BASE_REF}' touched no manifest or menu-layout, so NO manifest was inspected; ADR-080 store-plane naming/discovery is UNVERIFIED by this run. See ${_sp_log}." + # EMPTY SCOPE — `na`, not structural (#268). See _skip's header. A bugfix + # PR has no legitimate reason to edit src/manifest.json, so failing it here + # left the author only two moves: manufacture the gate's input, or switch + # --require-full-coverage off fleet-wide. Both are worse than the bug. + _skip 62 "store-plane" na "the diff against '${BASE_REF}' touched no manifest and no menu-layout, so NO manifest was inspected. Diff-scoped out under ADR-020 — not a gap: the manifests in this repo are unchanged from the base branch, so this PR introduces no store-plane naming or discovery decision (ADR-080) to judge. This gate runs on the next PR that touches a manifest. See ${_sp_log}." elif [ "${_sp_rc}" -eq 4 ]; then _skip 62 "store-plane" na "no src/manifest.json — a Tier-0 app declares no store plane for ADR-080 to constrain." else @@ -5676,7 +5770,8 @@ if [ "${_ss_rc}" -eq 0 ]; then elif [ "${_ss_rc}" -eq 3 ]; then # The log used to say "gate skipped" while the verdict beside it said PASS. # Those cannot both be true, and PASS is the one every consumer counted. - _skip 63 "settings-surface" structural "the diff against '${BASE_REF}' touched no manifest or menu-layout, so NO manifest was inspected; ADR-079 settings placement is UNVERIFIED by this run. See ${_ss_log}." + # It is not PASS any more (#242) — and it is `na`, not structural (#268). + _skip 63 "settings-surface" na "the diff against '${BASE_REF}' touched no manifest and no menu-layout, so NO manifest was inspected. Diff-scoped out under ADR-020 — not a gap: the manifests in this repo are unchanged from the base branch, so this PR introduces no settings placement (ADR-079) to judge. This gate runs on the next PR that touches a manifest. See ${_ss_log}." elif [ "${_ss_rc}" -eq 4 ]; then _skip 63 "settings-surface" na "no src/manifest.json — a Tier-0 app declares no settings surface for ADR-079 to place." else