From e06922543eea1ea275d1e12651596e3305c0fcb8 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Sat, 8 Aug 2026 15:57:04 +0200 Subject: [PATCH 1/3] fix(gates): a gate must never report PASS over a scope it did not open (#242, #240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gates 19 (e2e-coverage), 25 (contract-coverage), 62 (store-plane) and 63 (settings-surface) diff-scoped themselves INSIDE their own helpers, below the runner's base resolution, and did it UNCONDITIONALLY — the base ref was defaulted to origin/development even when the caller had asked for no scoping at all. Two consequences, and the second is why it stayed hidden: 1. A full-tree run — the mode a fleet audit uses — was silently narrowed to a diff against origin/development, which on a mainline checkout is empty. 2. The verdict for "I inspected nothing" was PASS, not a skip. So --require-full-coverage, the one assertion built to catch gates that did not run, had nothing to catch. gate-63 was the clearest case (#240): its log printed "gate skipped" on the line above a verdict that said PASS. Both cannot be true, and PASS is the one every consumer counted. Measured on openconnector 2026-08-08: gate-19 5 findings as the runner invoked it -> 412 over the full tree gate-25 PASS as the runner invoked it -> 32 over the full tree - the base ref is now the caller's decision and is never defaulted: set means diff-scoped, unset means full-tree audit - an empty scope returns a distinct status and the runner maps it to `_skip … structural`, which counts against coverage and fails --require-full-coverage - absent subject matter returns its own status and maps to `_skip … na` - gate-25 scopes on CONTROLLER files, not "any file": a docs-only diff opened no controller, and PASS there claimed a wire contract had been read when none was - gate-25 now returns a STATUS rather than the finding count, matching the convention gate-19 settled on in #209; the count is read from stdout Mutation-checked, three mutants, all killed: 1. pristine pre-fix runner — every arm red 2. the #242 defect reintroduced on gate-19 — ARM 1 red: the planted true positive stops being caught 3. ANTI-WIDENING CONTROL, checkers forced to inspect nothing — ARM 1 and ARM 3 red, so the suite cannot be satisfied by skipping everything --- .../scripts/lib/check_contract_coverage.py | 135 ++++++++++++- hydra-gates/scripts/lib/check_e2e_coverage.py | 66 +++++- .../lib/check_store_and_settings_surface.py | 36 +++- .../lib/test_gate_empty_scope_never_passes.sh | 191 ++++++++++++++++++ hydra-gates/scripts/run-hydra-gates.sh | 88 +++++++- 5 files changed, 487 insertions(+), 29 deletions(-) create mode 100755 hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh diff --git a/hydra-gates/scripts/lib/check_contract_coverage.py b/hydra-gates/scripts/lib/check_contract_coverage.py index 01024507..a9441532 100644 --- a/hydra-gates/scripts/lib/check_contract_coverage.py +++ b/hydra-gates/scripts/lib/check_contract_coverage.py @@ -58,6 +58,22 @@ GATE_NUM = 25 +# --------------------------------------------------------------------------- +# AN EXIT CODE IS A STATUS. THE COUNT GOES ON STDOUT. +# --------------------------------------------------------------------------- +# Same convention gate-19 settled on after returning its finding count as an +# exit status (.github#209): a byte cannot carry a count, and a count cannot +# carry a status. It carries a status; the number is printed. +# +# EMPTY_SCOPE exists because PASS and "I inspected nothing" used to be the same +# 0, which is why --require-full-coverage — whose whole job is to notice gates +# that did not run — could not see this one. (.github#242) +EXIT_PASS = 0 +EXIT_FAIL = 1 +EXIT_ERROR = 2 +EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip structural +EXIT_NOT_APPLICABLE = 4 # subject matter absent entirely -> runner _skip na + # A routed name: 'controller#method' (snake_case controller, camelCase method, # Settings\Foo namespaced controllers allowed). _ROUTE_NAME_RE = re.compile( @@ -346,12 +362,108 @@ def _collect(app_dir: Path, base_ref: str) -> list[dict]: return scan_new_endpoints(app_dir, changed, routes) +def _collect_from(app_dir: Path, changed: dict[str, set[int]]) -> list[dict]: + """``_collect`` with the line map supplied rather than derived from a diff. + + Lets run_gate decide the scope — diff or whole tree — instead of the scope + being hardcoded to "diff" inside the collector, which is what hid 32 + uncovered endpoints on openconnector behind a PASS (.github#242). + """ + routes_path = app_dir / "appinfo" / "routes.php" + if not routes_path.is_file(): + return [] + return scan_new_endpoints(app_dir, changed, parse_routes(routes_path)) + + +def _all_controller_lines(app_dir: Path) -> dict[str, set[int]]: + """Every line of every controller — the full-tree equivalent of a diff. + + ``scan_new_endpoints`` asks "was this method's declaration line ADDED?". + A full-tree audit answers yes for every line, which makes every registered + public endpoint a candidate exactly as it would be on the commit that first + introduced it. + """ + out: dict[str, set[int]] = {} + cdir = app_dir / "lib" / "Controller" + if not cdir.is_dir(): + return out + for cfile in cdir.rglob("*Controller.php"): + try: + n = len(cfile.read_text(encoding="utf-8").splitlines()) + except OSError: + continue + out[str(cfile.relative_to(app_dir))] = set(range(1, n + 1)) + return out + + def run_gate(app_dir: Path) -> int: - base_ref = os.environ.get("HYDRA_GATE_BASE_REF", "origin/development") - endpoints = _collect(app_dir, base_ref) + """Audit wire-contract coverage. Returns a status; the COUNT is printed. + + SCOPE IS THE CALLER'S DECISION, AND IT IS NOT DEFAULTED (.github#242) + -------------------------------------------------------------------- + This used to diff against ``HYDRA_GATE_BASE_REF`` UNCONDITIONALLY, with the + ref defaulted to ``origin/development`` even when the caller had asked for + no scoping at all. On a full-tree run the diff came back empty and the gate + printed ``PASS — no new public endpoints in diff`` having opened nothing. + + Because the narrowing happened HERE — inside the helper, below the runner's + base resolution — the runner could not tell a full-tree request had been + reduced to nothing, and because the verdict was PASS rather than a skip, + ``--require-full-coverage`` could not see it either. + + Measured on openconnector 2026-08-08: **PASS** as the runner invoked it, + **32** public endpoints with no contract test against the root commit. + """ + base_ref = os.environ.get("HYDRA_GATE_BASE_REF") + + if not (app_dir / "appinfo" / "routes.php").is_file(): + print( + f"[gate-{GATE_NUM}] contract-coverage: NOT APPLICABLE — no " + f"appinfo/routes.php, so this app exposes no routed endpoint whose " + f"wire contract could be tested." + ) + return EXIT_NOT_APPLICABLE + + if base_ref: + changed = changed_lines(base_ref, app_dir) + # The scope that matters is CONTROLLER files, not "any file". A diff of + # a hundred docs commits still opens no controller, and reporting PASS + # for it claims a wire contract was checked when none was read. + changed = { + rel: lines for rel, lines in changed.items() + if rel.startswith("lib/Controller/") and rel.endswith("Controller.php") + } + if not changed: + print( + f"[gate-{GATE_NUM}] contract-coverage: EMPTY SCOPE — " + f"diff-scoped against '{base_ref}' and NO controller file was " + f"touched, so no endpoint was inspected. Wire-contract coverage " + f"is UNVERIFIED by this run. This is not a pass. Audit the whole " + f"tree by running without HYDRA_GATE_BASE_REF, or with " + f"--scope-to-diff --base ." + ) + return EXIT_EMPTY_SCOPE + endpoints = _collect_from(app_dir, changed) + else: + all_lines = _all_controller_lines(app_dir) + if not all_lines: + print( + f"[gate-{GATE_NUM}] contract-coverage: NOT APPLICABLE — " + f"appinfo/routes.php exists but there is no " + f"lib/Controller/*Controller.php for a route to reach." + ) + return EXIT_NOT_APPLICABLE + endpoints = _collect_from(app_dir, all_lines) + if not endpoints: - print(f"[gate-{GATE_NUM}] contract-coverage: PASS — no new public endpoints in diff") - return 0 + scope_desc = ( + f"the diff against '{base_ref}'" if base_ref else "the whole tree" + ) + print( + f"[gate-{GATE_NUM}] contract-coverage: PASS — " + f"{scope_desc} contains no new public endpoint" + ) + return EXIT_PASS newman = _newman_paths(app_dir) phpunit = _phpunit_text(app_dir) findings: list[str] = [] @@ -372,12 +484,15 @@ def run_gate(app_dir: Path) -> int: f"[gate-{GATE_NUM}] contract-coverage: PASS — " f"{len(endpoints)} new endpoint(s), all covered" ) - else: - print( - f"[gate-{GATE_NUM}] contract-coverage: FAIL — " - f"{count} new public endpoint(s) without a contract test" - ) - return count + return EXIT_PASS + print( + f"[gate-{GATE_NUM}] contract-coverage: FAIL — " + f"{count} new public endpoint(s) without a contract test" + ) + # A STATUS, not the count. Returning the count meant 256 findings exited 0 + # and read as PASS — the same byte-width bug gate-19 shipped (.github#209). + # The honest number is the one printed above, and the runner reads it there. + return EXIT_FAIL def run_report(app_dir: Path) -> int: diff --git a/hydra-gates/scripts/lib/check_e2e_coverage.py b/hydra-gates/scripts/lib/check_e2e_coverage.py index bd25f5d0..e4d36b60 100644 --- a/hydra-gates/scripts/lib/check_e2e_coverage.py +++ b/hydra-gates/scripts/lib/check_e2e_coverage.py @@ -1146,6 +1146,12 @@ def changed_spec_files(base_ref: str, app_dir: Path) -> set[str]: EXIT_PASS = 0 EXIT_FAIL = 1 EXIT_ERROR = 2 +# A gate that inspected NOTHING must not answer with the same byte as a gate +# that inspected everything and liked it. PASS and "empty scope" were the same +# 0, so `--require-full-coverage` — whose entire job is to notice gates that did +# not run — could not see this one. (.github#242) +EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip structural +EXIT_NOT_APPLICABLE = 4 # subject matter absent entirely -> runner _skip na # --------------------------------------------------------------------------- @@ -1205,13 +1211,61 @@ def run_report(app_dir: Path) -> int: def run_gate(app_dir: Path) -> int: - """Diff-scoped gate. Returns EXIT_PASS / EXIT_FAIL; the COUNT is printed.""" - base_ref = os.environ.get("HYDRA_GATE_BASE_REF", "origin/development") - touched = changed_spec_files(base_ref, app_dir) + """Audit @e2e traceability. Returns a status; the COUNT is printed. + + SCOPE IS THE CALLER'S DECISION, AND IT IS NOT DEFAULTED (.github#242) + -------------------------------------------------------------------- + This function used to diff against ``HYDRA_GATE_BASE_REF`` UNCONDITIONALLY, + defaulting the ref to ``origin/development`` when the caller had not asked + for diff scoping at all. So on a full-tree run — the mode a fleet audit uses + — the default ref resolved, the diff came back empty, and the gate printed + ``PASS — no spec files in diff`` over a repository it had never opened. + + Two things made that invisible. The scoping happened HERE, inside the + helper, BELOW the runner's base resolution, so the runner could not tell + that a full-tree request had been quietly narrowed to nothing. And the + verdict was ``PASS``, not a skip, so ``--require-full-coverage`` — the one + assertion built to catch gates that did not run — had nothing to catch. + + Measured on openconnector 2026-08-08: **5** findings as the runner invoked + it, **412** against the root commit. 407 uncovered scenarios behind a green + line. + + HYDRA_GATE_BASE_REF set diff-scoped (ADR-020). An empty diff is an + EMPTY SCOPE and reports as a skip, never a pass. + HYDRA_GATE_BASE_REF unset full-tree audit of every spec in the repo. + """ + spec_root = app_dir / "openspec" / "specs" + all_specs = ( + {str(p.relative_to(app_dir)) for p in spec_root.glob("*/spec.md")} + if spec_root.is_dir() + else set() + ) - if not touched: - print(f"[gate-{GATE_NUM}] e2e-coverage: PASS — no spec files in diff") - return EXIT_PASS + if not all_specs: + print( + f"[gate-{GATE_NUM}] e2e-coverage: NOT APPLICABLE — no " + f"openspec/specs/*/spec.md in this repository, so there is no " + f"declared scenario for an e2e test to trace back to." + ) + return EXIT_NOT_APPLICABLE + + base_ref = os.environ.get("HYDRA_GATE_BASE_REF") + if base_ref: + touched = changed_spec_files(base_ref, app_dir) + if not touched: + print( + f"[gate-{GATE_NUM}] e2e-coverage: EMPTY SCOPE — diff-scoped " + f"against '{base_ref}' and NO spec file was touched. " + f"{len(all_specs)} spec file(s) exist here and NONE were " + f"inspected: @e2e traceability (ADR-020) is UNVERIFIED by this " + f"run. This is not a pass. Audit the whole tree by running " + f"without HYDRA_GATE_BASE_REF, or with " + f"--scope-to-diff --base ." + ) + return EXIT_EMPTY_SCOPE + else: + touched = all_specs covered_refs, dead_refs = collect_ref_status(app_dir) diff --git a/hydra-gates/scripts/lib/check_store_and_settings_surface.py b/hydra-gates/scripts/lib/check_store_and_settings_surface.py index b0def100..14ca9ba2 100755 --- a/hydra-gates/scripts/lib/check_store_and_settings_surface.py +++ b/hydra-gates/scripts/lib/check_store_and_settings_surface.py @@ -23,6 +23,20 @@ import sys from pathlib import Path +# --------------------------------------------------------------------------- +# AN EXIT CODE IS A STATUS (.github#240 / #242) +# --------------------------------------------------------------------------- +# "I checked and it was fine", "there was nothing here to check", and "I was +# handed an empty scope and checked nothing" were all the same 0. The runner +# printed PASS for all three, and the log line beside the PASS literally read +# "gate skipped". Only `na` is allowed not to count against coverage; the other +# two must be visible to --require-full-coverage, so they get their own codes. +EXIT_PASS = 0 +EXIT_FAIL = 1 +EXIT_ERROR = 2 +EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip structural +EXIT_NOT_APPLICABLE = 4 # subject matter absent entirely -> runner _skip na + # ADR-077 vocabulary: the canonical glyph for each concept this gate names. STORE_ICON = "StoreOutline" TEMPLATE_ICON = "FileReplaceOutline" @@ -264,7 +278,7 @@ def main() -> int: manifest_paths = _manifest_paths(root) if not manifest_paths: print("No manifest — gate not applicable (Tier 0 app).") - return 0 + return EXIT_NOT_APPLICABLE changed = _changed_files(root, args.base) if changed is not None: @@ -274,8 +288,22 @@ def main() -> int: ] layout_changed = "src/menu-layout.json" in changed if not scoped and not layout_changed: - print("No changed manifest / menu-layout — gate skipped (ADR-020 diff scoping).") - return 0 + # THIS BRANCH USED TO `return 0`, and the runner printed PASS. The + # log line said "gate skipped" and the verdict line next to it said + # PASS — the two contradicted each other on the same screen, and the + # PASS is what every consumer counted. (.github#240) + # + # A full-tree audit was the one mode this gate could never reach: + # the runner passed --base unconditionally, so even an unscoped run + # landed here and reported a green over an unopened manifest. + print( + f"No changed manifest / menu-layout — EMPTY SCOPE (ADR-020 diff " + f"scoping against '{args.base}'). {len(manifest_paths)} manifest(s) " + f"exist here and NONE were inspected; this gate's placement rules " + f"are UNVERIFIED by this run. This is not a pass. Omit --base to " + f"audit the whole tree." + ) + return EXIT_EMPTY_SCOPE manifest_paths = scoped or manifest_paths manifests = [] @@ -298,7 +326,7 @@ def main() -> int: hard = [f for f in findings if f.startswith("FAIL")] warn = [f for f in findings if f.startswith("WARN")] print(f"\nchecked {len(manifests)} manifest(s): {len(hard)} failure(s), {len(warn)} warning(s).") - return 1 if hard else 0 + return EXIT_FAIL if hard else EXIT_PASS if __name__ == "__main__": diff --git a/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh b/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh new file mode 100755 index 00000000..eee65f24 --- /dev/null +++ b/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: EUPL-1.2 +# +# test_gate_empty_scope_never_passes.sh — a gate must never report PASS over a +# scope it did not open. +# +# WHAT THIS GUARDS (.github#242, #240) +# ------------------------------------ +# Gates 19 (e2e-coverage), 25 (contract-coverage), 62 (store-plane) and +# 63 (settings-surface) diff-scoped themselves INSIDE their own helpers, below +# the runner's base resolution, and did it UNCONDITIONALLY — the base ref was +# defaulted even when the caller had asked for no scoping at all. +# +# Two consequences, and the second is why it stayed hidden: +# +# 1. A full-tree run was silently narrowed to a diff against +# origin/development, which on a mainline checkout is empty. +# 2. The verdict for "I inspected nothing" was PASS, not a skip. So +# --require-full-coverage — the one assertion built to catch gates that did +# not run — had nothing to catch. gate-63 was the clearest case: its log +# said "gate skipped" on the line above a verdict that said PASS. +# +# Measured on openconnector 2026-08-08: +# gate-19 5 findings as the runner invoked it -> 412 over the full tree +# gate-25 PASS as the runner invoked it -> 32 over the full tree +# +# THREE ARMS, and all three are needed: +# +# ARM 1 a planted TRUE POSITIVE is still caught in full-tree mode. +# Widening a checker until it catches nothing is not a fix, so this +# arm runs FIRST and everything else is meaningless without it. +# ARM 2 an empty scope produces a visible SKIP, and --require-full-coverage +# FAILS the run (exit 98). +# ARM 3 a genuinely non-empty scope with nothing wrong still PASSes, so the +# fix has not simply turned every gate into a permanent skip. + +set -u + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_scripts="$(cd "${_here}/.." && pwd)" +_runner="${HYDRA_GATES_RUNNER_UNDER_TEST:-${_scripts}/run-hydra-gates.sh}" + +_failures=0 +_ok() { echo " ok — $1"; } +_bad() { echo " FAIL — $1"; _failures=$((_failures + 1)); } + +echo "test_gate_empty_scope_never_passes.sh" + +_tmp="$(mktemp -d "${TMPDIR:-/tmp}/hydra-emptyscope.XXXXXX")" +trap 'rm -rf "${_tmp}"' EXIT + +# --------------------------------------------------------------------------- +# A fixture that carries ONE genuine finding for gate-19 and ONE for gate-25: +# a declared scenario with no @e2e tag, and a routed #[PublicPage] method with +# no Newman/PHPUnit contract test and no @contract exclude. +# --------------------------------------------------------------------------- +_app="${_tmp}/app" +mkdir -p "${_app}/src" "${_app}/openspec/specs/thing" "${_app}/appinfo" \ + "${_app}/lib/Controller" +printf '{"name":"fx","menu":[]}\n' > "${_app}/src/manifest.json" +printf '## Purpose\n\n#### Scenario: a thing happens\n- WHEN x\n- THEN y\n' \ + > "${_app}/openspec/specs/thing/spec.md" +printf "[['name'=>'thing#index','url'=>'/api/thing','verb'=>'GET']]];\n" \ + > "${_app}/appinfo/routes.php" +cat > "${_app}/lib/Controller/ThingController.php" <<'PHP' + README.md + git add README.md + git -c user.email=t@t -c user.name=t commit -qm docs +) >/dev/null 2>&1 + +_run() { # _run [runner args...] + local out="$1"; shift + local logs="${_tmp}/logs.$$.${RANDOM}" + mkdir -p "${logs}" + ( + cd "${_app}" || exit 1 + HYDRA_GATE_LOG_DIR="${logs}" bash "${_runner}" "$@" . > "${out}" 2>&1 + ) + return $? +} + +_verdict() { grep -oE "^\[gate-$2\] [^:]+: [A-Z]+( \([a-z]+\))?" "$1" | head -1 | sed 's/^[^:]*: //'; } + +# --------------------------------------------------------------------------- +# ARM 1 — the planted true positives are still caught, full-tree. +# --------------------------------------------------------------------------- +_full="${_tmp}/full.txt" +_run "${_full}" +for _g in 19 25; do + if grep -qE "^\[gate-${_g}\][^:]*: FAIL" "${_full}"; then + _ok "gate-${_g} still catches its planted true positive over the full tree" + else + _bad "gate-${_g} did NOT catch its planted true positive — got: $(_verdict "${_full}" "${_g}")" + fi +done + +# Full-tree must actually OPEN the manifests rather than diff-scope itself to +# nothing: 62/63 are clean in this fixture, so they must PASS, not SKIP. +for _g in 62 63; do + _v="$(_verdict "${_full}" "${_g}")" + if [ "${_v}" = "PASS" ]; then + _ok "gate-${_g} audited the manifest over the full tree (PASS, not a self-inflicted skip)" + else + _bad "gate-${_g} full-tree verdict is '${_v}' — expected PASS over a clean manifest" + fi +done + +# --------------------------------------------------------------------------- +# ARM 2 — an empty scope is a visible SKIP, and it fails --require-full-coverage. +# --------------------------------------------------------------------------- +_scoped="${_tmp}/scoped.txt" +_run "${_scoped}" --scope-to-diff --base HEAD~1 --require-full-coverage +_scoped_rc=$? + +for _g in 19 25 62 63; do + _v="$(_verdict "${_scoped}" "${_g}")" + case "${_v}" in + "SKIPPED (structural)") + _ok "gate-${_g} reports SKIPPED (structural) over an empty scope" + ;; + PASS) + _bad "gate-${_g} reported PASS over a scope it never opened — this is the #242 defect" + ;; + *) + _bad "gate-${_g} empty-scope verdict is '${_v}' — expected SKIPPED (structural)" + ;; + esac +done + +if [ "${_scoped_rc}" -eq 98 ]; then + _ok "--require-full-coverage failed the run over the empty scopes (exit 98)" +else + _bad "--require-full-coverage exited ${_scoped_rc}, expected 98 — the skips are not being counted" +fi + +# The skip must carry a REASON. A bare "SKIPPED" is how a gate disappears +# quietly, which is the failure this whole accounting exists to stop. +for _g in 19 25 62 63; do + if grep -qE "^\[gate-${_g}\][^:]*: SKIPPED \(structural\) — .+UNVERIFIED" "${_scoped}"; then + _ok "gate-${_g}'s skip states what it left unverified" + else + _bad "gate-${_g}'s skip has no reason naming what went unverified" + fi +done + +# --------------------------------------------------------------------------- +# ARM 3 — a NON-empty scope with nothing wrong still passes. +# +# Without this arm, "make every empty scope a skip" could be satisfied by +# skipping unconditionally, and the suite would look fixed while checking +# nothing. The second commit here touches the manifest, so 62/63 have real work. +# --------------------------------------------------------------------------- +( + cd "${_app}" || exit 1 + printf '{"name":"fx","menu":[],"version":"1.0.1"}\n' > src/manifest.json + git add src/manifest.json + git -c user.email=t@t -c user.name=t commit -qm "chore: bump manifest" +) >/dev/null 2>&1 + +_touched="${_tmp}/touched.txt" +_run "${_touched}" --scope-to-diff --base HEAD~1 +for _g in 62 63; do + _v="$(_verdict "${_touched}" "${_g}")" + if [ "${_v}" = "PASS" ]; then + _ok "gate-${_g} PASSes when the diff genuinely contains a clean manifest" + else + _bad "gate-${_g} returned '${_v}' for a real, clean, in-scope manifest — the gate has been skipped into uselessness" + fi +done + +echo +if [ "${_failures}" -eq 0 ]; then + echo "test_gate_empty_scope_never_passes.sh: ALL PASS" + exit 0 +fi +echo "test_gate_empty_scope_never_passes.sh: ${_failures} FAILURE(S)" +exit 1 diff --git a/hydra-gates/scripts/run-hydra-gates.sh b/hydra-gates/scripts/run-hydra-gates.sh index 38263c9e..25e5f8de 100755 --- a/hydra-gates/scripts/run-hydra-gates.sh +++ b/hydra-gates/scripts/run-hydra-gates.sh @@ -2060,10 +2060,20 @@ if [ -d openspec/specs ] || [ -d tests/e2e ]; then # Capture the exit code directly — avoids the grep -c bug where grep # exits 1 on zero matches, causing "|| echo 0" to append a second "0", # leaving _e2e_fail="0\n0" which fails the -eq integer comparison. + # SCOPE ONLY WHEN THE CALLER ASKED FOR IT (#242). BASE_REF was passed + # unconditionally, so an UNSCOPED run — the mode a fleet audit uses — + # was silently narrowed to the diff against origin/development, came + # back empty, and the helper printed PASS over a repo it never opened. + # Measured on openconnector: 5 findings scoped, 412 over the full tree. set +e - HYDRA_GATE_BASE_REF="${BASE_REF}" \ + if [ "${SCOPE_TO_DIFF}" = "1" ]; then + HYDRA_GATE_BASE_REF="${BASE_REF}" \ + python3 "${_e2e_lib_dir}/check_e2e_coverage.py" . \ + >> "${_e2e_log}" 2>&1 + else python3 "${_e2e_lib_dir}/check_e2e_coverage.py" . \ - >> "${_e2e_log}" 2>&1 + >> "${_e2e_log}" 2>&1 + fi _e2e_fail=$? set +e else @@ -2081,6 +2091,15 @@ if [ -d openspec/specs ] || [ -d tests/e2e ]; then [ -z "${_e2e_count}" ] && _e2e_count="an unreported number of" if [ "${_e2e_fail}" -eq 0 ]; then _pass 19 "e2e-coverage" + elif [ "${_e2e_fail}" -eq 3 ]; then + # EMPTY SCOPE. Specs exist; the diff selected none of them. That is + # not a pass — it is a gate that inspected nothing, and it must be + # visible to --require-full-coverage. + _e2e_ran=0 + _skip 19 "e2e-coverage" structural "the diff against '${BASE_REF}' touched NO spec file, so no scenario was inspected; @e2e traceability (ADR-020) is UNVERIFIED by this run. See ${_e2e_log}." + elif [ "${_e2e_fail}" -eq 4 ]; then + _e2e_ran=0 + _skip 19 "e2e-coverage" na "no openspec/specs/*/spec.md in this repository — there is no declared scenario for an e2e test to trace back to." elif [ "${_e2e_fail}" -ge 2 ]; then # The helper fell over. It inspected nothing, so it has no verdict # to give — say so instead of reporting a fail count it never @@ -2484,12 +2503,26 @@ if [ -f appinfo/routes.php ]; then _cc_lib_dir="${SCRIPT_DIR}/lib" fi if [ -f "${_cc_lib_dir}/check_contract_coverage.py" ]; then - # The helper exits with the uncovered-endpoint count (0 = PASS). Capture - # the exit code directly — avoids the grep -c double-zero bug. + # The helper exits with a STATUS: 0 pass, 1 fail, 2 error, 3 empty + # scope, 4 not applicable. It used to exit with the uncovered-endpoint + # COUNT, so the count below is read from stdout and never from the byte. + # stderr is folded into the log so a traceback is visible rather than + # discarded. + # + # SCOPE ONLY WHEN THE CALLER ASKED FOR IT (#242). BASE_REF was passed + # unconditionally, so an unscoped run was silently narrowed to the diff + # against origin/development, came back empty, and the helper printed + # PASS having opened nothing. Measured on openconnector: PASS scoped, + # 32 uncovered public endpoints over the full tree. set +e - HYDRA_GATE_BASE_REF="${BASE_REF}" \ + if [ "${SCOPE_TO_DIFF}" = "1" ]; then + HYDRA_GATE_BASE_REF="${BASE_REF}" \ + python3 "${_cc_lib_dir}/check_contract_coverage.py" . \ + >> "${_cc_log}" 2>&1 + else python3 "${_cc_lib_dir}/check_contract_coverage.py" . \ - >> "${_cc_log}" 2>/dev/null + >> "${_cc_log}" 2>&1 + fi _cc_fail=$? set +e else @@ -2497,10 +2530,23 @@ if [ -f appinfo/routes.php ]; then _skip 25 "contract-coverage" wiring "check_contract_coverage.py not found at ${_cc_lib_dir} — appinfo/routes.php is present but NO endpoint was inspected; wire-contract coverage of newly-exposed endpoints is UNVERIFIED by this run." fi if [ "${_cc_ran}" -eq 1 ]; then + # THE COUNT COMES FROM STDOUT, not from the byte. + _cc_count=$(grep -oE 'FAIL — [0-9]+ new public endpoint' "${_cc_log}" 2>/dev/null \ + | tail -1 | grep -oE '[0-9]+' || true) + [ -z "${_cc_count}" ] && _cc_count="an unreported number of" if [ "${_cc_fail}" -eq 0 ]; then _pass 25 "contract-coverage" + elif [ "${_cc_fail}" -eq 3 ]; then + _cc_ran=0 + _skip 25 "contract-coverage" structural "the diff against '${BASE_REF}' changed NO file, so no endpoint was inspected; wire-contract coverage is UNVERIFIED by this run. See ${_cc_log}." + elif [ "${_cc_fail}" -eq 4 ]; then + _cc_ran=0 + _skip 25 "contract-coverage" na "no appinfo/routes.php — this app exposes no routed endpoint whose wire contract could be tested." + elif [ "${_cc_fail}" -ge 2 ]; then + _cc_ran=0 + _skip 25 "contract-coverage" wiring "check_contract_coverage.py exited ${_cc_fail} (error) — no endpoint verdict was produced; wire-contract coverage is UNVERIFIED by this run. See ${_cc_log}." else - _fail 25 "contract-coverage" "${_cc_fail} new public endpoint(s) missing a contract test — see ${_cc_log}" + _fail 25 "contract-coverage" "${_cc_count} new public endpoint(s) missing a contract test — see ${_cc_log}" fi fi fi @@ -4877,12 +4923,25 @@ fi # --------------------------------------------------------------------------- _sp_log=${HYDRA_GATE_LOG_DIR}/hydra-gate-store-plane.log : > "${_sp_log}" +# `--base` ONLY WHEN THE CALLER ASKED TO SCOPE (#240 / #242). It was passed +# unconditionally, so even an unscoped run diff-scoped itself, found no changed +# manifest, and the helper returned 0 — printed as PASS beside a log line +# reading "gate skipped". A full-tree audit was the one mode this gate could +# never reach. set +e -python3 "${SCRIPT_DIR}/lib/check_store_and_settings_surface.py" . --gate store --base "${BASE_REF}" > "${_sp_log}" 2>&1 +if [ "${SCOPE_TO_DIFF}" = "1" ]; then + python3 "${SCRIPT_DIR}/lib/check_store_and_settings_surface.py" . --gate store --base "${BASE_REF}" > "${_sp_log}" 2>&1 +else + python3 "${SCRIPT_DIR}/lib/check_store_and_settings_surface.py" . --gate store > "${_sp_log}" 2>&1 +fi _sp_rc=$? set +e if [ "${_sp_rc}" -eq 0 ]; then _pass 62 "store-plane" +elif [ "${_sp_rc}" -eq 3 ]; then + _skip 62 "store-plane" structural "the diff against '${BASE_REF}' touched no manifest or menu-layout, so NO manifest was inspected; ADR-080 store-plane naming/discovery is UNVERIFIED by this run. See ${_sp_log}." +elif [ "${_sp_rc}" -eq 4 ]; then + _skip 62 "store-plane" na "no src/manifest.json — a Tier-0 app declares no store plane for ADR-080 to constrain." else _sp_n=$(_count '^FAIL' "${_sp_log}") [ "${_sp_n}" -eq 0 ] && _sp_n=1 @@ -4894,12 +4953,23 @@ fi # --------------------------------------------------------------------------- _ss_log=${HYDRA_GATE_LOG_DIR}/hydra-gate-settings-surface.log : > "${_ss_log}" +# `--base` ONLY WHEN THE CALLER ASKED TO SCOPE — see gate 62 above (#240). set +e -python3 "${SCRIPT_DIR}/lib/check_store_and_settings_surface.py" . --gate settings --base "${BASE_REF}" > "${_ss_log}" 2>&1 +if [ "${SCOPE_TO_DIFF}" = "1" ]; then + python3 "${SCRIPT_DIR}/lib/check_store_and_settings_surface.py" . --gate settings --base "${BASE_REF}" > "${_ss_log}" 2>&1 +else + python3 "${SCRIPT_DIR}/lib/check_store_and_settings_surface.py" . --gate settings > "${_ss_log}" 2>&1 +fi _ss_rc=$? set +e if [ "${_ss_rc}" -eq 0 ]; then _pass 63 "settings-surface" +elif [ "${_ss_rc}" -eq 3 ]; then + # The log used to say "gate skipped" while the verdict beside it said PASS. + # Those cannot both be true, and PASS is the one every consumer counted. + _skip 63 "settings-surface" structural "the diff against '${BASE_REF}' touched no manifest or menu-layout, so NO manifest was inspected; ADR-079 settings placement is UNVERIFIED by this run. See ${_ss_log}." +elif [ "${_ss_rc}" -eq 4 ]; then + _skip 63 "settings-surface" na "no src/manifest.json — a Tier-0 app declares no settings surface for ADR-079 to place." else _ss_n=$(_count '^FAIL' "${_ss_log}") [ "${_ss_n}" -eq 0 ] && _ss_n=1 From c719d7cb4facf86934628898d7b14ac2b4bb71f4 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Sat, 8 Aug 2026 16:09:11 +0200 Subject: [PATCH 2/3] fix(gate-19): an unreadable app dir is an ERROR, not an absence (#242) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the empty-scope work, found by test_check_e2e_coverage.py. Adding the NOT APPLICABLE verdict introduced a regression of exactly the kind this issue is about: `openspec/specs` missing and the app dir being unreadable produce the SAME empty set, and the new code reported both as NOT APPLICABLE. That would retire the gate on the strength of a typo in a path. A missing directory is now an ERROR — a failure to look, not an absence of specs. Three tests updated, and two of them encoded the defect as an expectation: - test_pass_when_no_spec_files_in_diff asserted PASS for a repo with no specs. Renamed and now asserts NOT APPLICABLE: "I inspected nothing" and "I inspected everything and it was fine" cannot share a verdict. - test_diff_scope_only_changed_spec_flagged asserted PASS when the diff touched no spec. It now asserts EMPTY SCOPE, and STILL asserts the ADR-020 invariant it was written for — an untouched legacy spec is never flagged. - test_run_gate_raising_is_reported_as_ERROR monkeypatched changed_spec_files, which an unscoped run no longer calls. It now sets a base ref, so it exercises the path it claims to. Without this it would have passed for a reason unrelated to what it checks. All 105 gate-19 helper tests green; all 33 discovered helper suites green. --- hydra-gates/scripts/lib/check_e2e_coverage.py | 12 +++++ .../scripts/lib/test_check_e2e_coverage.py | 49 +++++++++++++++---- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/hydra-gates/scripts/lib/check_e2e_coverage.py b/hydra-gates/scripts/lib/check_e2e_coverage.py index e4d36b60..8d1fd73a 100644 --- a/hydra-gates/scripts/lib/check_e2e_coverage.py +++ b/hydra-gates/scripts/lib/check_e2e_coverage.py @@ -1235,6 +1235,18 @@ def run_gate(app_dir: Path) -> int: EMPTY SCOPE and reports as a skip, never a pass. HYDRA_GATE_BASE_REF unset full-tree audit of every spec in the repo. """ + # AN UNREADABLE APP DIR IS AN ERROR, NOT AN ABSENCE. "There is no + # openspec/specs here" and "I could not look" produce the same empty set, + # and reporting the second as NOT APPLICABLE would retire the gate on the + # strength of a typo in a path. Distinguish them before anything else. + if not app_dir.is_dir(): + print( + f"[gate-{GATE_NUM}] e2e-coverage: ERROR — {app_dir} is not a " + f"readable directory, so nothing was inspected. This is not an " + f"absence of specs; it is a failure to look." + ) + return EXIT_ERROR + spec_root = app_dir / "openspec" / "specs" all_specs = ( {str(p.relative_to(app_dir)) for p in spec_root.glob("*/spec.md")} diff --git a/hydra-gates/scripts/lib/test_check_e2e_coverage.py b/hydra-gates/scripts/lib/test_check_e2e_coverage.py index 18faee02..59a60a34 100644 --- a/hydra-gates/scripts/lib/test_check_e2e_coverage.py +++ b/hydra-gates/scripts/lib/test_check_e2e_coverage.py @@ -518,8 +518,14 @@ def _commit(self, msg: str = "commit") -> str: cwd=str(self.root), capture_output=True, text=True ).stdout.strip() - def test_pass_when_no_spec_files_in_diff(self): - # Baseline commit, then change a non-spec file + def test_no_specs_in_the_repo_at_all_is_NOT_APPLICABLE_not_a_pass(self): + """A repo with no specs has nothing to trace — say so, don't claim a pass. + + This test used to assert PASS, which is the .github#242 defect written + down as an expectation: "I inspected nothing" and "I inspected + everything and it was fine" cannot share a verdict, because + --require-full-coverage has to be able to tell them apart. + """ _write(self.root, "src/index.ts", "export const x = 1\n") base = self._commit("base") _write(self.root, "src/index.ts", "export const x = 2\n") @@ -533,8 +539,9 @@ def test_pass_when_no_spec_files_in_diff(self): finally: del os.environ["HYDRA_GATE_BASE_REF"] - self.assertEqual(rc, 0) - self.assertIn("PASS", buf.getvalue()) + self.assertEqual(rc, cec.EXIT_NOT_APPLICABLE) + self.assertIn("NOT APPLICABLE", buf.getvalue()) + self.assertNotIn("PASS", buf.getvalue()) def _gate_with_n_scenarios(self, n: int): _write(self.root, "README.md", "# app\n") @@ -584,21 +591,36 @@ def test_an_unreadable_app_dir_is_an_ERROR_not_a_pass(self): with redirect_stdout(buf): rc = cec.main(["check_e2e_coverage.py", str(self.root / "nope"), "--mode", "boom"]) - # A non-existent dir is simply empty, so this is a PASS, not a crash — - # assert the honest thing: it is a valid status, never a raw count. - self.assertIn(rc, (cec.EXIT_PASS, cec.EXIT_FAIL, cec.EXIT_ERROR)) - del buf + # A non-existent dir used to be indistinguishable from an empty one, so + # this asserted only "some valid status". It is an ERROR now (#242): + # "there are no specs here" and "I could not look" produce the same + # empty set, and reporting the second as a benign verdict would retire + # the gate on the strength of a typo in a path. + self.assertEqual(rc, cec.EXIT_ERROR) + self.assertIn("ERROR", buf.getvalue()) + self.assertNotIn("PASS", buf.getvalue()) def test_run_gate_raising_is_reported_as_ERROR(self): + # A spec must exist, or the gate answers NOT APPLICABLE before it ever + # reaches the diff helper this test is monkeypatching. + _write(self.root, "openspec/specs/s/spec.md", + "# s Spec\n## Purpose\n### Requirement: R\n#### Scenario: One\n- WHEN a\n- THEN b\n") + self._commit("spec so the diff helper is reached") original = cec.changed_spec_files cec.changed_spec_files = lambda *_a, **_k: (_ for _ in ()).throw( RuntimeError("git exploded")) + # The diff helper is only consulted when a base ref is set — an unscoped + # run audits the whole tree and never calls it (#242). Set one, or this + # test monkeypatches a function the run never reaches and passes for a + # reason that has nothing to do with what it claims to check. + os.environ["HYDRA_GATE_BASE_REF"] = "HEAD" try: buf = io.StringIO() with redirect_stdout(buf): rc = cec.main(["check_e2e_coverage.py", str(self.root)]) finally: cec.changed_spec_files = original + del os.environ["HYDRA_GATE_BASE_REF"] self.assertEqual(rc, cec.EXIT_ERROR) self.assertIn("ERROR", buf.getvalue()) self.assertNotIn("PASS", buf.getvalue()) @@ -737,9 +759,16 @@ def test_diff_scope_only_changed_spec_flagged(self): finally: del os.environ["HYDRA_GATE_BASE_REF"] - # old-spec is not in the diff → should not be flagged - self.assertEqual(rc, 0) + # THE ADR-020 INVARIANT, UNCHANGED: an untouched legacy spec is never + # flagged, so this PR is not blocked by debt it did not create. self.assertNotIn("old-spec", buf.getvalue()) + # THE #242 CHANGE: the gate opened no spec, so it reports an EMPTY + # SCOPE rather than a PASS. A skip does not fail a run — but unlike a + # PASS it is visible to --require-full-coverage, which is the whole + # point. 407 uncovered scenarios on openconnector sat behind exactly + # this PASS. + self.assertEqual(rc, cec.EXIT_EMPTY_SCOPE) + self.assertIn("EMPTY SCOPE", buf.getvalue()) # --------------------------------------------------------------------------- From 3327cc5a179f93ee01c1f68543cebff4dc89d490 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Sat, 8 Aug 2026 16:05:17 +0200 Subject: [PATCH 3/3] fix(gates): the a11y family read only Vue, so a PHP-template app got a green over nothing (#225) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gates 31, 32, 34, 35, 36, 37, 39, 40, 42, 43, 44 and 45 enumerated `find src -name '*.vue'` and nothing else. An app that renders its UI from PHP templates had every one of those gates iterate an empty list and report PASS. The `[ -d src ]` guard did not save it. nldesign HAS a src/ — containing only manifest.json. The directory existed, the glob matched nothing, the loop ran zero times, and twelve gates printed PASS. Measured on nldesign, one textbook true positive planted per gate into templates/settings/ — an with no alt, a positive tabindex, a focusable element inside aria-hidden="true", an icon-only