diff --git a/hydra-gates/scripts/lib/check_contract_coverage.py b/hydra-gates/scripts/lib/check_contract_coverage.py index 0102450..a944153 100644 --- a/hydra-gates/scripts/lib/check_contract_coverage.py +++ b/hydra-gates/scripts/lib/check_contract_coverage.py @@ -58,6 +58,22 @@ GATE_NUM = 25 +# --------------------------------------------------------------------------- +# AN EXIT CODE IS A STATUS. THE COUNT GOES ON STDOUT. +# --------------------------------------------------------------------------- +# Same convention gate-19 settled on after returning its finding count as an +# exit status (.github#209): a byte cannot carry a count, and a count cannot +# carry a status. It carries a status; the number is printed. +# +# EMPTY_SCOPE exists because PASS and "I inspected nothing" used to be the same +# 0, which is why --require-full-coverage — whose whole job is to notice gates +# that did not run — could not see this one. (.github#242) +EXIT_PASS = 0 +EXIT_FAIL = 1 +EXIT_ERROR = 2 +EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip structural +EXIT_NOT_APPLICABLE = 4 # subject matter absent entirely -> runner _skip na + # A routed name: 'controller#method' (snake_case controller, camelCase method, # Settings\Foo namespaced controllers allowed). _ROUTE_NAME_RE = re.compile( @@ -346,12 +362,108 @@ def _collect(app_dir: Path, base_ref: str) -> list[dict]: return scan_new_endpoints(app_dir, changed, routes) +def _collect_from(app_dir: Path, changed: dict[str, set[int]]) -> list[dict]: + """``_collect`` with the line map supplied rather than derived from a diff. + + Lets run_gate decide the scope — diff or whole tree — instead of the scope + being hardcoded to "diff" inside the collector, which is what hid 32 + uncovered endpoints on openconnector behind a PASS (.github#242). + """ + routes_path = app_dir / "appinfo" / "routes.php" + if not routes_path.is_file(): + return [] + return scan_new_endpoints(app_dir, changed, parse_routes(routes_path)) + + +def _all_controller_lines(app_dir: Path) -> dict[str, set[int]]: + """Every line of every controller — the full-tree equivalent of a diff. + + ``scan_new_endpoints`` asks "was this method's declaration line ADDED?". + A full-tree audit answers yes for every line, which makes every registered + public endpoint a candidate exactly as it would be on the commit that first + introduced it. + """ + out: dict[str, set[int]] = {} + cdir = app_dir / "lib" / "Controller" + if not cdir.is_dir(): + return out + for cfile in cdir.rglob("*Controller.php"): + try: + n = len(cfile.read_text(encoding="utf-8").splitlines()) + except OSError: + continue + out[str(cfile.relative_to(app_dir))] = set(range(1, n + 1)) + return out + + def run_gate(app_dir: Path) -> int: - base_ref = os.environ.get("HYDRA_GATE_BASE_REF", "origin/development") - endpoints = _collect(app_dir, base_ref) + """Audit wire-contract coverage. Returns a status; the COUNT is printed. + + SCOPE IS THE CALLER'S DECISION, AND IT IS NOT DEFAULTED (.github#242) + -------------------------------------------------------------------- + This used to diff against ``HYDRA_GATE_BASE_REF`` UNCONDITIONALLY, with the + ref defaulted to ``origin/development`` even when the caller had asked for + no scoping at all. On a full-tree run the diff came back empty and the gate + printed ``PASS — no new public endpoints in diff`` having opened nothing. + + Because the narrowing happened HERE — inside the helper, below the runner's + base resolution — the runner could not tell a full-tree request had been + reduced to nothing, and because the verdict was PASS rather than a skip, + ``--require-full-coverage`` could not see it either. + + Measured on openconnector 2026-08-08: **PASS** as the runner invoked it, + **32** public endpoints with no contract test against the root commit. + """ + base_ref = os.environ.get("HYDRA_GATE_BASE_REF") + + if not (app_dir / "appinfo" / "routes.php").is_file(): + print( + f"[gate-{GATE_NUM}] contract-coverage: NOT APPLICABLE — no " + f"appinfo/routes.php, so this app exposes no routed endpoint whose " + f"wire contract could be tested." + ) + return EXIT_NOT_APPLICABLE + + if base_ref: + changed = changed_lines(base_ref, app_dir) + # The scope that matters is CONTROLLER files, not "any file". A diff of + # a hundred docs commits still opens no controller, and reporting PASS + # for it claims a wire contract was checked when none was read. + changed = { + rel: lines for rel, lines in changed.items() + if rel.startswith("lib/Controller/") and rel.endswith("Controller.php") + } + if not changed: + print( + f"[gate-{GATE_NUM}] contract-coverage: EMPTY SCOPE — " + f"diff-scoped against '{base_ref}' and NO controller file was " + f"touched, so no endpoint was inspected. Wire-contract coverage " + f"is UNVERIFIED by this run. This is not a pass. Audit the whole " + f"tree by running without HYDRA_GATE_BASE_REF, or with " + f"--scope-to-diff --base ." + ) + return EXIT_EMPTY_SCOPE + endpoints = _collect_from(app_dir, changed) + else: + all_lines = _all_controller_lines(app_dir) + if not all_lines: + print( + f"[gate-{GATE_NUM}] contract-coverage: NOT APPLICABLE — " + f"appinfo/routes.php exists but there is no " + f"lib/Controller/*Controller.php for a route to reach." + ) + return EXIT_NOT_APPLICABLE + endpoints = _collect_from(app_dir, all_lines) + if not endpoints: - print(f"[gate-{GATE_NUM}] contract-coverage: PASS — no new public endpoints in diff") - return 0 + scope_desc = ( + f"the diff against '{base_ref}'" if base_ref else "the whole tree" + ) + print( + f"[gate-{GATE_NUM}] contract-coverage: PASS — " + f"{scope_desc} contains no new public endpoint" + ) + return EXIT_PASS newman = _newman_paths(app_dir) phpunit = _phpunit_text(app_dir) findings: list[str] = [] @@ -372,12 +484,15 @@ def run_gate(app_dir: Path) -> int: f"[gate-{GATE_NUM}] contract-coverage: PASS — " f"{len(endpoints)} new endpoint(s), all covered" ) - else: - print( - f"[gate-{GATE_NUM}] contract-coverage: FAIL — " - f"{count} new public endpoint(s) without a contract test" - ) - return count + return EXIT_PASS + print( + f"[gate-{GATE_NUM}] contract-coverage: FAIL — " + f"{count} new public endpoint(s) without a contract test" + ) + # A STATUS, not the count. Returning the count meant 256 findings exited 0 + # and read as PASS — the same byte-width bug gate-19 shipped (.github#209). + # The honest number is the one printed above, and the runner reads it there. + return EXIT_FAIL def run_report(app_dir: Path) -> int: diff --git a/hydra-gates/scripts/lib/check_e2e_coverage.py b/hydra-gates/scripts/lib/check_e2e_coverage.py index bd25f5d..8d1fd73 100644 --- a/hydra-gates/scripts/lib/check_e2e_coverage.py +++ b/hydra-gates/scripts/lib/check_e2e_coverage.py @@ -1146,6 +1146,12 @@ def changed_spec_files(base_ref: str, app_dir: Path) -> set[str]: EXIT_PASS = 0 EXIT_FAIL = 1 EXIT_ERROR = 2 +# A gate that inspected NOTHING must not answer with the same byte as a gate +# that inspected everything and liked it. PASS and "empty scope" were the same +# 0, so `--require-full-coverage` — whose entire job is to notice gates that did +# not run — could not see this one. (.github#242) +EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip structural +EXIT_NOT_APPLICABLE = 4 # subject matter absent entirely -> runner _skip na # --------------------------------------------------------------------------- @@ -1205,13 +1211,73 @@ def run_report(app_dir: Path) -> int: def run_gate(app_dir: Path) -> int: - """Diff-scoped gate. Returns EXIT_PASS / EXIT_FAIL; the COUNT is printed.""" - base_ref = os.environ.get("HYDRA_GATE_BASE_REF", "origin/development") - touched = changed_spec_files(base_ref, app_dir) + """Audit @e2e traceability. Returns a status; the COUNT is printed. + + SCOPE IS THE CALLER'S DECISION, AND IT IS NOT DEFAULTED (.github#242) + -------------------------------------------------------------------- + This function used to diff against ``HYDRA_GATE_BASE_REF`` UNCONDITIONALLY, + defaulting the ref to ``origin/development`` when the caller had not asked + for diff scoping at all. So on a full-tree run — the mode a fleet audit uses + — the default ref resolved, the diff came back empty, and the gate printed + ``PASS — no spec files in diff`` over a repository it had never opened. + + Two things made that invisible. The scoping happened HERE, inside the + helper, BELOW the runner's base resolution, so the runner could not tell + that a full-tree request had been quietly narrowed to nothing. And the + verdict was ``PASS``, not a skip, so ``--require-full-coverage`` — the one + assertion built to catch gates that did not run — had nothing to catch. + + Measured on openconnector 2026-08-08: **5** findings as the runner invoked + it, **412** against the root commit. 407 uncovered scenarios behind a green + line. + + HYDRA_GATE_BASE_REF set diff-scoped (ADR-020). An empty diff is an + EMPTY SCOPE and reports as a skip, never a pass. + HYDRA_GATE_BASE_REF unset full-tree audit of every spec in the repo. + """ + # AN UNREADABLE APP DIR IS AN ERROR, NOT AN ABSENCE. "There is no + # openspec/specs here" and "I could not look" produce the same empty set, + # and reporting the second as NOT APPLICABLE would retire the gate on the + # strength of a typo in a path. Distinguish them before anything else. + if not app_dir.is_dir(): + print( + f"[gate-{GATE_NUM}] e2e-coverage: ERROR — {app_dir} is not a " + f"readable directory, so nothing was inspected. This is not an " + f"absence of specs; it is a failure to look." + ) + return EXIT_ERROR - if not touched: - print(f"[gate-{GATE_NUM}] e2e-coverage: PASS — no spec files in diff") - return EXIT_PASS + spec_root = app_dir / "openspec" / "specs" + all_specs = ( + {str(p.relative_to(app_dir)) for p in spec_root.glob("*/spec.md")} + if spec_root.is_dir() + else set() + ) + + if not all_specs: + print( + f"[gate-{GATE_NUM}] e2e-coverage: NOT APPLICABLE — no " + f"openspec/specs/*/spec.md in this repository, so there is no " + f"declared scenario for an e2e test to trace back to." + ) + return EXIT_NOT_APPLICABLE + + base_ref = os.environ.get("HYDRA_GATE_BASE_REF") + if base_ref: + touched = changed_spec_files(base_ref, app_dir) + if not touched: + print( + f"[gate-{GATE_NUM}] e2e-coverage: EMPTY SCOPE — diff-scoped " + f"against '{base_ref}' and NO spec file was touched. " + f"{len(all_specs)} spec file(s) exist here and NONE were " + f"inspected: @e2e traceability (ADR-020) is UNVERIFIED by this " + f"run. This is not a pass. Audit the whole tree by running " + f"without HYDRA_GATE_BASE_REF, or with " + f"--scope-to-diff --base ." + ) + return EXIT_EMPTY_SCOPE + else: + touched = all_specs covered_refs, dead_refs = collect_ref_status(app_dir) diff --git a/hydra-gates/scripts/lib/check_store_and_settings_surface.py b/hydra-gates/scripts/lib/check_store_and_settings_surface.py index b0def10..14ca9ba 100755 --- a/hydra-gates/scripts/lib/check_store_and_settings_surface.py +++ b/hydra-gates/scripts/lib/check_store_and_settings_surface.py @@ -23,6 +23,20 @@ import sys from pathlib import Path +# --------------------------------------------------------------------------- +# AN EXIT CODE IS A STATUS (.github#240 / #242) +# --------------------------------------------------------------------------- +# "I checked and it was fine", "there was nothing here to check", and "I was +# handed an empty scope and checked nothing" were all the same 0. The runner +# printed PASS for all three, and the log line beside the PASS literally read +# "gate skipped". Only `na` is allowed not to count against coverage; the other +# two must be visible to --require-full-coverage, so they get their own codes. +EXIT_PASS = 0 +EXIT_FAIL = 1 +EXIT_ERROR = 2 +EXIT_EMPTY_SCOPE = 3 # scope resolved, selected nothing -> runner _skip structural +EXIT_NOT_APPLICABLE = 4 # subject matter absent entirely -> runner _skip na + # ADR-077 vocabulary: the canonical glyph for each concept this gate names. STORE_ICON = "StoreOutline" TEMPLATE_ICON = "FileReplaceOutline" @@ -264,7 +278,7 @@ def main() -> int: manifest_paths = _manifest_paths(root) if not manifest_paths: print("No manifest — gate not applicable (Tier 0 app).") - return 0 + return EXIT_NOT_APPLICABLE changed = _changed_files(root, args.base) if changed is not None: @@ -274,8 +288,22 @@ def main() -> int: ] layout_changed = "src/menu-layout.json" in changed if not scoped and not layout_changed: - print("No changed manifest / menu-layout — gate skipped (ADR-020 diff scoping).") - return 0 + # THIS BRANCH USED TO `return 0`, and the runner printed PASS. The + # log line said "gate skipped" and the verdict line next to it said + # PASS — the two contradicted each other on the same screen, and the + # PASS is what every consumer counted. (.github#240) + # + # A full-tree audit was the one mode this gate could never reach: + # the runner passed --base unconditionally, so even an unscoped run + # landed here and reported a green over an unopened manifest. + print( + f"No changed manifest / menu-layout — EMPTY SCOPE (ADR-020 diff " + f"scoping against '{args.base}'). {len(manifest_paths)} manifest(s) " + f"exist here and NONE were inspected; this gate's placement rules " + f"are UNVERIFIED by this run. This is not a pass. Omit --base to " + f"audit the whole tree." + ) + return EXIT_EMPTY_SCOPE manifest_paths = scoped or manifest_paths manifests = [] @@ -298,7 +326,7 @@ def main() -> int: hard = [f for f in findings if f.startswith("FAIL")] warn = [f for f in findings if f.startswith("WARN")] print(f"\nchecked {len(manifests)} manifest(s): {len(hard)} failure(s), {len(warn)} warning(s).") - return 1 if hard else 0 + return EXIT_FAIL if hard else EXIT_PASS if __name__ == "__main__": diff --git a/hydra-gates/scripts/lib/test_check_e2e_coverage.py b/hydra-gates/scripts/lib/test_check_e2e_coverage.py index 18faee0..59a60a3 100644 --- a/hydra-gates/scripts/lib/test_check_e2e_coverage.py +++ b/hydra-gates/scripts/lib/test_check_e2e_coverage.py @@ -518,8 +518,14 @@ def _commit(self, msg: str = "commit") -> str: cwd=str(self.root), capture_output=True, text=True ).stdout.strip() - def test_pass_when_no_spec_files_in_diff(self): - # Baseline commit, then change a non-spec file + def test_no_specs_in_the_repo_at_all_is_NOT_APPLICABLE_not_a_pass(self): + """A repo with no specs has nothing to trace — say so, don't claim a pass. + + This test used to assert PASS, which is the .github#242 defect written + down as an expectation: "I inspected nothing" and "I inspected + everything and it was fine" cannot share a verdict, because + --require-full-coverage has to be able to tell them apart. + """ _write(self.root, "src/index.ts", "export const x = 1\n") base = self._commit("base") _write(self.root, "src/index.ts", "export const x = 2\n") @@ -533,8 +539,9 @@ def test_pass_when_no_spec_files_in_diff(self): finally: del os.environ["HYDRA_GATE_BASE_REF"] - self.assertEqual(rc, 0) - self.assertIn("PASS", buf.getvalue()) + self.assertEqual(rc, cec.EXIT_NOT_APPLICABLE) + self.assertIn("NOT APPLICABLE", buf.getvalue()) + self.assertNotIn("PASS", buf.getvalue()) def _gate_with_n_scenarios(self, n: int): _write(self.root, "README.md", "# app\n") @@ -584,21 +591,36 @@ def test_an_unreadable_app_dir_is_an_ERROR_not_a_pass(self): with redirect_stdout(buf): rc = cec.main(["check_e2e_coverage.py", str(self.root / "nope"), "--mode", "boom"]) - # A non-existent dir is simply empty, so this is a PASS, not a crash — - # assert the honest thing: it is a valid status, never a raw count. - self.assertIn(rc, (cec.EXIT_PASS, cec.EXIT_FAIL, cec.EXIT_ERROR)) - del buf + # A non-existent dir used to be indistinguishable from an empty one, so + # this asserted only "some valid status". It is an ERROR now (#242): + # "there are no specs here" and "I could not look" produce the same + # empty set, and reporting the second as a benign verdict would retire + # the gate on the strength of a typo in a path. + self.assertEqual(rc, cec.EXIT_ERROR) + self.assertIn("ERROR", buf.getvalue()) + self.assertNotIn("PASS", buf.getvalue()) def test_run_gate_raising_is_reported_as_ERROR(self): + # A spec must exist, or the gate answers NOT APPLICABLE before it ever + # reaches the diff helper this test is monkeypatching. + _write(self.root, "openspec/specs/s/spec.md", + "# s Spec\n## Purpose\n### Requirement: R\n#### Scenario: One\n- WHEN a\n- THEN b\n") + self._commit("spec so the diff helper is reached") original = cec.changed_spec_files cec.changed_spec_files = lambda *_a, **_k: (_ for _ in ()).throw( RuntimeError("git exploded")) + # The diff helper is only consulted when a base ref is set — an unscoped + # run audits the whole tree and never calls it (#242). Set one, or this + # test monkeypatches a function the run never reaches and passes for a + # reason that has nothing to do with what it claims to check. + os.environ["HYDRA_GATE_BASE_REF"] = "HEAD" try: buf = io.StringIO() with redirect_stdout(buf): rc = cec.main(["check_e2e_coverage.py", str(self.root)]) finally: cec.changed_spec_files = original + del os.environ["HYDRA_GATE_BASE_REF"] self.assertEqual(rc, cec.EXIT_ERROR) self.assertIn("ERROR", buf.getvalue()) self.assertNotIn("PASS", buf.getvalue()) @@ -737,9 +759,16 @@ def test_diff_scope_only_changed_spec_flagged(self): finally: del os.environ["HYDRA_GATE_BASE_REF"] - # old-spec is not in the diff → should not be flagged - self.assertEqual(rc, 0) + # THE ADR-020 INVARIANT, UNCHANGED: an untouched legacy spec is never + # flagged, so this PR is not blocked by debt it did not create. self.assertNotIn("old-spec", buf.getvalue()) + # THE #242 CHANGE: the gate opened no spec, so it reports an EMPTY + # SCOPE rather than a PASS. A skip does not fail a run — but unlike a + # PASS it is visible to --require-full-coverage, which is the whole + # point. 407 uncovered scenarios on openconnector sat behind exactly + # this PASS. + self.assertEqual(rc, cec.EXIT_EMPTY_SCOPE) + self.assertIn("EMPTY SCOPE", buf.getvalue()) # --------------------------------------------------------------------------- diff --git a/hydra-gates/scripts/lib/test_gate_a11y_markup_scope.sh b/hydra-gates/scripts/lib/test_gate_a11y_markup_scope.sh new file mode 100755 index 0000000..58c13f2 --- /dev/null +++ b/hydra-gates/scripts/lib/test_gate_a11y_markup_scope.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: EUPL-1.2 +# +# test_gate_a11y_markup_scope.sh — the accessibility gates must read the markup +# the app actually ships, not only the markup written in Vue. +# +# WHAT THIS GUARDS (.github#225) +# ------------------------------ +# Gates 31, 32, 34, 35, 36, 37, 39, 40, 42, 43, 44 and 45 enumerated +# `find src -name '*.vue'` and nothing else. An app that renders its UI from PHP +# templates had every one of those gates iterate an empty list and report PASS. +# +# The `[ -d src ]` guard did not save it: nldesign HAS a `src/`, containing only +# `manifest.json`. The directory existed, the glob matched nothing, the loop ran +# zero times, and twelve gates printed PASS. +# +# Measured 2026-08-08 on nldesign — one textbook true positive planted per gate +# into `templates/settings/`: +# +# before the fix 0 of 12 gates caught their planted true positive +# after the fix 12 of 12 +# +# and removing the plants surfaced 8 GENUINE pre-existing findings in nldesign's +# real admin template (gates 40, 43, 44) that no run had ever reported. +# +# TWO ARMS. The second is the one that keeps this honest: +# +# ARM 1 planted true positives in a PHP template ARE caught +# ARM 2 ANTI-WIDENING CONTROL — a CLEAN PHP template still PASSes, and a +# clean .vue still PASSes. A checker that flags everything is not a +# checker, and "widen the glob" is exactly the change that could turn +# these gates into noise generators. + +set -u + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_scripts="$(cd "${_here}/.." && pwd)" +_runner="${HYDRA_GATES_RUNNER_UNDER_TEST:-${_scripts}/run-hydra-gates.sh}" + +_failures=0 +_ok() { echo " ok — $1"; } +_bad() { echo " FAIL — $1"; _failures=$((_failures + 1)); } + +echo "test_gate_a11y_markup_scope.sh" + +_tmp="$(mktemp -d "${TMPDIR:-/tmp}/hydra-a11y-scope.XXXXXX")" +trap 'rm -rf "${_tmp}"' EXIT + +_mkapp() { # _mkapp — a PHP-template app whose src/ holds only a manifest + mkdir -p "$1/src" "$1/templates/settings" + printf '{"name":"fx","menu":[]}\n' > "$1/src/manifest.json" + ( + cd "$1" || exit 1 + git init -q . + git add -A + git -c user.email=t@t -c user.name=t commit -qm init + ) >/dev/null 2>&1 +} + +_run() { # _run + local logs="${_tmp}/logs.$$.${RANDOM}" + mkdir -p "${logs}" + ( + cd "$1" || exit 1 + HYDRA_GATE_LOG_DIR="${logs}" bash "${_runner}" . > "$2" 2>&1 + ) +} + +# --------------------------------------------------------------------------- +# ARM 1 — planted true positives in a PHP template are caught. +# +# One violation per gate, each the textbook example from that gate's own +# docblock. If a gate stops catching its own documented example, this goes red. +# --------------------------------------------------------------------------- +_bad_app="${_tmp}/bad" +_mkapp "${_bad_app}" +cat > "${_bad_app}/templates/settings/admin.php" <<'PHP' + +
+ + +
Click me
+ focus trap + + + + click here +
ab
+
+ + +PHP +( + cd "${_bad_app}" || exit 1 + git add -A + git -c user.email=t@t -c user.name=t commit -qm plant +) >/dev/null 2>&1 + +_bad_out="${_tmp}/bad.txt" +_run "${_bad_app}" "${_bad_out}" + +# gate -> what it should have found in the template above +_expect="31:img without alt +32:click handler on a non-semantic element +34:window.confirm in an inline script +35:empty alt on a semantically-named src +36:positive tabindex +37:aria-hidden=true on a focusable element +39:icon-only button with no accessible name +40:input with no associated label +42:non-descriptive link text +43:table with no th scope +44:semantic input with no autocomplete +45:style block with motion and no prefers-reduced-motion" + +_caught=0 +_total=0 +while IFS=: read -r _g _what; do + [ -z "${_g}" ] && continue + _total=$((_total + 1)) + if grep -qE "^\[gate-${_g}\][^:]*: FAIL" "${_bad_out}"; then + _caught=$((_caught + 1)) + else + _v=$(grep -oE "^\[gate-${_g}\] [^:]+: [A-Z]+( \([a-z]+\))?" "${_bad_out}" | head -1 | sed 's/^[^:]*: //') + _bad "gate-${_g} did not catch: ${_what} (verdict: ${_v:-none emitted})" + fi +done <<< "${_expect}" + +if [ "${_caught}" -eq "${_total}" ]; then + _ok "all ${_total} accessibility gates caught their planted true positive in a PHP template" +fi + +# --------------------------------------------------------------------------- +# ARM 2 — ANTI-WIDENING CONTROL. A clean template must still pass. +# --------------------------------------------------------------------------- +_good_app="${_tmp}/good" +_mkapp "${_good_app}" +cat > "${_good_app}/templates/settings/admin.php" <<'PHP' + +
+ Company logo + + + + Read the configuration guide + + + +
NameValue
ab
+
+ +PHP +( + cd "${_good_app}" || exit 1 + git add -A + git -c user.email=t@t -c user.name=t commit -qm clean +) >/dev/null 2>&1 + +_good_out="${_tmp}/good.txt" +_run "${_good_app}" "${_good_out}" + +_noisy="" +for _g in 31 32 34 35 36 37 39 40 42 43 44 45; do + if grep -qE "^\[gate-${_g}\][^:]*: FAIL" "${_good_out}"; then + _noisy="${_noisy} ${_g}" + fi +done +if [ -z "${_noisy}" ]; then + _ok "a CLEAN PHP template raises no accessibility finding (no false positives from the widening)" +else + _bad "clean PHP template wrongly flagged by gate(s):${_noisy} — the widened glob is manufacturing findings" +fi + +# The widening must not have cost the .vue coverage it already had. +_vue_app="${_tmp}/vue" +_mkapp "${_vue_app}" +mkdir -p "${_vue_app}/src/views" +printf '\n' \ + > "${_vue_app}/src/views/Thing.vue" +( + cd "${_vue_app}" || exit 1 + git add -A + git -c user.email=t@t -c user.name=t commit -qm vue +) >/dev/null 2>&1 +_vue_out="${_tmp}/vue.txt" +_run "${_vue_app}" "${_vue_out}" +if grep -qE '^\[gate-31\][^:]*: FAIL' "${_vue_out}"; then + _ok "a .vue violation is still caught — widening did not displace the original scope" +else + _bad "gate-31 stopped catching a .vue without alt — the widening REPLACED the Vue scope instead of adding to it" +fi + +echo +if [ "${_failures}" -eq 0 ]; then + echo "test_gate_a11y_markup_scope.sh: ALL PASS" + exit 0 +fi +echo "test_gate_a11y_markup_scope.sh: ${_failures} FAILURE(S)" +exit 1 diff --git a/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh b/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh new file mode 100755 index 0000000..eee65f2 --- /dev/null +++ b/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: EUPL-1.2 +# +# test_gate_empty_scope_never_passes.sh — a gate must never report PASS over a +# scope it did not open. +# +# WHAT THIS GUARDS (.github#242, #240) +# ------------------------------------ +# Gates 19 (e2e-coverage), 25 (contract-coverage), 62 (store-plane) and +# 63 (settings-surface) diff-scoped themselves INSIDE their own helpers, below +# the runner's base resolution, and did it UNCONDITIONALLY — the base ref was +# defaulted even when the caller had asked for no scoping at all. +# +# Two consequences, and the second is why it stayed hidden: +# +# 1. A full-tree run was silently narrowed to a diff against +# origin/development, which on a mainline checkout is empty. +# 2. The verdict for "I inspected nothing" was PASS, not a skip. So +# --require-full-coverage — the one assertion built to catch gates that did +# not run — had nothing to catch. gate-63 was the clearest case: its log +# said "gate skipped" on the line above a verdict that said PASS. +# +# Measured on openconnector 2026-08-08: +# gate-19 5 findings as the runner invoked it -> 412 over the full tree +# gate-25 PASS as the runner invoked it -> 32 over the full tree +# +# THREE ARMS, and all three are needed: +# +# ARM 1 a planted TRUE POSITIVE is still caught in full-tree mode. +# Widening a checker until it catches nothing is not a fix, so this +# arm runs FIRST and everything else is meaningless without it. +# ARM 2 an empty scope produces a visible SKIP, and --require-full-coverage +# FAILS the run (exit 98). +# ARM 3 a genuinely non-empty scope with nothing wrong still PASSes, so the +# fix has not simply turned every gate into a permanent skip. + +set -u + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_scripts="$(cd "${_here}/.." && pwd)" +_runner="${HYDRA_GATES_RUNNER_UNDER_TEST:-${_scripts}/run-hydra-gates.sh}" + +_failures=0 +_ok() { echo " ok — $1"; } +_bad() { echo " FAIL — $1"; _failures=$((_failures + 1)); } + +echo "test_gate_empty_scope_never_passes.sh" + +_tmp="$(mktemp -d "${TMPDIR:-/tmp}/hydra-emptyscope.XXXXXX")" +trap 'rm -rf "${_tmp}"' EXIT + +# --------------------------------------------------------------------------- +# A fixture that carries ONE genuine finding for gate-19 and ONE for gate-25: +# a declared scenario with no @e2e tag, and a routed #[PublicPage] method with +# no Newman/PHPUnit contract test and no @contract exclude. +# --------------------------------------------------------------------------- +_app="${_tmp}/app" +mkdir -p "${_app}/src" "${_app}/openspec/specs/thing" "${_app}/appinfo" \ + "${_app}/lib/Controller" +printf '{"name":"fx","menu":[]}\n' > "${_app}/src/manifest.json" +printf '## Purpose\n\n#### Scenario: a thing happens\n- WHEN x\n- THEN y\n' \ + > "${_app}/openspec/specs/thing/spec.md" +printf "[['name'=>'thing#index','url'=>'/api/thing','verb'=>'GET']]];\n" \ + > "${_app}/appinfo/routes.php" +cat > "${_app}/lib/Controller/ThingController.php" <<'PHP' + README.md + git add README.md + git -c user.email=t@t -c user.name=t commit -qm docs +) >/dev/null 2>&1 + +_run() { # _run [runner args...] + local out="$1"; shift + local logs="${_tmp}/logs.$$.${RANDOM}" + mkdir -p "${logs}" + ( + cd "${_app}" || exit 1 + HYDRA_GATE_LOG_DIR="${logs}" bash "${_runner}" "$@" . > "${out}" 2>&1 + ) + return $? +} + +_verdict() { grep -oE "^\[gate-$2\] [^:]+: [A-Z]+( \([a-z]+\))?" "$1" | head -1 | sed 's/^[^:]*: //'; } + +# --------------------------------------------------------------------------- +# ARM 1 — the planted true positives are still caught, full-tree. +# --------------------------------------------------------------------------- +_full="${_tmp}/full.txt" +_run "${_full}" +for _g in 19 25; do + if grep -qE "^\[gate-${_g}\][^:]*: FAIL" "${_full}"; then + _ok "gate-${_g} still catches its planted true positive over the full tree" + else + _bad "gate-${_g} did NOT catch its planted true positive — got: $(_verdict "${_full}" "${_g}")" + fi +done + +# Full-tree must actually OPEN the manifests rather than diff-scope itself to +# nothing: 62/63 are clean in this fixture, so they must PASS, not SKIP. +for _g in 62 63; do + _v="$(_verdict "${_full}" "${_g}")" + if [ "${_v}" = "PASS" ]; then + _ok "gate-${_g} audited the manifest over the full tree (PASS, not a self-inflicted skip)" + else + _bad "gate-${_g} full-tree verdict is '${_v}' — expected PASS over a clean manifest" + fi +done + +# --------------------------------------------------------------------------- +# ARM 2 — an empty scope is a visible SKIP, and it fails --require-full-coverage. +# --------------------------------------------------------------------------- +_scoped="${_tmp}/scoped.txt" +_run "${_scoped}" --scope-to-diff --base HEAD~1 --require-full-coverage +_scoped_rc=$? + +for _g in 19 25 62 63; do + _v="$(_verdict "${_scoped}" "${_g}")" + case "${_v}" in + "SKIPPED (structural)") + _ok "gate-${_g} reports SKIPPED (structural) over an empty scope" + ;; + PASS) + _bad "gate-${_g} reported PASS over a scope it never opened — this is the #242 defect" + ;; + *) + _bad "gate-${_g} empty-scope verdict is '${_v}' — expected SKIPPED (structural)" + ;; + esac +done + +if [ "${_scoped_rc}" -eq 98 ]; then + _ok "--require-full-coverage failed the run over the empty scopes (exit 98)" +else + _bad "--require-full-coverage exited ${_scoped_rc}, expected 98 — the skips are not being counted" +fi + +# The skip must carry a REASON. A bare "SKIPPED" is how a gate disappears +# quietly, which is the failure this whole accounting exists to stop. +for _g in 19 25 62 63; do + if grep -qE "^\[gate-${_g}\][^:]*: SKIPPED \(structural\) — .+UNVERIFIED" "${_scoped}"; then + _ok "gate-${_g}'s skip states what it left unverified" + else + _bad "gate-${_g}'s skip has no reason naming what went unverified" + fi +done + +# --------------------------------------------------------------------------- +# ARM 3 — a NON-empty scope with nothing wrong still passes. +# +# Without this arm, "make every empty scope a skip" could be satisfied by +# skipping unconditionally, and the suite would look fixed while checking +# nothing. The second commit here touches the manifest, so 62/63 have real work. +# --------------------------------------------------------------------------- +( + cd "${_app}" || exit 1 + printf '{"name":"fx","menu":[],"version":"1.0.1"}\n' > src/manifest.json + git add src/manifest.json + git -c user.email=t@t -c user.name=t commit -qm "chore: bump manifest" +) >/dev/null 2>&1 + +_touched="${_tmp}/touched.txt" +_run "${_touched}" --scope-to-diff --base HEAD~1 +for _g in 62 63; do + _v="$(_verdict "${_touched}" "${_g}")" + if [ "${_v}" = "PASS" ]; then + _ok "gate-${_g} PASSes when the diff genuinely contains a clean manifest" + else + _bad "gate-${_g} returned '${_v}' for a real, clean, in-scope manifest — the gate has been skipped into uselessness" + fi +done + +echo +if [ "${_failures}" -eq 0 ]; then + echo "test_gate_empty_scope_never_passes.sh: ALL PASS" + exit 0 +fi +echo "test_gate_empty_scope_never_passes.sh: ${_failures} FAILURE(S)" +exit 1 diff --git a/hydra-gates/scripts/run-hydra-gates.sh b/hydra-gates/scripts/run-hydra-gates.sh index 38263c9..c4d3fa7 100755 --- a/hydra-gates/scripts/run-hydra-gates.sh +++ b/hydra-gates/scripts/run-hydra-gates.sh @@ -861,6 +861,45 @@ _optout_text() { # than a default. A typo that silently resolved to `na` would be a lever for # making any gate's absence stop counting — which is precisely the accounting # hole this whole block exists to close, re-opened from the inside. +# _a11y_markup_files — every file in this repo that ships MARKUP A USER SEES. +# +# WHY THIS EXISTS (.github#225) +# ---------------------------- +# The whole accessibility family — gates 31, 32, 34, 35, 36, 37, 39, 40, 42, 43, +# 44, 45 — enumerated `find src -name '*.vue'` and nothing else. An app that +# renders its UI from PHP templates therefore had every one of those gates +# iterate an empty list and report PASS. +# +# Measured 2026-08-08 on nldesign, which ships one `templates/settings/admin.php` +# and a `src/` containing only `manifest.json`: one textbook true positive was +# planted per gate — an `` with no alt, a positive `tabindex`, a focusable +# element inside `aria-hidden="true"`, an icon-only `