diff --git a/scripts/ci/test-verify-predicates.sh b/scripts/ci/test-verify-predicates.sh index 782d90b..66620d5 100755 --- a/scripts/ci/test-verify-predicates.sh +++ b/scripts/ci/test-verify-predicates.sh @@ -78,5 +78,132 @@ assert pass verify_softens_itself 'npm run typecheck --workspaces --if-present' assert fail verify_softens_itself 'pnpm lint && pnpm typecheck && pnpm test' 'a clean verify is not flagged' assert fail verify_softens_itself '' 'an empty verify is not flagged here — absence is the audit job' +# assert2 +assert2() { + local want="$1" pred="$2" a="$3" b="$4" desc="$5" + if "$pred" "$a" "$b"; then got=pass; else got=fail; fi + [ "$got" = "$want" ] && ok "$desc" || no "$desc (expected $want, got $got)" +} + +VERIFY_NPM='npm run lint && npm run typecheck && npm run test' + +# aoz-housing's real shape: the same three scripts, split across parallel jobs +# for speed, none of them softened. This is the case that proved the +# string-match rule wrong. +AOZ_WF=' - name: Lint + run: npm run lint + - name: Type check + run: npm run typecheck + - name: Run unit tests + run: npm test -- --ci --coverage + - name: Build + run: npm run build' + +# Same split, but one gate quietly absent. +AOZ_MISSING_TYPECHECK=' - name: Lint + run: npm run lint + - name: Run unit tests + run: npm test -- --ci --coverage' + +echo "verify_gate_scripts" +got=$(verify_gate_scripts "$VERIFY_NPM" | tr '\n' ' ') +[ "$got" = "lint test typecheck " ] \ + && ok "decomposes verify into its named gates" \ + || no "decomposes verify into its named gates (got '$got')" +got=$(verify_gate_scripts 'eslint . && tsc --noEmit && jest') +[ -z "$got" ] \ + && ok "a verify that shells out directly decomposes to nothing" \ + || no "a verify that shells out directly decomposes to nothing (got '$got')" + +echo "ci_runs_verify_gates" +assert2 pass ci_runs_verify_gates "$AOZ_WF" "$VERIFY_NPM" \ + 'aoz-housing: gates run individually, unsoftened → satisfied' +assert2 fail ci_runs_verify_gates "$AOZ_MISSING_TYPECHECK" "$VERIFY_NPM" \ + 'a gate in verify that CI never runs → NOT satisfied' +assert2 fail ci_runs_verify_gates "$HANDCOPIED" 'npm run lint && npm run test' \ + 'botsmann: hand-copied AND --if-present on every step → still caught' +assert2 fail ci_runs_verify_gates "$AOZ_WF" 'eslint . && tsc --noEmit' \ + 'an undecomposable verify is unproven, not waved through' +assert2 fail ci_runs_verify_gates '' "$VERIFY_NPM" \ + 'no workflows at all → NOT satisfied' +assert2 pass ci_runs_verify_gates 'run: pnpm lint +run: pnpm typecheck +run: pnpm test' 'pnpm lint && pnpm typecheck && pnpm test' \ + 'pnpm implicit-run spelling works on both sides' + +# --- gh_get: the three states ------------------------------------------------ +# This is the test that would have caught the 2026-08-16 miscount. `gh` is +# stubbed so the FAILURE path is reachable without a real outage — which is +# exactly why the bug survived: nothing could exercise it. + +GH_GET_BACKOFF=0 # do not actually sleep through the retries in tests + +echo "gh_get" + +gh() { printf 'ok-body'; return 0; } +out=$(gh_get 'any/path'); rc=$? +[ "$rc" = 0 ] && [ "$out" = "ok-body" ] \ + && ok "success returns 0 and the body" \ + || no "success returns 0 and the body (rc=$rc out='$out')" + +gh() { echo 'gh: Not Found (HTTP 404)' >&2; return 1; } +gh_get 'missing/path' >/dev/null; rc=$? +[ "$rc" = 2 ] \ + && ok "a real 404 is ABSENT (2), distinct from a failure" \ + || no "a real 404 is ABSENT (2) (rc=$rc)" + +gh() { echo 'gh: HTTP 403 rate limit exceeded' >&2; return 1; } +gh_get 'blocked/path' >/dev/null; rc=$? +[ "$rc" = 1 ] \ + && ok "a 403 is COULD-NOT-LOOK (1), never mistaken for absence" \ + || no "a 403 is COULD-NOT-LOOK (1) (rc=$rc)" + +gh() { echo 'gh: HTTP 502 Bad Gateway' >&2; return 1; } +gh_get 'flaky/path' >/dev/null; rc=$? +[ "$rc" = 1 ] \ + && ok "a 5xx is COULD-NOT-LOOK (1), not absence" \ + || no "a 5xx is COULD-NOT-LOOK (1) (rc=$rc)" + +# Transient then success: the retry must actually rescue the call, otherwise +# every blip still costs a repo its verdict. +ATTEMPTS_FILE=$(mktemp) +echo 0 > "$ATTEMPTS_FILE" +gh() { + local n; n=$(cat "$ATTEMPTS_FILE"); n=$((n + 1)); echo "$n" > "$ATTEMPTS_FILE" + if [ "$n" -lt 2 ]; then echo 'gh: HTTP 502' >&2; return 1; fi + printf 'recovered'; return 0 +} +out=$(gh_get 'flaky/path'); rc=$? +[ "$rc" = 0 ] && [ "$out" = "recovered" ] \ + && ok "a transient failure is retried and recovers" \ + || no "a transient failure is retried and recovers (rc=$rc out='$out')" + +# A 404 must NOT burn retries — it is an answer, and retrying it would triple +# the cost of every genuinely-absent file across the fleet. +echo 0 > "$ATTEMPTS_FILE" +gh() { + local n; n=$(cat "$ATTEMPTS_FILE"); n=$((n + 1)); echo "$n" > "$ATTEMPTS_FILE" + echo 'gh: Not Found (HTTP 404)' >&2; return 1 +} +gh_get 'missing/path' >/dev/null +[ "$(cat "$ATTEMPTS_FILE")" = 1 ] \ + && ok "a 404 is not retried (costs one call, not three)" \ + || no "a 404 is not retried (took $(cat "$ATTEMPTS_FILE") calls)" + +# An exhausted rate limit is not retried either: seconds of backoff cannot +# outlive an hour-long window, and retrying burns calls when they're scarcest. +# It is still COULD-NOT-LOOK (1) — a transport fact, never absence. +echo 0 > "$ATTEMPTS_FILE" +gh() { + local n; n=$(cat "$ATTEMPTS_FILE"); n=$((n + 1)); echo "$n" > "$ATTEMPTS_FILE" + echo 'gh: HTTP 403 API rate limit exceeded for user' >&2; return 1 +} +gh_get 'starved/path' >/dev/null; rc=$? +[ "$rc" = 1 ] && [ "$(cat "$ATTEMPTS_FILE")" = 1 ] \ + && ok "a rate-limited call fails fast as COULD-NOT-LOOK (1 call, rc=1)" \ + || no "a rate-limited call fails fast (rc=$rc, took $(cat "$ATTEMPTS_FILE") calls)" +rm -f "$ATTEMPTS_FILE" +unset -f gh + printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" [ "$FAIL" -eq 0 ] diff --git a/scripts/ci/verify-floor-audit.sh b/scripts/ci/verify-floor-audit.sh index 95245ba..e16bb81 100755 --- a/scripts/ci/verify-floor-audit.sh +++ b/scripts/ci/verify-floor-audit.sh @@ -48,6 +48,33 @@ LIMIT="${GH_LIMIT:-100}" WARN_ONLY=0 [ "${1:-}" = "--warn-only" ] && WARN_ONLY=1 +# --- Fetching is not knowing. ----------------------------------------------- +# +# Every remote read here used to be `gh api ... 2>/dev/null`, and the empty +# output that a failure produces was then read as a FACT — "no package.json", +# "no workflows". A transient 403/5xx is not a fact. Measured 2026-08-16, one +# sweep produced FOUR wrong verdicts from that conflation: +# +# vitareba, aoz-housing — live JS apps, reported as "not a JS repo" and +# dropped from the floor entirely +# ai-forms, fleetcrown — reported "CI never runs verify" while line 19 and +# line 52 of their ci.yml do exactly that +# +# The tell was arithmetic: two runs an hour apart inspected 24 and 22 repos +# with no repo created or deleted between them. A check whose output moves +# while the thing it measures holds still is not measuring it. +# +# So the failure mode is now a THIRD state, never silence: +# 0 = fetched 2 = genuinely absent (404) 1 = could not look +# +# This is the same discipline the OPAQUE column already applies to unreadable +# verify scripts, applied to the transport instead of the content. +# +# `gh_get` itself lives in verify-predicates.sh so it can be tested against a +# stubbed `gh` — the bug it fixes was invisible precisely because nothing could +# exercise the failure path without a real outage. +trap 'rm -f "$GH_ERR"' EXIT + # Declared before any branch that could skip the assignment — under `set -u` a # later reference would otherwise kill the whole run instead of one repo. ok_list="" @@ -59,6 +86,7 @@ discarded_list="" opaque_list="" forks_list="" uncalled_list="" +decomposed_list="" softened_list="" total=0 @@ -141,11 +169,18 @@ while IFS=$'\t' read -r name branch is_fork; do # Resolve package.json ON THE REPO'S OWN DEFAULT BRANCH. Asking for ?ref=main # on a master repo 404s, which means "wrong ref", not "no package.json" — # that mistake previously produced a confidently wrong fleet survey. - pkg=$(gh api "repos/$OWNER/$name/contents/package.json?ref=$branch" \ - --jq '.content' 2>/dev/null | tr -d '\n' | base64 -d 2>/dev/null) + pkg_raw=$(gh_get "repos/$OWNER/$name/contents/package.json?ref=$branch") + case $? in + 0) pkg=$(printf '%s' "$pkg_raw" | jq -r '.content // ""' 2>/dev/null \ + | tr -d '\n' | base64 -d 2>/dev/null) ;; + 2) skipped_list="${skipped_list} $name (no package.json on $branch)\n" + continue ;; + *) unreadable_list="${unreadable_list} $name — could not read package.json on $branch after 3 tries (API error, NOT absence)\n" + continue ;; + esac if [ -z "$pkg" ]; then - skipped_list="${skipped_list} $name (no package.json on $branch)\n" + unreadable_list="${unreadable_list} $name — package.json on $branch fetched but decoded empty\n" continue fi @@ -163,6 +198,8 @@ while IFS=$'\t' read -r name branch is_fork; do absent="" # gates the repo has no script for at all opaque_gates="" # gates we cannot see EITHER WAY, because verify delegates # to a script file this audit does not execute or parse + sub_unreadable="" # sub-manifests the API would not hand over — a transport + # failure, which must not be read as "the gate isn't there" # Expand `npm run X` / `pnpm X` inside verify two levels deep, so a gate # counts whether it is reached via a named script or run directly. Matching @@ -201,8 +238,16 @@ while IFS=$'\t' read -r name branch is_fork; do sub_dir=$(printf '%s' "$pair" | awk -F'|' '{print $2}') sub_script=$(printf '%s' "$pair" | awk -F'|' '{print $NF}') [ -n "$sub_dir" ] && [ -n "$sub_script" ] || continue - sub_pkg=$(gh api "repos/$OWNER/$name/contents/$sub_dir/package.json?ref=$branch" \ - --jq '.content' 2>/dev/null | tr -d '\n' | base64 -d 2>/dev/null) + # A failed read here would make the sub-package's gate look ABSENT, i.e. a + # "needs real work" verdict earned by a network blip. Retry, then record + # that we could not look rather than letting silence mean absence. + sub_raw=$(gh_get "repos/$OWNER/$name/contents/$sub_dir/package.json?ref=$branch") + case $? in + 0) sub_pkg=$(printf '%s' "$sub_raw" | jq -r '.content // ""' 2>/dev/null \ + | tr -d '\n' | base64 -d 2>/dev/null) ;; + 2) continue ;; # genuinely no sub-manifest + *) sub_unreadable="$sub_unreadable $sub_dir/package.json"; continue ;; + esac [ -n "$sub_pkg" ] || continue sub_scripts=$(printf '%s' "$sub_pkg" | jq -r '.scripts // {}' 2>/dev/null) [ -n "$sub_scripts" ] && [ "$sub_scripts" != null ] || continue @@ -230,7 +275,9 @@ $(printf '%s' "$sub_scripts" | jq -r 'to_entries[] | "\(.key) \(.value)"')" # the confident-absence bug this script just stopped committing. A truncated # tree is reported as unknown rather than zero, for the same reason. test_files=unknown - tree=$(gh api "repos/$OWNER/$name/git/trees/$branch?recursive=1" 2>/dev/null) + # Already fails SAFE (unknown, never zero), but route it through the retrying + # fetch anyway — an unknown here silently disables the no-test-files rule. + tree=$(gh_get "repos/$OWNER/$name/git/trees/$branch?recursive=1") || tree="" if [ -n "$tree" ]; then if [ "$(printf '%s' "$tree" | jq -r '.truncated // false')" = true ]; then test_files=unknown # too big to enumerate; do not guess @@ -352,12 +399,29 @@ $(printf '%s' "$sub_scripts" | jq -r 'to_entries[] | "\(.key) \(.value)"')" # and not a per-push hook. ci_calls_verify=no ci_softened_in="" - - for wf in $(gh api "repos/$OWNER/$name/contents/.github/workflows?ref=$branch" \ - --jq '.[] | select(.name | test("\\.ya?ml$")) | .name' 2>/dev/null); do - wf_body=$(gh api "repos/$OWNER/$name/contents/.github/workflows/$wf?ref=$branch" \ - --jq '.content' 2>/dev/null | tr -d '\n' | base64 -d 2>/dev/null) - [ -n "$wf_body" ] || continue + all_wf="" + # Set the moment any workflow read fails. Every wiring verdict below is then + # withheld: an unread workflow is the one that might have contained the call + # we are about to report missing. This is what accused ai-forms of never + # running `verify` when its ci.yml line 19 is `npm run verify`. + wiring_unknown="" + + wf_index=$(gh_get "repos/$OWNER/$name/contents/.github/workflows?ref=$branch") + case $? in + 0) wf_names=$(printf '%s' "$wf_index" \ + | jq -r '.[]? | select(.name | test("\\.ya?ml$")) | .name' 2>/dev/null) ;; + 2) wf_names="" ;; # genuinely no workflows directory + *) wf_names=""; wiring_unknown="could not list .github/workflows" ;; + esac + + for wf in $wf_names; do + wf_raw=$(gh_get "repos/$OWNER/$name/contents/.github/workflows/$wf?ref=$branch") + case $? in + 0) wf_body=$(printf '%s' "$wf_raw" | jq -r '.content // ""' 2>/dev/null \ + | tr -d '\n' | base64 -d 2>/dev/null) ;; + *) wiring_unknown="could not read $wf"; continue ;; + esac + [ -n "$wf_body" ] || { wiring_unknown="$wf decoded empty"; continue; } hits=$(printf '%s\n' "$wf_body" | scan_discarded_gates "$wf") while IFS= read -r hit; do [ -n "$hit" ] || continue @@ -369,12 +433,31 @@ $(printf '%s' "$sub_scripts" | jq -r 'to_entries[] | "\(.key) \(.value)"')" ci_calls_verify=yes ci_verify_softened "$wf_body" && ci_softened_in="$wf" fi + all_wf="${all_wf} +${wf_body}" done # Orthogonal to the content audit above: a repo can be AT FLOOR on what # `verify` contains and still have nothing on the branch that runs it. + # + # Calling `npm run verify` is the direct way to satisfy this. It is not the + # only way: a repo may run each constituent script by name, unsoftened, to + # fan the gates into parallel jobs. That satisfies the actual contract — every + # gate runs and every one can still fail — so it is reported separately rather + # than counted as a violation. Demanding the literal string was asking + # aoz-housing to serialize the slowest pipeline in the fleet to please a + # regex. if [ -n "$verify" ] && [ "$ci_calls_verify" = no ]; then - uncalled_list="${uncalled_list} $name — CI never runs \`verify\`\n" + if [ -n "$wiring_unknown" ]; then + # Withheld, not decided. The unread file is exactly the one that could + # have contained the call, so "not found" here would be a claim about a + # place we never looked. + unreadable_list="${unreadable_list} $name — wiring verdict withheld: $wiring_unknown\n" + elif ci_runs_verify_gates "$all_wf" "$verify"; then + decomposed_list="${decomposed_list} $name — CI runs each gate by name instead of \`verify\`\n" + else + uncalled_list="${uncalled_list} $name — CI never runs \`verify\`\n" + fi fi if [ -n "$ci_softened_in" ]; then softened_list="${softened_list} $name — $ci_softened_in runs verify with --if-present\n" @@ -383,7 +466,11 @@ $(printf '%s' "$sub_scripts" | jq -r 'to_entries[] | "\(.key) \(.value)"')" softened_list="${softened_list} $name — \`verify\` softens itself: $verify\n" fi - if [ -z "$verify" ]; then + if [ -n "$sub_unreadable" ] && [ -n "$absent" ]; then + # The gate looks missing, but a sub-manifest that could have defined it is + # exactly what we failed to fetch. Withheld rather than charged. + opaque_list="${opaque_list} $name — gate verdict withheld; could not read:$sub_unreadable\n" + elif [ -z "$verify" ]; then missing_list="${missing_list} $name — no \`verify\` script at all\n" elif [ -n "$absent" ]; then # Not fixable by editing verify: the repo has no such script. Adding a @@ -439,6 +526,16 @@ printf ' anything on the branch actually runs it. A repo can pass one and\n' printf ' fail the other — botsmann did.)\n' printf '%b' "${uncalled_list:- (none)\n}" +echo +printf '≡ DECOMPOSED — CI runs every gate by name, unsoftened, but never the\n' +printf ' word `verify`. NOT a violation: the contract is that each gate runs\n' +printf ' and can still fail, and it does. aoz-housing splits lint+typecheck,\n' +printf ' unit tests and build into parallel jobs on purpose — collapsing that\n' +printf ' into one verify step would serialize the fleet-slowest pipeline.\n' +printf ' Residual risk, stated: a gate ADDED to verify later will not reach\n' +printf ' CI by itself. That is exactly what this check keeps watching for.\n' +printf '%b' "${decomposed_list:- (none)\n}" + echo printf '⊙ SOFTENED — verify is invoked, or written, so that it cannot fail\n' printf '%b' "${softened_list:- (none)\n}" diff --git a/scripts/ci/verify-predicates.sh b/scripts/ci/verify-predicates.sh index fbceac7..ad967d7 100755 --- a/scripts/ci/verify-predicates.sh +++ b/scripts/ci/verify-predicates.sh @@ -17,6 +17,51 @@ # design (it reads every repo's own default branch), and a rule that can only be # exercised by a live API call is a rule nobody re-tests after changing it. +# --- Transport: a failed fetch is not a finding ----------------------------- +# +# The audit's remote reads were all `gh api ... 2>/dev/null`, and the empty +# string a failure yields was then read as a FACT. On 2026-08-16 that produced +# four wrong verdicts in a single sweep: vitareba and aoz-housing (live JS apps) +# reported as "not a JS repo", ai-forms and fleetcrown reported as never running +# `verify` when their ci.yml does so on lines 19 and 52. The tell was +# arithmetic — two runs an hour apart inspected 24 and 22 repos with no repo +# created or destroyed between them. +# +# Three states, never silence: +# 0 = fetched stdout carries the body +# 2 = genuinely absent the API said 404, which is an ANSWER +# 1 = could not look transport failed after retries; caller must WITHHOLD +# +# It lives here rather than in the audit so the failure path can be exercised +# against a stubbed `gh`. That is the whole lesson: this bug survived because +# nothing could reach its failure path without a real outage. +GH_ERR="${GH_ERR:-$(mktemp)}" + +gh_get() { + local path="$1" attempt=1 out + while [ "$attempt" -le "${GH_GET_TRIES:-3}" ]; do + if out=$(gh api "$path" 2>"$GH_ERR"); then + printf '%s' "$out" + return 0 + fi + # A 404 is an ANSWER — the resource is not there. Do not retry it, and do + # not let it share an exit code with "the API refused to talk to us". + if grep -qiE 'not found|HTTP 404' "$GH_ERR"; then + return 2 + fi + # An exhausted rate limit is also an answer — about the transport, not the + # repo. No backoff measured in seconds outlives a window measured in + # hours; retrying triples the burn exactly when calls are scarcest (a + # 29-repo sweep spent 87 calls rediscovering the same fact, 2026-08-20). + if grep -qi 'rate limit' "$GH_ERR"; then + return 1 + fi + sleep "${GH_GET_BACKOFF:-$((attempt * 2))}" + attempt=$((attempt + 1)) + done + return 1 +} + # Does any workflow actually invoke the verify SSOT? # # Accepts every package manager AND the implicit-run spellings (`pnpm verify`, @@ -51,3 +96,67 @@ verify_softens_itself() { *) return 1 ;; esac } + +# --- Calling verify is one way to satisfy the contract. It is not the only one. +# +# `ci_invokes_verify` matches a STRING (`npm run verify`). The property anyone +# actually cares about is different and weaker: +# +# every gate `verify` composes also runs in CI, unsoftened. +# +# Calling `npm run verify` satisfies that. So does calling each constituent +# script by name — which aoz-housing does deliberately, to fan lint+typecheck, +# unit tests and build into parallel jobs with their own coverage artifact. +# Collapsing that into one `npm run verify` step would serialize the slowest +# pipeline in the fleet and delete the artifact, i.e. the string-match rule was +# asking a conforming repo to get worse to satisfy a proxy. +# +# botsmann is still caught, because its hand-copy put `--if-present` on every +# step: a renamed script became a silent pass. That is the difference these +# predicates encode — hand-copying is fine, hand-copying with a soft landing is +# not. + +# Which named scripts does `verify` compose? +# +# Only npm-script invocations count. A verify that shells out directly +# (`eslint . && tsc --noEmit`) yields nothing, and the caller must then fall +# back to demanding a literal verify call — we cannot prove a bare binary in a +# workflow is the same gate. +verify_gate_scripts() { + printf '%s\n' "$1" \ + | grep -oE '(npm|pnpm|yarn|bun)([[:space:]]+run)?[[:space:]]+[A-Za-z0-9:._-]+' \ + | sed -E 's/^(npm|pnpm|yarn|bun)([[:space:]]+run)?[[:space:]]+//' \ + | grep -vE '^(run|ci|install|i|add|exec|x|dlx|verify)$' \ + | sort -u +} + +# Does CI run this one script, in a way that can still fail? +# +# "Unsoftened" is the whole point: among the lines that invoke the script, at +# least one must lack `--if-present`. A repo may legitimately soften an optional +# step elsewhere; it may not soften every invocation of a floor gate. +ci_runs_script() { + local escaped + escaped=$(printf '%s' "$2" | sed 's/[.[\*^$]/\\&/g') + printf '%s\n' "$1" \ + | grep -E "(npm|pnpm|yarn|bun)([[:space:]]+run)?[[:space:]]+${escaped}([[:space:]]|\$)" \ + | grep -qv -- '--if-present' +} + +# Does CI run every gate that `verify` composes? +# +# Returns false when `verify` cannot be decomposed, so an undecidable case is +# reported as unproven rather than waved through — the same "could not look" vs +# "is not there" split the OPAQUE column exists for. +ci_runs_verify_gates() { + local gates gate + gates=$(verify_gate_scripts "$2") + [ -n "$gates" ] || return 1 + while IFS= read -r gate; do + [ -n "$gate" ] || continue + ci_runs_script "$1" "$gate" || return 1 + done < every gate `verify` composes also runs in CI, **unsoftened**. + +Calling `npm run verify` satisfies that. So does calling each script by name — +which `aoz-housing` does deliberately, fanning lint+typecheck, unit tests and +build into parallel jobs each with its own artifact. It was reported +`⊗ UNCALLED` from this audit's first run onward while in fact running all three +gates on every PR. **The rule was asking a conforming repo to serialize the +slowest pipeline in the fleet to satisfy a regex.** + +botsmann is still caught, because the distinction is not "hand-copied" but +"hand-copied with a soft landing": every one of its steps carried +`--if-present`, so a rename passed silently. aoz-housing's carry none, so a +rename fails CI exactly as hard as it fails `verify`. + +The residual risk in the decomposed shape is real and is why it stays reported +rather than being dropped: **a gate added to `verify` later does not reach CI by +itself.** The check now watches for precisely that — it decomposes `verify` and +demands each part appear — instead of watching for a word. + +That is the second false positive from this audit's own rules in one day, after +the no-test-files rule flagged `ivy-portal`. Both had the same shape: **the rule +encoded the first example it was written from, not the property that example +illustrated.** botsmann's hand-copy was softened, so "hand-copied" got treated as +the defect; `jest` with no files can only fail, so "no test files" got treated as +the defect. Write the rule against the property, then find a conforming repo that +does it differently and check the rule stays quiet. + The rules live in `scripts/ci/verify-predicates.sh` rather than inline, so they can be tested against fixtures without reaching GitHub — the audit is remote-only by design, and a rule that can only be exercised by a live API call is a rule @@ -240,6 +273,53 @@ repo's CI) proves each rule bites **and** that conforming shapes are not flagged Both directions matter: a checker that cries wolf gets ignored, which is the same end state as no checker, reached more expensively. +#### The audit's own worst bug: a failed fetch reported as a finding + +Every remote read was `gh api … 2>/dev/null`, and the empty string a failure +yields was then read as a **fact**. One sweep on 2026-08-16 produced **four +wrong verdicts** from that single conflation: + +| Repo | Reported | Truth | +|---|---|---| +| `vitareba` | "no package.json — not a JS repo" | live Next.js app, 1,339-byte package.json | +| `aoz-housing` | "no package.json — not a JS repo" | live Next.js app, 2,011-byte package.json | +| `ai-forms` | "CI never runs `verify`" | `ci.yml` line 19 is `npm run verify` | +| `fleetcrown` | "CI never runs `verify`" | `ci.yml` line 52 is `npm run verify` | + +Two of them were **silently dropped from the floor entirely** — not flagged, +not counted, just absent from the report. An audit that quietly stops auditing +a repo is the exact failure this file exists to prevent, committed by the file's +own enforcement script. + +**The tell was arithmetic, not intuition.** Two runs an hour apart inspected +**24** and **22** repos, with no repo created or deleted between them. A check +whose output moves while the thing it measures holds still is not measuring it. +If your audit reports a count, diff the count between runs — that is the cheapest +non-determinism detector there is. + +**The fix is a third state.** Fetches now return `0 = fetched`, `2 = genuinely +absent (404)`, `1 = could not look`, and every caller **withholds** its verdict +on `1` instead of charging the repo. A 404 is an answer and is not retried; a +403/5xx is retried with backoff and, if it persists, reported as unreadable. +An exhausted rate limit is also not retried — it too is an answer, about the +transport rather than the repo, and no backoff measured in seconds outlives a +window measured in hours. The first post-fix sweep proved both halves at once: +it hit the rate limit mid-run, withheld 29 verdicts honestly instead of +inventing 29 findings, and spent 3× the calls rediscovering the same exhausted +limit — hence the fail-fast. + +**Why it survived so long:** nothing could reach the failure path without a real +outage. `gh_get` therefore lives in `verify-predicates.sh` with the rules, and +`test-verify-predicates.sh` stubs `gh` to exercise all three states — including +that a transient failure recovers on retry and that a 404 does *not* burn three +calls. Proven by mutation: collapsing `return 1` back to `return 2` turns exactly +two cases red. + +> Generalisation worth carrying: **`2>/dev/null` on a read you will draw a +> conclusion from converts an outage into a lie.** Silence is not data. If a +> tool cannot distinguish "absent" from "unreachable", every clean report it +> produces is unfalsifiable. + **Audit remotes, never local checkouts.** A first attempt at this swept working trees under `~/dev` and reported two violations that had already been fixed on `main` — the checkouts were stale — which produced one redundant PR and one that