Skip to content

fix(ci): a failed fetch is not a finding — the audit withholds what it cannot see #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
github-actions merged 1 commit into from
Aug 20, 2026
Merged

fix(ci): a failed fetch is not a finding — the audit withholds what it cannot see #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 127 additions & 0 deletions scripts/ci/test-verify-predicates.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,5 +78,132 @@ assert pass verify_softens_itself 'npm run typecheck --workspaces --if-present'
assert fail verify_softens_itself 'pnpm lint && pnpm typecheck && pnpm test' 'a clean verify is not flagged'
assert fail verify_softens_itself '' 'an empty verify is not flagged here — absence is the audit job'

# assert2 <expected pass|fail> <predicate> <arg1> <arg2> <description>
assert2() {
local want="$1" pred="$2" a="$3" b="$4" desc="$5"
if "$pred" "$a" "$b"; then got=pass; else got=fail; fi
[ "$got" = "$want" ] && ok "$desc" || no "$desc (expected $want, got $got)"
}

VERIFY_NPM='npm run lint && npm run typecheck && npm run test'

# aoz-housing's real shape: the same three scripts, split across parallel jobs
# for speed, none of them softened. This is the case that proved the
# string-match rule wrong.
AOZ_WF=' - name: Lint
run: npm run lint
- name: Type check
run: npm run typecheck
- name: Run unit tests
run: npm test -- --ci --coverage
- name: Build
run: npm run build'

# Same split, but one gate quietly absent.
AOZ_MISSING_TYPECHECK=' - name: Lint
run: npm run lint
- name: Run unit tests
run: npm test -- --ci --coverage'

echo "verify_gate_scripts"
got=$(verify_gate_scripts "$VERIFY_NPM" | tr '\n' ' ')
[ "$got" = "lint test typecheck " ] \
&& ok "decomposes verify into its named gates" \
|| no "decomposes verify into its named gates (got '$got')"
got=$(verify_gate_scripts 'eslint . && tsc --noEmit && jest')
[ -z "$got" ] \
&& ok "a verify that shells out directly decomposes to nothing" \
|| no "a verify that shells out directly decomposes to nothing (got '$got')"

echo "ci_runs_verify_gates"
assert2 pass ci_runs_verify_gates "$AOZ_WF" "$VERIFY_NPM" \
'aoz-housing: gates run individually, unsoftened → satisfied'
assert2 fail ci_runs_verify_gates "$AOZ_MISSING_TYPECHECK" "$VERIFY_NPM" \
'a gate in verify that CI never runs → NOT satisfied'
assert2 fail ci_runs_verify_gates "$HANDCOPIED" 'npm run lint && npm run test' \
'botsmann: hand-copied AND --if-present on every step → still caught'
assert2 fail ci_runs_verify_gates "$AOZ_WF" 'eslint . && tsc --noEmit' \
'an undecomposable verify is unproven, not waved through'
assert2 fail ci_runs_verify_gates '' "$VERIFY_NPM" \
'no workflows at all → NOT satisfied'
assert2 pass ci_runs_verify_gates 'run: pnpm lint
run: pnpm typecheck
run: pnpm test' 'pnpm lint && pnpm typecheck && pnpm test' \
'pnpm implicit-run spelling works on both sides'

# --- gh_get: the three states ------------------------------------------------
# This is the test that would have caught the 2026-08-16 miscount. `gh` is
# stubbed so the FAILURE path is reachable without a real outage — which is
# exactly why the bug survived: nothing could exercise it.

GH_GET_BACKOFF=0 # do not actually sleep through the retries in tests

echo "gh_get"

gh() { printf 'ok-body'; return 0; }
out=$(gh_get 'any/path'); rc=$?
[ "$rc" = 0 ] && [ "$out" = "ok-body" ] \
&& ok "success returns 0 and the body" \
|| no "success returns 0 and the body (rc=$rc out='$out')"

gh() { echo 'gh: Not Found (HTTP 404)' >&2; return 1; }
gh_get 'missing/path' >/dev/null; rc=$?
[ "$rc" = 2 ] \
&& ok "a real 404 is ABSENT (2), distinct from a failure" \
|| no "a real 404 is ABSENT (2) (rc=$rc)"

gh() { echo 'gh: HTTP 403 rate limit exceeded' >&2; return 1; }
gh_get 'blocked/path' >/dev/null; rc=$?
[ "$rc" = 1 ] \
&& ok "a 403 is COULD-NOT-LOOK (1), never mistaken for absence" \
|| no "a 403 is COULD-NOT-LOOK (1) (rc=$rc)"

gh() { echo 'gh: HTTP 502 Bad Gateway' >&2; return 1; }
gh_get 'flaky/path' >/dev/null; rc=$?
[ "$rc" = 1 ] \
&& ok "a 5xx is COULD-NOT-LOOK (1), not absence" \
|| no "a 5xx is COULD-NOT-LOOK (1) (rc=$rc)"

# Transient then success: the retry must actually rescue the call, otherwise
# every blip still costs a repo its verdict.
ATTEMPTS_FILE=$(mktemp)
echo 0 > "$ATTEMPTS_FILE"
gh() {
local n; n=$(cat "$ATTEMPTS_FILE"); n=$((n + 1)); echo "$n" > "$ATTEMPTS_FILE"
if [ "$n" -lt 2 ]; then echo 'gh: HTTP 502' >&2; return 1; fi
printf 'recovered'; return 0
}
out=$(gh_get 'flaky/path'); rc=$?
[ "$rc" = 0 ] && [ "$out" = "recovered" ] \
&& ok "a transient failure is retried and recovers" \
|| no "a transient failure is retried and recovers (rc=$rc out='$out')"

# A 404 must NOT burn retries — it is an answer, and retrying it would triple
# the cost of every genuinely-absent file across the fleet.
echo 0 > "$ATTEMPTS_FILE"
gh() {
local n; n=$(cat "$ATTEMPTS_FILE"); n=$((n + 1)); echo "$n" > "$ATTEMPTS_FILE"
echo 'gh: Not Found (HTTP 404)' >&2; return 1
}
gh_get 'missing/path' >/dev/null
[ "$(cat "$ATTEMPTS_FILE")" = 1 ] \
&& ok "a 404 is not retried (costs one call, not three)" \
|| no "a 404 is not retried (took $(cat "$ATTEMPTS_FILE") calls)"

# An exhausted rate limit is not retried either: seconds of backoff cannot
# outlive an hour-long window, and retrying burns calls when they're scarcest.
# It is still COULD-NOT-LOOK (1) — a transport fact, never absence.
echo 0 > "$ATTEMPTS_FILE"
gh() {
local n; n=$(cat "$ATTEMPTS_FILE"); n=$((n + 1)); echo "$n" > "$ATTEMPTS_FILE"
echo 'gh: HTTP 403 API rate limit exceeded for user' >&2; return 1
}
gh_get 'starved/path' >/dev/null; rc=$?
[ "$rc" = 1 ] && [ "$(cat "$ATTEMPTS_FILE")" = 1 ] \
&& ok "a rate-limited call fails fast as COULD-NOT-LOOK (1 call, rc=1)" \
|| no "a rate-limited call fails fast (rc=$rc, took $(cat "$ATTEMPTS_FILE") calls)"
rm -f "$ATTEMPTS_FILE"
unset -f gh

printf '\n%d passed, %d failed\n' "$PASS" "$FAIL"
[ "$FAIL" -eq 0 ]
125 changes: 111 additions & 14 deletions scripts/ci/verify-floor-audit.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,33 @@ LIMIT="${GH_LIMIT:-100}"
WARN_ONLY=0
[ "${1:-}" = "--warn-only" ] && WARN_ONLY=1

# --- Fetching is not knowing. -----------------------------------------------
#
# Every remote read here used to be `gh api ... 2>/dev/null`, and the empty
# output that a failure produces was then read as a FACT — "no package.json",
# "no workflows". A transient 403/5xx is not a fact. Measured 2026-08-16, one
# sweep produced FOUR wrong verdicts from that conflation:
#
# vitareba, aoz-housing — live JS apps, reported as "not a JS repo" and
# dropped from the floor entirely
# ai-forms, fleetcrown — reported "CI never runs verify" while line 19 and
# line 52 of their ci.yml do exactly that
#
# The tell was arithmetic: two runs an hour apart inspected 24 and 22 repos
# with no repo created or deleted between them. A check whose output moves
# while the thing it measures holds still is not measuring it.
#
# So the failure mode is now a THIRD state, never silence:
# 0 = fetched 2 = genuinely absent (404) 1 = could not look
#
# This is the same discipline the OPAQUE column already applies to unreadable
# verify scripts, applied to the transport instead of the content.
#
# `gh_get` itself lives in verify-predicates.sh so it can be tested against a
# stubbed `gh` — the bug it fixes was invisible precisely because nothing could
# exercise the failure path without a real outage.
trap 'rm -f "$GH_ERR"' EXIT

# Declared before any branch that could skip the assignment — under `set -u` a
# later reference would otherwise kill the whole run instead of one repo.
ok_list=""
Expand All @@ -59,6 +86,7 @@ discarded_list=""
opaque_list=""
forks_list=""
uncalled_list=""
decomposed_list=""
softened_list=""
total=0

Expand Down Expand Up @@ -141,11 +169,18 @@ while IFS=$'\t' read -r name branch is_fork; do
# Resolve package.json ON THE REPO'S OWN DEFAULT BRANCH. Asking for ?ref=main
# on a master repo 404s, which means "wrong ref", not "no package.json" —
# that mistake previously produced a confidently wrong fleet survey.
pkg=$(gh api "repos/$OWNER/$name/contents/package.json?ref=$branch" \
--jq '.content' 2>/dev/null | tr -d '\n' | base64 -d 2>/dev/null)
pkg_raw=$(gh_get "repos/$OWNER/$name/contents/package.json?ref=$branch")
case $? in
0) pkg=$(printf '%s' "$pkg_raw" | jq -r '.content // ""' 2>/dev/null \
| tr -d '\n' | base64 -d 2>/dev/null) ;;
2) skipped_list="${skipped_list} $name (no package.json on $branch)\n"
continue ;;
*) unreadable_list="${unreadable_list} $name — could not read package.json on $branch after 3 tries (API error, NOT absence)\n"
continue ;;
esac

if [ -z "$pkg" ]; then
skipped_list="${skipped_list} $name (no package.json on $branch)\n"
unreadable_list="${unreadable_list} $name — package.json on $branch fetched but decoded empty\n"
continue
fi

Expand All @@ -163,6 +198,8 @@ while IFS=$'\t' read -r name branch is_fork; do
absent="" # gates the repo has no script for at all
opaque_gates="" # gates we cannot see EITHER WAY, because verify delegates
# to a script file this audit does not execute or parse
sub_unreadable="" # sub-manifests the API would not hand over — a transport
# failure, which must not be read as "the gate isn't there"

# Expand `npm run X` / `pnpm X` inside verify two levels deep, so a gate
# counts whether it is reached via a named script or run directly. Matching
Expand Down Expand Up @@ -201,8 +238,16 @@ while IFS=$'\t' read -r name branch is_fork; do
sub_dir=$(printf '%s' "$pair" | awk -F'|' '{print $2}')
sub_script=$(printf '%s' "$pair" | awk -F'|' '{print $NF}')
[ -n "$sub_dir" ] && [ -n "$sub_script" ] || continue
sub_pkg=$(gh api "repos/$OWNER/$name/contents/$sub_dir/package.json?ref=$branch" \
--jq '.content' 2>/dev/null | tr -d '\n' | base64 -d 2>/dev/null)
# A failed read here would make the sub-package's gate look ABSENT, i.e. a
# "needs real work" verdict earned by a network blip. Retry, then record
# that we could not look rather than letting silence mean absence.
sub_raw=$(gh_get "repos/$OWNER/$name/contents/$sub_dir/package.json?ref=$branch")
case $? in
0) sub_pkg=$(printf '%s' "$sub_raw" | jq -r '.content // ""' 2>/dev/null \
| tr -d '\n' | base64 -d 2>/dev/null) ;;
2) continue ;; # genuinely no sub-manifest
*) sub_unreadable="$sub_unreadable $sub_dir/package.json"; continue ;;
esac
[ -n "$sub_pkg" ] || continue
sub_scripts=$(printf '%s' "$sub_pkg" | jq -r '.scripts // {}' 2>/dev/null)
[ -n "$sub_scripts" ] && [ "$sub_scripts" != null ] || continue
Expand Down Expand Up @@ -230,7 +275,9 @@ $(printf '%s' "$sub_scripts" | jq -r 'to_entries[] | "\(.key) \(.value)"')"
# the confident-absence bug this script just stopped committing. A truncated
# tree is reported as unknown rather than zero, for the same reason.
test_files=unknown
tree=$(gh api "repos/$OWNER/$name/git/trees/$branch?recursive=1" 2>/dev/null)
# Already fails SAFE (unknown, never zero), but route it through the retrying
# fetch anyway — an unknown here silently disables the no-test-files rule.
tree=$(gh_get "repos/$OWNER/$name/git/trees/$branch?recursive=1") || tree=""
if [ -n "$tree" ]; then
if [ "$(printf '%s' "$tree" | jq -r '.truncated // false')" = true ]; then
test_files=unknown # too big to enumerate; do not guess
Expand Down Expand Up @@ -352,12 +399,29 @@ $(printf '%s' "$sub_scripts" | jq -r 'to_entries[] | "\(.key) \(.value)"')"
# and not a per-push hook.
ci_calls_verify=no
ci_softened_in=""

for wf in $(gh api "repos/$OWNER/$name/contents/.github/workflows?ref=$branch" \
--jq '.[] | select(.name | test("\\.ya?ml$")) | .name' 2>/dev/null); do
wf_body=$(gh api "repos/$OWNER/$name/contents/.github/workflows/$wf?ref=$branch" \
--jq '.content' 2>/dev/null | tr -d '\n' | base64 -d 2>/dev/null)
[ -n "$wf_body" ] || continue
all_wf=""
# Set the moment any workflow read fails. Every wiring verdict below is then
# withheld: an unread workflow is the one that might have contained the call
# we are about to report missing. This is what accused ai-forms of never
# running `verify` when its ci.yml line 19 is `npm run verify`.
wiring_unknown=""

wf_index=$(gh_get "repos/$OWNER/$name/contents/.github/workflows?ref=$branch")
case $? in
0) wf_names=$(printf '%s' "$wf_index" \
| jq -r '.[]? | select(.name | test("\\.ya?ml$")) | .name' 2>/dev/null) ;;
2) wf_names="" ;; # genuinely no workflows directory
*) wf_names=""; wiring_unknown="could not list .github/workflows" ;;
esac

for wf in $wf_names; do
wf_raw=$(gh_get "repos/$OWNER/$name/contents/.github/workflows/$wf?ref=$branch")
case $? in
0) wf_body=$(printf '%s' "$wf_raw" | jq -r '.content // ""' 2>/dev/null \
| tr -d '\n' | base64 -d 2>/dev/null) ;;
*) wiring_unknown="could not read $wf"; continue ;;
esac
[ -n "$wf_body" ] || { wiring_unknown="$wf decoded empty"; continue; }
hits=$(printf '%s\n' "$wf_body" | scan_discarded_gates "$wf")
while IFS= read -r hit; do
[ -n "$hit" ] || continue
Expand All @@ -369,12 +433,31 @@ $(printf '%s' "$sub_scripts" | jq -r 'to_entries[] | "\(.key) \(.value)"')"
ci_calls_verify=yes
ci_verify_softened "$wf_body" && ci_softened_in="$wf"
fi
all_wf="${all_wf}
${wf_body}"
done

# Orthogonal to the content audit above: a repo can be AT FLOOR on what
# `verify` contains and still have nothing on the branch that runs it.
#
# Calling `npm run verify` is the direct way to satisfy this. It is not the
# only way: a repo may run each constituent script by name, unsoftened, to
# fan the gates into parallel jobs. That satisfies the actual contract — every
# gate runs and every one can still fail — so it is reported separately rather
# than counted as a violation. Demanding the literal string was asking
# aoz-housing to serialize the slowest pipeline in the fleet to please a
# regex.
if [ -n "$verify" ] && [ "$ci_calls_verify" = no ]; then
uncalled_list="${uncalled_list} $name — CI never runs \`verify\`\n"
if [ -n "$wiring_unknown" ]; then
# Withheld, not decided. The unread file is exactly the one that could
# have contained the call, so "not found" here would be a claim about a
# place we never looked.
unreadable_list="${unreadable_list} $name — wiring verdict withheld: $wiring_unknown\n"
elif ci_runs_verify_gates "$all_wf" "$verify"; then
decomposed_list="${decomposed_list} $name — CI runs each gate by name instead of \`verify\`\n"
else
uncalled_list="${uncalled_list} $name — CI never runs \`verify\`\n"
fi
fi
if [ -n "$ci_softened_in" ]; then
softened_list="${softened_list} $name — $ci_softened_in runs verify with --if-present\n"
Expand All @@ -383,7 +466,11 @@ $(printf '%s' "$sub_scripts" | jq -r 'to_entries[] | "\(.key) \(.value)"')"
softened_list="${softened_list} $name — \`verify\` softens itself: $verify\n"
fi

if [ -z "$verify" ]; then
if [ -n "$sub_unreadable" ] && [ -n "$absent" ]; then
# The gate looks missing, but a sub-manifest that could have defined it is
# exactly what we failed to fetch. Withheld rather than charged.
opaque_list="${opaque_list} $name — gate verdict withheld; could not read:$sub_unreadable\n"
elif [ -z "$verify" ]; then
missing_list="${missing_list} $name — no \`verify\` script at all\n"
elif [ -n "$absent" ]; then
# Not fixable by editing verify: the repo has no such script. Adding a
Expand Down Expand Up @@ -439,6 +526,16 @@ printf ' anything on the branch actually runs it. A repo can pass one and\n'
printf ' fail the other — botsmann did.)\n'
printf '%b' "${uncalled_list:- (none)\n}"

echo
printf '≡ DECOMPOSED — CI runs every gate by name, unsoftened, but never the\n'
printf ' word `verify`. NOT a violation: the contract is that each gate runs\n'
printf ' and can still fail, and it does. aoz-housing splits lint+typecheck,\n'
printf ' unit tests and build into parallel jobs on purpose — collapsing that\n'
printf ' into one verify step would serialize the fleet-slowest pipeline.\n'
printf ' Residual risk, stated: a gate ADDED to verify later will not reach\n'
printf ' CI by itself. That is exactly what this check keeps watching for.\n'
printf '%b' "${decomposed_list:- (none)\n}"

echo
printf '⊙ SOFTENED — verify is invoked, or written, so that it cannot fail\n'
printf '%b' "${softened_list:- (none)\n}"
Expand Down
Loading