diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json index 352f29b1..c3e2ca61 100644 --- a/node_modules/.package-lock.json +++ b/node_modules/.package-lock.json @@ -1,5 +1,5 @@ { - "name": "pr-782", + "name": "pr-821", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/package-lock.json b/package-lock.json index 5da17dbd..226b2af9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "pr-782", + "name": "pr-821", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 34471135..2824fc68 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -454,6 +454,12 @@ _run_json() { summary="$(_gh_err_summary "$err")" if [ "$attempt" -ge "$attempts" ]; then echo "::error::_run_json: failed to fetch run list for $repo (workflow=$wf) after $attempts attempt(s) — $summary" >&2 + # Record the fetch failure for the sync-issues resilience layer (#820). A sustained + # outage does NOT cleanly abort the caller (command substitutions don't inherit errexit), + # so _frontier_state can limp on and emit a misleadingly-clean line; a file flag survives + # the subshell and lets _frontier_state_resilient know the data is a partial/failed fetch. + # No-op unless sync-issues armed the flag, so every other read path is unchanged. + [ -n "${_CANARY_FETCH_FAIL_FLAG:-}" ] && printf '%s\n' "$repo" >> "$_CANARY_FETCH_FAIL_FLAG" 2>/dev/null || true rm -f "$errfile"; return 1 fi # Prefer the server's explicit throttle hint; otherwise exponential + full jitter. @@ -1308,10 +1314,10 @@ _suspect_guidance() { | "- **\(.id):** \(.guidance)"' } -# _blocker_body [] [] [] [] [] [] [] +# _blocker_body [] [] [] [] [] [] [] [] _blocker_body() { local agent="$1" transition="$2" cand="$3" cum_fail="$4" cum_startup="$5" triage="$6" host="$7" evidence="$8" mix_shift="${9:--}" mix_table="${10:-}" note - local downgrade="${11:--}" dg_cand_rate="${12:-0}" dg_cand_sample="${13:-0}" dg_base_rate="${14:-0}" dg_base_sample="${15:-0}" + local downgrade="${11:--}" dg_cand_rate="${12:-0}" dg_cand_sample="${13:-0}" dg_base_rate="${14:-0}" dg_base_sample="${15:-0}" data_gap="${16:-0}" # Correctness variant (#668 L2): a decision-mix SHIFT holds as SUSPECT but is NOT a reliability # failure — the candidate exited green yet its decision distribution moved. Distinct note + a # candidate-vs-baseline mix table replace the failing-runs evidence (there are none). @@ -1365,6 +1371,16 @@ $(printf '%s\n' "$guidance" | sed 's/^/> /')" # nothing environmental to report) — say what actually happened. note="> ℹ️ **INDETERMINATE (gate held — candidate cut date unresolved)** — the gate could not resolve this candidate's cut date, so it fails closed and holds the promotion instead of evaluating an unbounded run history. This is **not** a detected run failure (cumulative failures: $cum_fail). It usually clears on its own once the candidate's release tag/cut date becomes resolvable; if it persists, verify that candidate \`${cand:0:12}\` is a tagged, resolvable commit on \`$host\`." fi + # Data-gap fail-closed (#820): run history was unreadable this tick, so the cumulative counts + # are NOT reliable and the triage verdict rests on the reusable diff alone. Prepend a banner + # (before the triage note) and blank out the misleading "0" cumulative-failures cell. + local cum_row="**$cum_fail** (startup_failures: $cum_startup)" + if [ "$data_gap" = "1" ]; then + cum_row="_unknown — run history unavailable this tick_" + note="> ⚠️ **PARTIAL DATA (run-history fetch failed) — FAILING CLOSED.** The canary gate could not read this agent's recent run history this tick (a sustained GitHub API failure — see the workflow log). Rather than report a false all-clear, the gate holds the promotion and keeps this issue open. The cumulative counts below are **not reliable for this tick**; the triage verdict is derived from the reusable diff alone. This clears automatically once run history is readable again and the gate re-evaluates. +> +$note" + fi cat < **Automated canary-rollout blocker.** The release gate is holding \`$agent\` and will not promote it until this clears. Filed + maintained by the Canary Rollout workflow (gate standard: .github#548); this issue is **regenerated each run and auto-closes** when the gate passes — do not edit the table below by hand. @@ -1375,7 +1391,7 @@ $(printf '%s\n' "$guidance" | sed 's/^/> /')" | transition | \`$transition\` | | candidate | \`${cand:0:12}\` | | host repo | \`$host\` | -| cumulative failures | **$cum_fail** (startup_failures: $cum_startup) | +| cumulative failures | $cum_row | | triage | **$triage** | $note @@ -1454,10 +1470,79 @@ $1 EOF } +# _frontier_state_resilient — sync-issues resilience wrapper (#820). _frontier_state +# reads run history (_run_json), and on a SUSTAINED fetch failure (#738 fail-closed) it does +# NOT cleanly stop: command substitutions don't inherit errexit, so it limps on and emits a +# misleadingly-CLEAN state line (or aborts to empty). Either way sync-issues could no longer +# tell a CLEARED agent from an UNREADABLE one — silently dropping (or auto-closing) a +# regression's tracked issue: the safety net defeated by exactly the failures it should record. +# So _run_json records each fetch failure to a file flag we arm here; when it fires we DISCARD +# _frontier_state's output and instead determine everything that does NOT need run history +# (candidate, frontier, transition, reusable-differs → triage) from tag/blob resolution, and +# FAIL CLOSED to BLOCKED so the tracked issue is upserted (never a green no-op). Emits the 18 +# _frontier_state fields PLUS a trailing DATA-GAP flag (0=normal state resolved, 1=partial +# run-history). Returns NON-ZERO only when even the candidate/frontier cannot be resolved — a +# TOTAL inability stays a hard error, surfaced by the caller. +_frontier_state_resilient() { + local agent="$1" out="" rc=0 fetch_failed=0 + # Arm a file flag that _run_json appends to on a sustained fetch failure. A file (not a shell + # var) is used because _frontier_state runs in a command-substitution subshell, and a var set + # there would not survive back to us — the file does. + local flag; flag="$(mktemp 2>/dev/null || echo "")" + if [ -z "$flag" ]; then + # Cannot arm the fetch-failure detector at all — treat as an unreadable tick rather + # than silently falling back to the old undetectable-outage behavior. + fetch_failed=1 + out="$(_frontier_state "$agent")" || rc=$? + else + # `|| rc=$?` catches any abort so set -e does not tear down the fleet loop on one agent. + export _CANARY_FETCH_FAIL_FLAG="$flag" + out="$(_frontier_state "$agent")" || rc=$? + unset _CANARY_FETCH_FAIL_FLAG + [ -s "$flag" ] && fetch_failed=1 + rm -f "$flag" + fi + # Normal path (no fetch failure, a state line was produced): transparent pass-through, datagap=0. + if [ "$fetch_failed" -eq 0 ] && [ -n "$out" ] && [ "$rc" -eq 0 ]; then + printf '%s 0\n' "$out" + return 0 + fi + # Non-data-gap _frontier_state failure: the fetch succeeded but _frontier_state still failed — + # propagate the error rather than mis-annotating it as a data gap and emitting datagap=1. + if [ "$fetch_failed" -eq 0 ]; then + [ "$rc" -ne 0 ] && return "$rc" + return 1 + fi + # Data gap: the run-history fetch failed — reconstruct the tag-only facts (none read run history). + local cand chans frontier="" ch c transition prior differs triage + cand="$(channel_commit "$agent" next || true)" + chans="$(ordered_channels "$agent" || true)" + local chan_array=() + IFS=, read -r -a chan_array <<< "$chans" + for ch in "${chan_array[@]}"; do + c="$(channel_commit "$agent" "$ch" || true)" + if [ "$ch" = "next" ] || [ "$c" = "$cand" ]; then :; else frontier="$ch"; break; fi + done + # Total inability: cannot resolve even the candidate or a pending frontier → hard error. + # Fail closed (never a silent green); the caller surfaces ::error:: and ends non-zero. + { [ -z "$cand" ] || [ -z "$frontier" ]; } && return 1 + transition="$(transition_key "$frontier" "$chans")" + prior="$(channel_commit "$agent" "$frontier" || true)" + differs="$(_reusable_differs "$agent" "$cand" "$prior")" + # classify_failure with an unknown category + no suspect signal: differs=1 → REGRESSION + # (fail closed — a changed reusable with UNREADABLE health is treated as a suspected + # regression that needs a human), differs=0 → PRE_EXISTING (a byte-identical reusable cannot + # be a candidate regression). Either verdict still tracks the agent as BLOCKED. + triage="$(classify_failure "$differs" unknown 0)" + echo "$cand $frontier $transition BLOCKED 0 0 0 0 0 0 0 $triage - - 0 0 0 0 1" +} + # cmd_sync_issues [--dry-run] — upsert one blocker issue per BLOCKED agent, and render the # fleet-status table into the run's job summary (GITHUB_STEP_SUMMARY). Blocker issues are -# idempotent (marker-keyed) and auto-close when the gate clears. Best-effort: never fails the -# run. Reads ISSUE_REPO (default THIS_REPO). +# idempotent (marker-keyed) and auto-close when the gate clears. Best-effort for issue-write +# failures (never aborts mid-fleet), but FAILS CLOSED (non-zero) when an agent's state is +# totally undeterminable — a green no-op would mask a regression (#820). Reads ISSUE_REPO +# (default THIS_REPO). cmd_sync_issues() { local dry=false; [ "${1:-}" = "--dry-run" ] && dry=true local agents; agents="$(_jq -r '.agents? | keys[]?' 2>/dev/null || true)" @@ -1477,12 +1562,21 @@ cmd_sync_issues() { # frontier — a SEPARATE label/issue from canary-blocker so the two concerns never collide. gh label create canary-confirm --repo "$ISSUE_REPO" --color fbca04 --description "canary-rollout: human go/no-go at a require_confirmation ring boundary" >/dev/null 2>&1 || true fi - local rows="" agent + local rows="" agent hard_fail=0 while IFS= read -r agent; do [ -z "$agent" ] && continue local cand frontier transition state _d _f _s _t cum_fail cum_startup _cb triage mix_shift host - local downgrade dg_cand_rate dg_cand_sample dg_base_rate dg_base_sample - read -r cand frontier transition state _d _f _s _t cum_fail cum_startup _cb triage mix_shift downgrade dg_cand_rate dg_cand_sample dg_base_rate dg_base_sample < <(_frontier_state "$agent") + local downgrade dg_cand_rate dg_cand_sample dg_base_rate dg_base_sample datagap + # Resilient read (#820): the wrapper fails closed to a BLOCKED+datagap line on a run-history + # fetch outage, and returns non-zero (no output) only on a TOTAL inability to determine + # state. A failed `read` there means we could not even resolve the candidate/frontier — + # fail closed rather than report a false all-clear (a green no-op could mask a regression). + if ! read -r cand frontier transition state _d _f _s _t cum_fail cum_startup _cb triage mix_shift downgrade dg_cand_rate dg_cand_sample dg_base_rate dg_base_sample datagap < <(_frontier_state_resilient "$agent"); then + echo "::error::sync-issues: cannot determine canary state for '$agent' (run-history AND tag resolution unavailable) — failing closed rather than reporting a false all-clear." >&2 + hard_fail=1 + rows+="| \`$agent\` | UNKNOWN | \`-\` | ? | ? | ⚠️ data unavailable (fail-closed) |"$'\n' + continue + fi host="$(_agent_field "$agent" host)" local blk="—" num_state num istate # Best-effort (#1081): these substitutions call gh/jq. Under `set -euo pipefail` @@ -1493,11 +1587,18 @@ cmd_sync_issues() { num="${num_state%%$'\t'*}"; istate="${num_state##*$'\t'}" if [ "$state" = "BLOCKED" ]; then local evidence body title mix_table="" - evidence="$(_blocker_evidence "$agent" "$cand" || true)" + if [ "${datagap:-0}" = "1" ]; then + # Run history was unreadable this tick — there are no listable failing runs to cite. + evidence="_(⚠️ run-history fetch failed this tick — the failing runs could not be listed. The gate FAILS CLOSED: the promotion is held and this issue stays open until run history is readable again and the gate can re-evaluate.)_" + else + evidence="$(_blocker_evidence "$agent" "$cand" || true)" + fi [ "$mix_shift" = "SHIFT" ] && mix_table="$(_decision_mix_table "$agent" "$cand" || true)" - body="$(_blocker_body "$agent" "$transition" "$cand" "$cum_fail" "$cum_startup" "$triage" "$host" "$evidence" "$mix_shift" "$mix_table" "$downgrade" "$dg_cand_rate" "$dg_cand_sample" "$dg_base_rate" "$dg_base_sample")" + body="$(_blocker_body "$agent" "$transition" "$cand" "$cum_fail" "$cum_startup" "$triage" "$host" "$evidence" "$mix_shift" "$mix_table" "$downgrade" "$dg_cand_rate" "$dg_cand_sample" "$dg_base_rate" "$dg_base_sample" "${datagap:-0}")" if [ "$mix_shift" = "SHIFT" ]; then title="Canary blocker: $agent $transition (decision-mix shift, SUSPECT)" + elif [ "${datagap:-0}" = "1" ]; then + title="Canary blocker: $agent $transition ($triage, partial run-history — fail-closed)" else title="Canary blocker: $agent $transition (cum_fail=$cum_fail, $triage)" fi @@ -1596,6 +1697,9 @@ cmd_sync_issues() { echo "──── fleet status ────" printf '%s\n' "$dmd" fi + # Fail closed (#820): if any agent's state was totally undeterminable this tick, end non-zero + # AFTER rendering the dashboard — a green no-op would mask a possible regression. + [ "$hard_fail" -eq 1 ] && return 1 return 0 } diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index 8488b8fb..5c5e0e6b 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -1446,15 +1446,26 @@ GHEOF # evidence; a cleared agent's issue auto-closes; the fleet-status table is rendered to the job # summary. dev-lead-only registry keeps the fleet loop to one agent; the gh stub logs issue ops. _sync_stub() { - # $1 conclusion (failure→BLOCKED | success→cleared); $2 blocker-list JSON returned by `gh issue list` + # $1 = conclusion (failure→BLOCKED | success→cleared | gap→run-list always fails) + # $2 = blocker-list JSON returned by `gh issue list` + # $3 = blob_mode: "same" (default, ref=cccc and ref=bbbb return identical blobs) + # or "differ" (ref=cccc and ref=bbbb return distinct blobs → reusable differs) # dev-lead is cross-repo (host=.github-private, THIS_REPO=.github): channel tags, release # date, and reusable blobs resolve via gh api; git is a no-op. - local concl="$1" blocker_list="${2:-[]}" + local concl="$1" blocker_list="${2:-[]}" blob_mode="${3:-same}" + local cccc_blob="blobAAAA" bbbb_blob="blobAAAA" + [ "$blob_mode" = "differ" ] && { cccc_blob="reuseCAND"; bbbb_blob="reusePRIOR"; } STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" export ISSUE_LOG="$STUB_BIN/issue.log"; : > "$ISSUE_LOG" local cut_iso run_iso cut_iso="$(date -u -d '-3 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-3d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" run_iso="$(date -u -d '-2 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-2d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + local run_list_case + if [ "$concl" = "gap" ]; then + run_list_case='echo "gh: run history unavailable (HTTP 500)" >&2; exit 1' + else + run_list_case="jq -nc --arg d \"$run_iso\" --arg c \"$concl\" '[range(3)|{conclusion:\$c,createdAt:\$d,databaseId:12345,workflowName:\"Dev-Lead Agent\"}]'" + fi cat > "$STUB_BIN/git" <<'GITEOF' #!/usr/bin/env bash : # dev-lead is cross-repo; all tag/blob resolution goes via gh api @@ -1469,9 +1480,9 @@ case "\$*" in *"git/ref/tags/dev-lead/stable"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb commit" ;; *"matching-refs/tags/dev-lead/v"*) printf 'refs/tags/dev-lead/v2.0.0\ttagobj\ttag\n' ;; *"git/tags/tagobj"*) printf '%s\t%s\n' "cccccccccccccccccccccccccccccccccccccccc" "$cut_iso" ;; - *"ref=cccc"*) echo "blobAAAA" ;; - *"ref=bbbb"*) echo "blobAAAA" ;; - *"run list"*) jq -nc --arg d "$run_iso" --arg c "$concl" '[range(3)|{conclusion:\$c,createdAt:\$d,databaseId:12345,workflowName:"Dev-Lead Agent"}]' ;; + *"ref=cccc"*) echo "$cccc_blob" ;; + *"ref=bbbb"*) echo "$bbbb_blob" ;; + *"run list"*) $run_list_case ;; *"run view"*) echo '{"jobs":[{"steps":[{"name":"Some step","conclusion":"failure"}]}]}' ;; "issue list"*) echo '$blocker_list' ;; "issue create"*) echo "CREATE|\$*" >> "$ISSUE_LOG"; echo "https://github.com/petry-projects/.github-private/issues/777" ;; @@ -2619,6 +2630,89 @@ GHEOF jq '{org_infra_repos, agents: {"dev-lead": .agents["dev-lead"]}}' "$RINGS" > "$SYNC_RINGS" } +# ── sync-issues fail-closed on partial/failed run-history data (#820) ───────────────── +# sync-issues reads run history through _run_json; on a SUSTAINED fetch failure (#738) +# _frontier_state aborts to empty output under `set -e`, and the agent used to fall into the +# "not BLOCKED" branch — silently skipping (or auto-closing) its regression tracking issue. +# The resilience contract: on a data gap, still determine everything that does NOT need run +# history (candidate/frontier/transition/reusable-differs → triage) and FAIL CLOSED to a +# tracked BLOCKED issue with the gap annotated. A TOTAL inability (not even candidate/frontier +# resolvable) stays a hard error (non-zero), never a green no-op. +# +# _sync_gap_stub — like _sync_stub but `gh run list` FAILS every attempt (the fetch outage), +# while tag/blob resolution stays intact and the candidate's reusable DIFFERS from the prior +# channel (ref=cccc vs ref=bbbb blobs differ → differs=1 → REGRESSION fail-closed triage). +_sync_gap_stub() { + # Thin wrapper: delegates to _sync_stub with gap mode (run list always fails) and differing + # blobs (ref=cccc vs ref=bbbb are distinct, so reusable-differs=1 → REGRESSION fail-closed). + local blocker_list="${1:-[]}" + _sync_stub "gap" "$blocker_list" "differ" + SYNC_RINGS="$BATS_TEST_TMPDIR/sync-rings-gap.json" + jq '{org_infra_repos, agents: {"dev-lead": .agents["dev-lead"]}}' "$RINGS" > "$SYNC_RINGS" +} + +@test "orchestrator: sync-issues fails CLOSED on a partial run-history fetch — opens a REGRESSION needs-human blocker with the gap annotated (#820)" { + # The run-history fetch is down but tags/blobs resolve and the reusable DIFFERS → the gate + # cannot confirm health but MUST NOT report a false all-clear: it fails closed to a tracked + # BLOCKED issue, triage REGRESSION (fail-closed), routed to dev-lead + needs-human. + _sync_gap_stub '[]' + local summ="$BATS_TEST_TMPDIR/summary_gap.md"; : > "$summ" + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" \ + CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=2 GITHUB_STEP_SUMMARY="$summ" \ + bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + # A blocker issue is OPENED (never silently skipped) and routed for action + human review. + [[ "$output" == *"opened blocker issue #777 for dev-lead"* ]] + grep -q '^CREATE|' "$ISSUE_LOG" + grep -q -- "--add-label dev-lead" "$ISSUE_LOG" + grep -q -- "--add-label needs-human" "$ISSUE_LOG" + # The created issue body annotates the data gap so a human knows counts are unreliable. + grep -q "PARTIAL DATA" "$ISSUE_LOG" + grep -qi "run.history" "$ISSUE_LOG" + # The fleet dashboard still renders. + grep -q "Canary Rollout — fleet status" "$summ" +} + +@test "orchestrator: sync-issues partial-fetch UPDATE keeps an existing regression's issue open (no auto-close) (#820)" { + # An OPEN blocker exists for dev-lead. A fetch outage must NOT be read as 'cleared' and close + # it — it must be UPDATED (kept open) and re-annotated with the gap. + _sync_gap_stub '[{"number":501,"state":"OPEN","body":""}]' + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" \ + CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=2 bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + [[ "$output" == *"updated blocker issue #501 for dev-lead"* ]] + grep -q -- "--add-label needs-human" "$ISSUE_LOG" + # It must NOT have been auto-closed by the fetch outage. + ! grep -q "^CLOSE|" "$ISSUE_LOG" +} + +@test "orchestrator: _frontier_state_resilient tags a normal state line with datagap=0 and passes it through (#820)" { + # When _frontier_state resolves normally the wrapper is transparent: same fields, trailing 0. + run bash -c "source '$ORCH'; _frontier_state() { echo 'ccccccc ring0 next->ring0 BLOCKED 0 4 0 3 2 0 0 REGRESSION - - 0 0 0 0'; }; _frontier_state_resilient dev-lead" + [ "$status" -eq 0 ] + [[ "$output" == "ccccccc ring0 next->ring0 BLOCKED 0 4 0 3 2 0 0 REGRESSION - - 0 0 0 0 0" ]] +} + +@test "orchestrator: _frontier_state_resilient fails CLOSED (non-zero) on a TOTAL inability to determine state (#820)" { + # _frontier_state aborts to empty AND the candidate/frontier cannot be re-resolved from tags: + # a total inability must stay a HARD ERROR (non-zero), never a green no-op. + run bash -c "source '$ORCH'; _frontier_state() { return 1; }; channel_commit() { echo ''; }; ordered_channels() { echo 'next,ring0,ring1,stable'; }; _frontier_state_resilient dev-lead" + [ "$status" -ne 0 ] +} + +@test "orchestrator: sync-issues ends NON-ZERO when an agent's state is totally undeterminable (fail-closed, not a green no-op) (#820)" { + # Force a total inability for the single registered agent and assert the step fails closed. + _sync_gap_stub '[]' + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" \ + CANARY_GH_RETRY_SLEEP=0 CANARY_GH_RETRIES=2 bash -c " + source '$ORCH' + _frontier_state_resilient() { return 1; } + cmd_sync_issues + " + [ "$status" -ne 0 ] + [[ "$output" == *"cannot determine canary state"* ]] +} + @test "orchestrator: sync-issues CREATE path applies needs-human for SUSPECT triage" { # No existing issue → create path; SUSPECT (differs=1 + suspect step) must add needs-human. _sync_suspect_stub '[]'