diff --git a/.agents/skills/harness-adapters/SKILL.md b/.agents/skills/harness-adapters/SKILL.md index 5d8971cf09..654b51dc98 100644 --- a/.agents/skills/harness-adapters/SKILL.md +++ b/.agents/skills/harness-adapters/SKILL.md @@ -198,6 +198,7 @@ The false-wedge exposure is real and was reproduced live (docs/codex-busy-detect The busy-signature matching in `fm-watch.sh` and `fm-tmux-lib.sh` stays deliberately unchanged - the row still matches whenever it renders - but the absorb path now carries an interface-text-INDEPENDENT backstop scoped to codex. `codex_static_pane_upgrade` in `bin/fm-watch.sh` treats an otherwise-stale codex pane as provably working (absorb plus wedge timer) whenever the codex agent PROCESS is confidently alive: `fm_backend_agent_alive` reads `alive` because the pane's foreground command stays `codex` for the entire turn (verified 100/100 samples), a signal that does not depend on the busy row at all. A crashed codex reads `dead` (bare shell) and still surfaces at once, and a genuinely wedged codex still escalates past `STALE_ESCALATE_SECS` and on to demand-deep-inspection, so the backstop hides a healthy worker without hiding a stuck one. +One stale pane never reaches the backstop at all: when `bin/fm-crew-state.sh`'s run-step authoritatively reports the run stopped at a no-mistakes decision gate, that pane surfaces on first sight for every harness alike, so codex gets no first-sight absorb the others lack (docs/architecture.md owns the gate rule). This also covers the keymap-remap blind spot below, because it does not read the interrupt-hint text. The keymap case is the one that requires an operator to have changed something. diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 64ee51465f..9544a53a07 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -363,15 +363,29 @@ signal_reason_is_actionable() { # ... # (e.g. waiting on CI); # paused - the crew's authoritative current state is a declared external-wait # pause (paused:), which is EXPECTED to idle; +# parked - the crew's authoritative RUN-STEP state is a decision gate +# (no-mistakes awaiting_approval/fix_review). The pipeline stopped on +# purpose and is waiting for an answer, so an idle pane is the +# CORRECT behavior rather than a wedge. Deliberately restricted to +# source: run-step - a status-log-sourced `parked` is the same +# append-only needs-decision EVENT the stale path already reasons +# about, is arbitrarily old, and reads identically whether the crew is +# at the gate or died an hour ago. +# This token says the RUN is parked; it says NOTHING about the worker +# still being there, because a crashed worker leaves the run parked +# forever. A caller must corroborate with an interface-text- +# independent liveness signal before absorbing on it +# (bin/fm-watch.sh's parked_gate_liveness_class); # degraded- the reader could not consult its authoritative source at all: a # required tool is absent, or the call it makes could not be made or # did not answer (fm-crew-state.sh's `degraded` verdict, whose cause # token says which). NOT a statement about the crew: the supervisor # must report the broken instrument rather than draw either # conclusion from it; -# none - neither, so the wake must surface (a stopped/finished/parked/failed/ -# torn-down/unknown crew, or an unreadable verdict). -# One fm-crew-state.sh read serves BOTH absorb reasons at once. Reading the state +# none - none of the above, so the wake must surface (a stopped/finished/ +# failed/torn-down/unknown crew, a gate reported only by the status +# log, or an unreadable verdict). +# One fm-crew-state.sh read serves EVERY absorb reason at once. Reading the state # authoritatively (not the status log) is what keeps run-step precedence: a crew # that appended paused: but then STARTED a run reports working, never paused. # NOT a pure read: fm-crew-state.sh may make a bounded no-mistakes call, so callers @@ -386,10 +400,13 @@ crew_absorb_class() { # state=${line#state: }; state=${state%% *} if [ "$state" = paused ]; then printf 'paused'; return; fi if [ "$state" = degraded ]; then printf 'degraded'; return; fi + src=${line#*source: }; src=${src%% *} if [ "$state" = working ]; then - src=${line#*source: }; src=${src%% *} case "$src" in run-step|pane) printf 'working'; return ;; esac fi + if [ "$state" = parked ]; then + case "$src" in run-step) printf 'parked'; return ;; esac + fi printf 'none' } diff --git a/bin/fm-mark-parked.sh b/bin/fm-mark-parked.sh index eb08d6e2f3..5c456d9c41 100755 --- a/bin/fm-mark-parked.sh +++ b/bin/fm-mark-parked.sh @@ -3,6 +3,11 @@ # terminal outcome has been relayed and only external human action remains. # bin/fm-watch.sh owns window validation, secondmate rejection, key derivation, # and marker creation; this wrapper only provides a seatbelt-safe command shape. +# NOT for a mid-pipeline worker waiting at a no-mistakes decision gate: that one +# is neither terminal nor waiting on anything external, the watcher recognizes it +# on its own from the authoritative run-step plus agent liveness +# (bin/fm-watch.sh's parked_gate_liveness_class), and a hand-placed marker here +# would keep muting that pane after the gate was answered. set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 6fa36bd163..3d56582db5 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -9,8 +9,14 @@ # otherwise, so a crew that finishes (or stops and waits) without a current # working signal is never silently swallowed. A declared external-wait pause or # a firstmate-declared parked terminal task is the separate idle absorb case and -# re-surfaces only on its long bounded cadence. A new status write still surfaces -# immediately in normal mode and clears parked tracking. +# re-surfaces only on its long bounded cadence. A run the authoritative run-step +# reports stopped at a DECISION GATE still surfaces on first sight - nothing has +# relayed that gate yet - but does not ESCALATE as a wedge afterwards, since +# idling at a gate is the correct behavior; that hold lasts only as long as the +# worker's agent PROCESS is confirmed alive, because a crashed worker leaves the +# run parked identically and forever (parked_gate_liveness_class). +# A new status write still surfaces immediately in normal mode and clears parked +# tracking. # While state/.afk exists, the away daemon owns triage and this watcher queues # every actionable wake without running the more expensive normal-mode # classifiers, though signal records are collapsed to one per task in away mode @@ -26,8 +32,11 @@ # firstmate hands it to a no-mistakes validation. A declared # external-wait pause or firstmate-declared parked terminal # wait is absorbed instead with its own long re-surface -# cadence, never as a wedge. Only when neither absorb class -# applies does the log's last line decide: +# cadence, never as a wedge. A run parked at a decision gate +# whose worker is confirmed alive surfaces its first sighting +# like any other stopped crew, then holds the wedge ladder on +# the same bounded-recheck terms as an active run. Only when +# no absorb class applies does the log's last line decide: # terminal (captain-relevant) or non-terminal (no verb), # both surfaced at once. A provably-working stale past the # wedge threshold also surfaces, with an "escalation N" @@ -358,6 +367,32 @@ wake() { # below). FM_WEDGE_DEMAND_INSPECT_COUNT=${FM_WEDGE_DEMAND_INSPECT_COUNT:-3} +# ladder_hold: the one implementation of "this stale pane has a positive reason not +# to climb the wedge ladder". Refreshes the escalation timer so the reason is +# RE-READ every window rather than trusted once, and surfaces one bounded recheck +# whenever the pane has been frozen for a full PAUSE_RESURFACE_SECS - measured on +# the frozen hash's own age, not on the timer this hold keeps refreshing, so a hold +# can never postpone its own recheck. It never touches the escalation counter, so a +# held pane never climbs toward demand-deep-inspection. +# names the evidence in the wake text and the triage log; +# is what firstmate should confirm. Both callers pass their own, because "the run +# is running" and "the run is parked at a gate" need different confirmations. +ladder_hold() { # + local win=$1 since_file=$2 label=$3 situation=$4 next_step=$5 wkey hold_age rf reason + date +%s > "$since_file" + wkey=$(window_state_key "$win") + hold_age=$(age_of "$STATE/.stale-$wkey") + rf="$STATE/.wedgeheld-$wkey" + if [ "$hold_age" -ge "$PAUSE_RESURFACE_SECS" ] && [ "$(age_of "$rf")" -ge "$PAUSE_RESURFACE_SECS" ]; then + reason="stale: $win (pane unchanged ${hold_age}s while $situation - bounded recheck on a long cadence, not a wedge escalation; $next_step)" + fm_wake_append stale "$win" "$reason" || exit 1 + date +%s > "$rf" + wake "$reason" + return + fi + triage_log "absorbed $label ($situation, ladder held): $win" +} + # Repeat-poll wedge-timer bookkeeping for an already-classified stale hash # absorbed as provably-working - repairs a missing/corrupt timer (self-heals a # watcher restart between recording the hash and recording the timer), or @@ -378,8 +413,13 @@ FM_WEDGE_DEMAND_INSPECT_COUNT=${FM_WEDGE_DEMAND_INSPECT_COUNT:-3} # read is what distinguishes them, it runs once per interval rather than once per # poll, and the moment a run ends or dies the crew stops reading as working and # the next elapse escalates on the unchanged schedule. +# +# A run PARKED at a decision gate holds the ladder on the same terms, with one +# extra requirement the active-run case does not need: the gate reading is only +# trusted while agent liveness confirms a worker is still there to answer it +# (parked_gate_liveness_class). Both holds are one function - ladder_hold above. wedge_timer_check() { # - local win=$1 since_file=$2 label=$3 escalation_file=$4 since age n reason wkey hold_age rf wtask wclass + local win=$1 since_file=$2 label=$3 escalation_file=$4 since age n reason wtask wclass since=$(cat "$since_file" 2>/dev/null || true) case "$since" in ''|*[!0-9]*) @@ -391,6 +431,15 @@ wedge_timer_check() { # "$since_file" # Bounded insurance against the one way an active run can lie: a run # whose agent died mid-step keeps reporting `running` indefinitely, and # holding the ladder on that reading alone would let a dead crew rot - # invisibly. So a hold that has lasted a full PAUSE_RESURFACE_SECS - the - # fleet's existing "recheck a known-idle wait" cadence, measured on the - # frozen pane hash's own age, not on the timer this hold keeps - # refreshing - surfaces ONE bounded recheck and then waits out another - # full window. It never touches the escalation counter, so an active run - # still never climbs the wedge ladder toward demand-deep-inspection. - wkey=$(window_state_key "$win") - hold_age=$(age_of "$STATE/.stale-$wkey") - rf="$STATE/.wedgeheld-$wkey" - if [ "$hold_age" -ge "$PAUSE_RESURFACE_SECS" ] && [ "$(age_of "$rf")" -ge "$PAUSE_RESURFACE_SECS" ]; then - reason="stale: $win (pane unchanged ${hold_age}s while the run still reports active - bounded recheck on a long cadence, not a wedge escalation; confirm the run is really progressing)" - fm_wake_append stale "$win" "$reason" || exit 1 - date +%s > "$rf" - wake "$reason" - return - fi - triage_log "absorbed $label (run still active at escalation, ladder held): $win" + # invisibly - so the hold earns one bounded recheck per window. + ladder_hold "$win" "$since_file" "$label" \ + "the run still reports active" \ + "confirm the run is really progressing" + return + fi + if [ "$wclass" = parked ]; then + # The run is stopped at a decision gate AND the worker is confirmed + # alive: the pane is idle because the crew asked a question and waited, + # which is the behavior its brief requires. Hold the ladder on the same + # terms as an active run - including the bounded recheck, so a gate + # nobody ever answers still cannot rot invisibly. + ladder_hold "$win" "$since_file" "$label" \ + "the run is parked at a decision gate and this worker is still alive" \ + "answer the gate or confirm the decision is still pending" return fi n=$(( $(cat "$escalation_file" 2>/dev/null || echo 0) + 1 )) @@ -726,6 +772,56 @@ codex_static_pane_upgrade() { # printf '%s' "$fallback" } +# parked_gate_liveness_class: decide what a `parked` absorb class (crew_absorb_class +# in bin/fm-classify-lib.sh - the authoritative run-step reports the pipeline stopped +# at a decision gate) is worth on the WEDGE-ESCALATION path, by corroborating it with +# a signal that does not read interface text: the harness-agent PROCESS. +# +# Scoped to the ladder on purpose: a gate only ever holds an escalation on a stale +# pane firstmate has ALREADY been woken for. The first sighting of that pane still +# surfaces at once, whatever the run-step says, because nothing has told anyone +# about the gate yet. +# +# The gate answers the wedge question outright for the crew that is still there. A +# worker parked at an ask-user finding is idle because it did exactly what its brief +# requires - ask, then stop. The 2026-08-08 evidence: two such workers escalated as +# `idle 254s, possible wedge, escalation 1` while fm-crew-state.sh, run against the +# same tasks at the same moment, returned `state: parked - source: run-step - parked +# at review: 3 finding(s) (ask-user: captain decision)`. The fleet already knew. +# +# But `parked` alone is NOT that answer, and absorbing on it would be the failure +# this whole absorb path exists to avoid. A worker that crashed one second after +# printing its gate prompt leaves the run parked in exactly the same way, forever: +# no-mistakes has no idea its agent is gone, so the run-step reading is IDENTICAL +# for a healthy gate and a dead one. Only `alive` - a confirmed agent process, the +# one piece of evidence that changes when the worker dies - licenses the absorb: +# alive -> parked (hold the ladder on the long bounded cadence) +# dead -> none (the caller's ordinary escalation path: a run parked with +# nobody left to answer it is a real failure, and it must +# reach firstmate on the unchanged timings, never be +# swallowed by the parked case) +# unknown -> none (fm_backend_agent_alive's contract: never license an action +# from unknown. Today that covers pi, whose launcher execs +# into a generic `node`, and every backend past tmux/herdr - +# those crews keep exactly today's behavior) +# Secondmates are excluded like the codex backstop excludes them: they never drive +# a run, so a `parked` verdict cannot be theirs to begin with. +# +# COST: this adds no crew-state read anywhere. The wedge escalation already had the +# fm-crew-state.sh verdict in hand, and the only new work is one liveness probe on +# the branch where that verdict is `parked`, once per escalation window. Nothing +# here runs per poll per task, which is the budget the watcher's whole absorb design +# is built around. +parked_gate_liveness_class() { # + local win=$1 alive + [ "$(window_kind "$win")" != secondmate ] || { printf 'none'; return; } + alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || alive=unknown + case "$alive" in + alive) printf 'parked' ;; + *) printf 'none' ;; + esac +} + pause_state_class() { # local win=$1 task=$2 key last recheck_file class agent_alive key=${win//:/_} @@ -735,7 +831,24 @@ pause_state_class() { # recheck_file="$STATE/.paused-rechecked-$key" if ! status_is_paused_or_captain_held "$last"; then rm -f "$recheck_file" - codex_static_pane_upgrade "$win" "$(crew_absorb_class "$task")" + class=$(crew_absorb_class "$task") + # A gate reading never absorbs a FIRST sighting, for ANY harness: at first + # sight nothing has relayed the gate to firstmate yet (this path is reached + # precisely when the crew's last status line is not captain-relevant), so + # swallowing it would leave the decision waiting on the long bounded cadence + # with nobody told. The gate only ever holds the wedge LADDER, once the pane + # is a known stale firstmate has already seen (wedge_timer_check). + # It short-circuits BEFORE codex_static_pane_upgrade rather than being folded + # into its `none` fallback: that backstop answers "no run-step says anything, + # is the process still there?" for a codex worker mid-turn on a static pane, + # and a run-step that authoritatively reports the run STOPPED at a gate is not + # that question. Routing the gate through it would hand codex alone the + # first-sight absorb every other harness just lost. + if [ "$class" = parked ]; then + printf 'none' + return + fi + codex_static_pane_upgrade "$win" "$class" return fi if [ -e "$STATE/.paused-$key" ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then @@ -751,6 +864,12 @@ pause_state_class() { # return fi class=$(crew_absorb_class "$task") + # Below this line the crew has DECLARED a pause or a verified captain hold, and + # the branch's own dead-agent rule already owns that idle pane. The run-step gate + # reading adds nothing here and its liveness corroboration would collide with the + # declaration's, so `parked` keeps the exact `none` handling it had before the + # gate class existed; only the undeclared path above consults it. + [ "$class" = parked ] && class=none if [ "$class" = working ]; then rm -f "$recheck_file" printf 'working' diff --git a/docs/architecture.md b/docs/architecture.md index 5231c68e22..7b96558276 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,7 @@ firstmate's always-loaded operating contract and routing index for conditional p ## Event-driven supervision A zero-token bash watcher (`bin/fm-watch.sh`) sleeps on the fleet, classifies detected wakes in bash, and wakes the first mate only when something is actionable. -Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, confirmed unhealthy certsync health or cannot-run certsync status reads from the heartbeat path, stale panes whose crew is not provably working and whose captain-relevant status the signal path has not already surfaced, whether their status log looks terminal or non-terminal, absorbed stale panes that persist past `FM_STALE_ESCALATE_SECS` and no longer read as provably working at that moment, declared external waits (a crew's `paused:` status or a firstmate-declared `.parked-` marker) that remain past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. +Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, confirmed unhealthy certsync health or cannot-run certsync status reads from the heartbeat path, stale panes whose crew is not provably working and whose captain-relevant status the signal path has not already surfaced, whether their status log looks terminal or non-terminal, absorbed stale panes that persist past `FM_STALE_ESCALATE_SECS` and at that moment neither read as provably working nor sit at a decision gate a confirmed-alive worker can still answer, declared external waits (a crew's `paused:` status or a firstmate-declared `.parked-` marker) that remain past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. Repeated stale escalations on the same unchanged pane add an escalation count to the wake reason and, at `FM_WEDGE_DEMAND_INSPECT_COUNT`, a `demand-deep-inspection` marker. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) before detector state advances, so losing a model-facing delivery wait cannot lose the wake. No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when `bin/fm-crew-state.sh` reports positive evidence that the crew is still working: an actively running no-mistakes step for that crew's branch or a backend busy signature. @@ -19,6 +19,11 @@ For an ordinary crew that has stopped, the normal-mode watcher first surfaces on Live or inconclusive liveness remains fail-open at that initial surface, and the secondmate idle-endpoint exemption is unchanged. Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck. A terminal task firstmate marks parked (`bin/fm-mark-parked.sh `) after relaying its outcome and confirming only external human action remains is absorbed the same way and on the same cadence, keyed off the marker instead of a status verb; a later status write or metadata change clears the marker and returns the task to normal classification. +A mid-pipeline run that `bin/fm-crew-state.sh` reports stopped at a no-mistakes decision gate is a distinct case from that operator-declared marker, and the watcher recognizes it without one: a worker parked at an ask-user finding is idle precisely because it followed its instructions, so once its stale pane has been surfaced it holds the wedge ladder on the bounded-recheck cadence instead of escalating. +The first sighting of that pane is surfaced like any other stopped crew, because a gate the crew never wrote a captain-relevant status for has reached nobody yet, and an absorb there would leave the decision waiting silently. +That is true for every harness including codex: an authoritative run-step gate is not the no-current-state-proof case the codex static-pane backstop covers, so it never reaches that backstop and codex gets no first-sight absorb the other harnesses lack. +That hold rests on `fm_backend_agent_alive` confirming the worker's agent process and on nothing else, because a worker that crashed at the gate leaves the run parked identically and forever; a `dead` or unreadable liveness verdict keeps the unchanged escalation, and the reading is re-taken at every escalation window rather than trusted once. +A gate reported only by the append-only status log earns no hold at all, since that event is arbitrarily old and reads the same either way. Fresh stale panes use the same current-state read before trusting the status log, so an active run or busy pane outranks an old captain-relevant status-log line left behind before validation. No-change heartbeats are also benign. Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. @@ -80,7 +85,7 @@ A presence-gated sub-supervisor (`bin/fm-supervise-daemon.sh`) extends this for The watcher and daemon share `bin/fm-classify-lib.sh` for captain-relevant status verbs, declared-external-wait vocabulary, and status-scan primitives. Terminal verbs remain captain-relevant, while a nonterminal progress verb cannot become terminal merely because its prose contains a legacy free-text token such as `merged`; bare legacy free-text lines remain compatible. The always-on watcher also uses that library's absorb classification on no-verb signals and first-sighting stale panes before status-log terminality is trusted, while the daemon maintains distinct wedge and declared-pause recheck cadences. -The watcher reads that classification once more when a wedge timer would escalate, so an idle pane whose run is still active never climbs the escalation ladder, and a hold lasting a full pause-recheck window still surfaces one bounded recheck. +The watcher reads that classification once more when a wedge timer would escalate, so an idle pane whose run is still active - or parked at the decision gate above with its worker confirmed alive - never climbs the escalation ladder, and a hold lasting a full pause-recheck window still surfaces one bounded recheck. A captain-relevant status the signal path already surfaced does not wake firstmate a second time when the same crew's pane then settles stale; the suppression is keyed on the surfaced status content, never on the window, and it still starts the wedge timer. In away mode, seen-status dedupe does not clear possible-wedge aging for nonterminal progress, so housekeeping still re-escalates an unchanged idle pane at the configured bound. The daemon escalates captain-relevant events, plus a bounded recheck for a declared pause that remains idle, as one batched, single-line digest using the canonical `away-supervisor` syntax from `bin/fm-operational-input.sh`. diff --git a/docs/configuration.md b/docs/configuration.md index d5a60e7b3e..9c85ad6dab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -750,8 +750,8 @@ FM_WATCHER_STALE_GRACE=300 # defaults to FM_GUARD_GRACE; seconds a live watche FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals into one wake FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked -FM_STALE_ESCALATE_SECS=240 # idle seconds before an absorbed stale pane escalates, re-checked against the crew state at that moment: an active run holds the ladder (bounded recheck once per FM_PAUSE_RESURFACE_SECS), a crew with no active run escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb -FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon; also the cadence of the bounded recheck for a wedge ladder held by an active run +FM_STALE_ESCALATE_SECS=240 # idle seconds before an absorbed stale pane escalates, re-checked against the crew state at that moment: an active run, or a run parked at a decision gate whose worker's agent process is confirmed alive, holds the ladder (bounded recheck once per FM_PAUSE_RESURFACE_SECS), while a crew with neither escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb +FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon; also the cadence of the bounded recheck for a wedge ladder held by an active run or by a decision gate whose worker is confirmed alive FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive stale escalations on the same unchanged pane before demand-deep-inspection is added FM_WATCH_TRIAGE_LOG_MAX_BYTES=262144 # size cap for the watcher's absorbed-wake debug log FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT= # optional seconds allowed for bootstrap's best-effort clone refresh; unset/blank defaults to max(20, 5 + 3 * origin-backed-project-count) diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index c8192f5bda..94eba99086 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -268,12 +268,22 @@ test_crew_absorb_class_classifier() { ! crew_is_provably_working a || fail "a paused crew was treated as provably working" FM_FAKE_CREW_STATE='state: working · source: status-log · working: compiling' [ "$(crew_absorb_class a)" = none ] || fail "stale working: status-log classed absorbable" + FM_FAKE_CREW_STATE='state: parked · source: run-step · parked at review: 3 finding(s) (ask-user: captain decision)' + [ "$(crew_absorb_class a)" = parked ] || fail "an authoritative run-step decision gate not classed parked" + ! crew_is_provably_working a || fail "a parked run was treated as provably working" + ! crew_is_paused a || fail "a parked run was conflated with a declared external-wait pause" + ! crew_is_degraded a || fail "a parked run was conflated with an unread crew state" + # A gate known only from the append-only status log is an arbitrarily old EVENT + # that reads the same whether the crew is at the gate or died an hour ago, so it + # earns no absorb class of its own. + FM_FAKE_CREW_STATE='state: parked · source: status-log · needs-decision: pick A or B' + [ "$(crew_absorb_class a)" = none ] || fail "a status-log-sourced gate was classed absorbable" FM_FAKE_CREW_STATE='state: unknown · source: none · worktree gone' [ "$(crew_absorb_class a)" = none ] || fail "unknown crew classed absorbable" ! crew_is_paused a || fail "unknown crew classed paused" [ "$(crew_absorb_class "")" = none ] || fail "empty id not classed none" unset FM_FAKE_CREW_STATE - pass "crew_absorb_class: working/paused/none from one read; crew_is_paused and crew_is_provably_working agree" + pass "crew_absorb_class: working/paused/parked/none from one read, and the predicates keep their own meanings" } # signal_crew_provably_working: a no-verb "signal:" wake is benign ONLY when EVERY @@ -933,6 +943,272 @@ test_codex_backstop_scoped_to_codex() { pass "the agent-liveness absorb is scoped to codex: a claude worker with an alive process still surfaces immediately" } +# --- parked-at-a-decision-gate liveness gate --------------------------------- +# A worker parked at an ask-user finding is idle because it did exactly what its +# brief requires: ask, then stop. fm-crew-state.sh answers that outright +# ("state: parked · source: run-step · parked at review: 3 finding(s) (ask-user: +# captain decision)") and the watcher escalated it as a possible wedge anyway, +# every four minutes, because crew_absorb_class collapsed `parked` into `none`. +# The gate buys no silence at a FIRST sighting - that wake is how firstmate learns +# the pane is idle at all - only a hold on the wedge LADDER afterwards, corroborated +# by the agent PROCESS and by nothing else: a worker that crashed one second after +# printing its gate prompt leaves the run parked identically and forever, so `dead` +# and `unknown` keep today's escalation exactly. The first-sight rule is harness- +# independent: codex is covered explicitly, because its static-pane liveness +# backstop answers a different question (no run-step at all) and must not hand +# codex alone the absorb every other harness gives up here. +# +# The authoritative reading the watcher used to ignore. Passed into each watcher +# launch rather than exported by the fixture, because the fixture runs inside a +# command substitution and an export there would never reach the watcher. +PARKED_GATE_STATE='state: parked · source: run-step · parked at review: 3 finding(s) (ask-user: captain decision)' + +_parked_gate_case() { # [harness] + local dir state fakebin capture_file window key pane_hash sig statusf harness + dir=$(make_case "$1"); state="$dir/state"; fakebin="$dir/fakebin" + capture_file="$dir/pane.txt"; window=$2; statusf="$state/gate.status"; harness=${5:-claude} + printf '%s' "$4" > "$capture_file" + printf 'window=%s\nkind=ship\nharness=%s\n' "$window" "$harness" > "$state/gate.meta" + printf '%s\n' "$3" > "$statusf" + sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "$4") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + printf '%s\t%s\t%s\t%s\n' "$dir" "$state" "$fakebin" "$key" +} + +# The gate never buys silence on a pane nobody has been told about yet: this crew's +# last status line is not captain-relevant, so no needs-decision/blocked line ever +# reached firstmate, and a first-sight absorb would leave the decision waiting out +# the long bounded cadence unannounced. The gate only ever holds the wedge LADDER, +# from the second sighting of that already-surfaced pane onward. Driven once per +# harness, since the harness is exactly what decides whether a `none` verdict gets +# a liveness absorb of its own. +_assert_parked_gate_surfaces_then_holds() { # + local fields dir state fakebin key out drain_out window harness agent pid pane_hash + window=$2; harness=$3; agent=$4 + fields=$(_parked_gate_case "$1" "$window" 'working: running the pipeline' 'idle at the review gate' "$harness") + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out"; drain_out="$dir/drain.out"; pane_hash=$(hash_text "idle at the review gate") + + # Phase A: first sighting. A live agent at a parked gate changes nothing here. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND="$agent" FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "$harness: the first sighting of a run parked at an unrelayed gate was swallowed: $(cat "$out")" + grep -Fx "stale: $window" "$out" >/dev/null || fail "$harness: the first sighting did not print the immediate stale wake: $(cat "$out")" + [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "$harness: stale suppressor not advanced on the first-sight surface" + [ ! -e "$state/.stale-since-$key" ] || fail "$harness: the first sighting started a wedge timer instead of surfacing at once" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "$harness: drain after the parked-gate first surface failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "$harness: the first sighting's wake was not queued" + + # Phase B: same pane, already surfaced, now past the escalation threshold. THIS + # is where the gate earns its hold - the alarm firstmate has already seen must + # not climb the wedge ladder against a worker that is alive and waiting for an + # answer. + echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND="$agent" FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "$harness: a live worker parked at a gate still escalated on an already-surfaced pane: $(cat "$out")" + fi + grep -F "possible wedge" "$out" >/dev/null && { reap "$pid"; fail "$harness: the parked gate was reported as a possible wedge: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "$harness: the held ladder enqueued a wake: $(cat "$state/.wake-queue")"; } + [ ! -e "$state/.wedge-escalations-$key" ] || { reap "$pid"; fail "$harness: a parked-gate hold climbed the wedge-escalation ladder"; } + [ -s "$state/.stale-since-$key" ] || { reap "$pid"; fail "$harness: the parked-gate hold dropped the wedge timer instead of refreshing it"; } + grep -F "the run is parked at a decision gate and this worker is still alive" "$state/.watch-triage.log" >/dev/null \ + || { reap "$pid"; fail "$harness: the hold did not record WHY it believed the worker was fine: $(cat "$state/.watch-triage.log" 2>/dev/null)"; } + reap "$pid" +} + +test_parked_gate_first_sight_surfaces_then_holds_the_ladder() { + _assert_parked_gate_surfaces_then_holds parked-gate-alive "test:fm-gatealive" claude claude + pass "a run parked at an unrelayed gate surfaces its first sighting, then holds the wedge ladder while its worker is alive" +} + +# Same run, same live process, harness=codex. codex is the one harness whose +# otherwise-`none` static pane earns a liveness absorb of its own +# (codex_static_pane_upgrade), and an authoritative run-step gate must not be +# routed into it: that would give codex alone the 3600s bounded recheck where +# every other harness surfaces at once, on the very decision nobody has been told +# about. The gate short-circuits first, so codex surfaces like the rest and still +# holds the ladder afterwards. +test_parked_gate_codex_first_sight_surfaces_then_holds_the_ladder() { + _assert_parked_gate_surfaces_then_holds parked-gate-codex "test:fm-gatecodex" codex codex + pass "a codex worker parked at an unrelayed gate surfaces its first sighting too, then holds the wedge ladder" +} + +test_parked_gate_dead_surfaces() { + local fields dir state fakebin key out drain_out window pid + window="test:fm-gatedead" + fields=$(_parked_gate_case parked-gate-dead "$window" 'working: running the pipeline' 'idle at the review gate') + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out"; drain_out="$dir/drain.out" + # Identical run-step reading, agent gone (bare shell). The gate can no longer be + # answered by anyone, so the parked case must NOT swallow it. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=bash FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a run parked at a gate whose worker died was never surfaced" + grep -Fx "stale: $window" "$out" >/dev/null || fail "the dead parked worker did not print the immediate stale wake: $(cat "$out")" + [ ! -e "$state/.stale-since-$key" ] || fail "a dead parked worker started the wedge timer instead of surfacing at once" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the dead parked-worker surface failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "the dead parked worker's wake was not queued" + pass "a run parked at a gate whose worker has died surfaces immediately (the parked case never masks a crash)" +} + +# The captain's exact repeat: the crew appended needs-decision:, the signal path +# already relayed it, and each poll thereafter climbed the wedge ladder against a +# run that fm-crew-state.sh reported parked at an ask-user gate. +_parked_gate_ladder_case() { # + local fields dir state fakebin key last pane_hash + fields=$(_parked_gate_case "$1" "$2" 'needs-decision: review raised an ask-user finding for the captain' 'idle at the review gate') + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + last='needs-decision: review raised an ask-user finding for the captain' + printf '%s' "$last" > "$state/.hb-surfaced-gate" + pane_hash=$(hash_text "idle at the review gate") + printf '%s' "$pane_hash" > "$state/.stale-$key" + echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key" + printf '%s\t%s\t%s\t%s\n' "$dir" "$state" "$fakebin" "$key" +} + +test_parked_gate_alive_holds_the_wedge_ladder() { + local fields dir state fakebin key out window pid + window="test:fm-ladderalive" + fields=$(_parked_gate_ladder_case parked-gate-ladder-alive "$window") + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "a live worker parked at an ask-user gate still escalated as a possible wedge: $(cat "$out")" + fi + grep -F "possible wedge" "$out" >/dev/null && { reap "$pid"; fail "the parked gate was reported as a possible wedge: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "the held ladder still enqueued a wake: $(cat "$state/.wake-queue")"; } + [ ! -e "$state/.wedge-escalations-$key" ] || { reap "$pid"; fail "a parked-gate hold climbed the wedge-escalation ladder"; } + [ -s "$state/.stale-since-$key" ] || { reap "$pid"; fail "the parked-gate hold dropped the wedge timer instead of refreshing it"; } + grep -F "ladder held" "$state/.watch-triage.log" >/dev/null \ + || { reap "$pid"; fail "the parked-gate ladder-hold branch never executed: $(cat "$state/.watch-triage.log" 2>/dev/null)"; } + reap "$pid" + pass "a live worker parked at an ask-user gate holds the wedge ladder instead of escalating" +} + +# The failure direction that matters. Same frozen pane, same parked run-step, same +# already-relayed needs-decision line - only the agent is gone. The escalation must +# fire on the unchanged schedule; a parked case that swallowed this would be silence +# nobody notices. +test_parked_gate_dead_escalates_on_the_ladder() { + local fields dir state fakebin key out drain_out window pid + window="test:fm-ladderdead" + fields=$(_parked_gate_ladder_case parked-gate-ladder-dead "$window") + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out"; drain_out="$dir/drain.out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=bash FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a parked run whose worker died was held by the ladder instead of escalating: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null || fail "the dead parked worker did not escalate: $(cat "$out")" + grep -F "escalation 1" "$out" >/dev/null || fail "the dead parked worker escalated off the normal schedule: $(cat "$out")" + [ "$(cat "$state/.wedge-escalations-$key" 2>/dev/null || true)" = 1 ] || fail "the dead parked worker did not climb the wedge ladder" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the dead parked-worker escalation failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "the dead parked worker's escalation was not queued" + pass "a run parked at a gate whose worker died escalates on the unchanged wedge schedule" +} + +# pi's launcher execs into a generic `node`, so its liveness reads unknown +# (docs/tmux-backend.md "Known gaps"). Unknown is not evidence of anything and +# never licenses a hold: these crews keep exactly the escalation they have today. +test_parked_gate_unknown_liveness_escalates_on_the_ladder() { + local fields dir state fakebin key out window pid + window="test:fm-ladderunknown" + fields=$(_parked_gate_ladder_case parked-gate-ladder-unknown "$window") + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=node FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "an unreadable agent liveness was treated as proof the parked worker was fine: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null || fail "the unknown-liveness parked worker did not escalate: $(cat "$out")" + grep -F "escalation 1" "$out" >/dev/null || fail "the unknown-liveness parked worker escalated off the normal schedule: $(cat "$out")" + [ "$(cat "$state/.wedge-escalations-$key" 2>/dev/null || true)" = 1 ] || fail "the unknown-liveness parked worker did not climb the wedge ladder" + pass "a parked worker whose agent liveness cannot be read keeps today's escalation" +} + +# The other half of the "a stopped alarm is worse than a noisy one" rule: the hold +# is a HOLD, not silence. A gate nobody ever answers - the crew is alive and will +# wait forever - must still reach the captain on the long bounded cadence, in the +# recheck's own words rather than as a wedge, and without ever climbing the ladder. +# The anchor is the frozen hash's own mtime, which the hold refreshes nowhere, so a +# permanent hold cannot postpone its own recheck. +test_parked_gate_hold_gets_bounded_recheck() { + local fields dir state fakebin key out window pid back stale_mtime + window="test:fm-gaterot" + fields=$(_parked_gate_ladder_case parked-gate-hold-recheck "$window") + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out" + # This pane has been frozen at the gate for a full recheck window. + back=$(( $(date +%s) - 500 )) + if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$state/.stale-$key" + else touch -m -d "@$back" "$state/.stale-$key"; fi + stale_mtime=$(file_mtime "$state/.stale-$key") + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a gate held all day never got its bounded recheck: $(cat "$out")" + grep -F "bounded recheck" "$out" >/dev/null || fail "the recheck did not identify itself as a bounded recheck: $(cat "$out")" + grep -F "the run is parked at a decision gate and this worker is still alive" "$out" >/dev/null \ + || fail "the recheck did not tell the captain WHY the pane was held: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null && fail "a parked-gate recheck was mislabeled a wedge escalation: $(cat "$out")" + [ ! -e "$state/.wedge-escalations-$key" ] || fail "a parked-gate bounded recheck climbed the wedge-escalation ladder" + [ -s "$state/.wedgeheld-$key" ] || fail "the parked-gate bounded recheck did not record its throttle marker" + [ "$(file_mtime "$state/.stale-$key")" = "$stale_mtime" ] \ + || fail "the parked hold refreshed the .stale- anchor, which would postpone its own recheck forever" + grep "$(printf '\tstale\t')" "$state/.wake-queue" | grep -F "$window" >/dev/null \ + || fail "the parked-gate bounded recheck was not queued for the captain: $(cat "$state/.wake-queue")" + + # Throttled: the next poll inside the same recheck window holds again silently. + echo "$back" > "$state/.stale-since-$key" + : > "$out" + : > "$state/.wake-queue" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "the parked-gate bounded recheck repeated inside its own cadence: $(cat "$out")" + fi + [ ! -s "$state/.wake-queue" ] || fail "a throttled parked-gate recheck enqueued another wake" + [ ! -e "$state/.wedge-escalations-$key" ] || fail "a throttled parked-gate recheck climbed the wedge-escalation ladder" + reap "$pid" + pass "a parked gate nobody answers still earns one bounded recheck per window, never a wedge escalation" +} + # --- non-terminal stale, crew NOT provably working: surfaced immediately ------ # The key requirement: a crew with no running pipeline that has gone quiet (and is # not busy) has stopped - it may be done via interactive menus, waiting, or wedged. @@ -2262,6 +2538,13 @@ test_nonterminal_stale_provably_working_absorbed_then_escalated test_codex_static_pane_alive_absorbed test_codex_static_pane_dead_surfaces test_codex_backstop_scoped_to_codex +test_parked_gate_first_sight_surfaces_then_holds_the_ladder +test_parked_gate_codex_first_sight_surfaces_then_holds_the_ladder +test_parked_gate_dead_surfaces +test_parked_gate_alive_holds_the_wedge_ladder +test_parked_gate_dead_escalates_on_the_ladder +test_parked_gate_unknown_liveness_escalates_on_the_ladder +test_parked_gate_hold_gets_bounded_recheck test_terminal_stale_already_surfaced_absorbed_then_escalates test_terminal_stale_changed_line_still_surfaces test_wedge_escalation_marks_demand_deep_inspection_after_threshold