Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions bin/fm-crew-state.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
# no-endpoint-recorded the meta records no backend target
# endpoint-unreadable the target did not answer and its CLI is installed
# kind-skips-run-lookup kind=scout/secondmate, which never drive a run
# delivery-mode-skips-run-lookup kind=ship but mode=direct-PR/local-only, which never drive a run
# no-branch kind=ship at detached HEAD, so no run can be attributed
# no-run-attributed the run lookup answered and named no run for this crew
# run-attribution-rejected a run WAS found and this reader refused it
Expand Down Expand Up @@ -80,7 +81,10 @@
# is not a reading and must not be passed off as one.
#
# Logic, in order:
# 1. Resolve worktree + backend target + kind from state/<id>.meta.
# 1. Resolve worktree + backend target + kind + delivery mode from state/<id>.meta.
# A kind=ship task whose mode is not no-mistakes (direct-PR, local-only)
# never starts a no-mistakes run at all, so the lookup in step 2 is skipped
# for it exactly like it is for kind=scout/secondmate.
# 2. Matching no-mistakes run for this crew's branch AND current code identity,
# active or terminal (from `axi status`, or the coarse `no-mistakes runs`
# fallback)? Branch name alone is not enough: a historical run on a reused
Expand Down Expand Up @@ -191,6 +195,14 @@ meta_value() { # <key>
WT=$(meta_value worktree)
KIND=$(meta_value kind)
[ -n "$KIND" ] || KIND=ship
# Delivery mode (bin/fm-project-mode.sh: no-mistakes|direct-PR|local-only, or
# secondmate for kind=secondmate). Only no-mistakes drives a no-mistakes run;
# direct-PR and local-only push straight to a branch/PR and never create one.
# Absent mode= (older meta, or a lookup ahead of fm-spawn recording it) defaults
# to no-mistakes, matching fm-spawn's own "mode=${MODE:-no-mistakes}" fallback -
# a missing field must not silently start skipping a real run's lookup.
DELIVERY_MODE=$(meta_value mode)
[ -n "$DELIVERY_MODE" ] || DELIVERY_MODE=no-mistakes

# A torn-down (or never-created) worktree has no current state to read. The two
# ways to get here are different facts about the crew record: a meta that never
Expand Down Expand Up @@ -650,6 +662,12 @@ if [ "$KIND" != ship ]; then
# directly. Nothing failed here, and the cause says so.
UNKNOWN_CAUSE=kind-skips-run-lookup
UNKNOWN_REASON="kind=$KIND never drives a run"
elif [ "$DELIVERY_MODE" != no-mistakes ]; then
# A direct-PR or local-only ship task pushes straight to a branch/PR itself and
# never starts a no-mistakes run, so there is nothing here to reconcile against -
# this is a known, answerable shape, not a failed read.
UNKNOWN_CAUSE=delivery-mode-skips-run-lookup
UNKNOWN_REASON="mode=$DELIVERY_MODE never drives a run"
elif [ "$GIT_MISSING" = 1 ]; then
READER_MISSING=1
READER_CAUSE=run-reader-missing
Expand All @@ -664,7 +682,7 @@ elif ! command -v no-mistakes >/dev/null 2>&1; then
READER_SOURCE=missing-dependency
READER_WHY='no-mistakes CLI not on PATH'
fi
if [ "$KIND" = ship ] && [ -n "$CREW_BRANCH" ] && [ "$READER_MISSING" = 0 ]; then
if [ "$KIND" = ship ] && [ "$DELIVERY_MODE" = no-mistakes ] && [ -n "$CREW_BRANCH" ] && [ "$READER_MISSING" = 0 ]; then
nm_rc=0
RUN_OUT=$(nm_run axi status) || nm_rc=$?
if [ "$nm_rc" = "$NM_RUN_RC_UNBOUNDED" ]; then
Expand Down
3 changes: 2 additions & 1 deletion docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ Routine watcher polling, supervision no-ops, elapsed waiting time, and absorbed
A declared external wait trades that silence for one bounded recheck per pause window, so a forgotten pause cannot remain invisible indefinitely.
Crew status files are append-only wake-event logs, not current-state fields.
`bin/fm-crew-state.sh <id>` is the cheap current-state read for an actionable heartbeat review: it attributes a no-mistakes run, active or terminal, only when it matches the crew's branch and current code identity, treating an active run's head that this worktree has not received yet as a pipeline-owned commit rather than a mismatch, then keeps that run-step authoritative even if the pane has closed.
For ship tasks, it consults no-mistakes runs only when the recorded delivery mode is `no-mistakes`; `direct-PR` and `local-only` ship tasks skip that source because those modes never start a no-mistakes run.
The script header owns the exact run-head ancestry rules.
During no-mistakes' `ci` monitor phase, it also reads the ci step log tail because `axi status` reports both "still waiting on checks" and "checks green, waiting on merge" as `ci,running`.
The most recent recognized ci log marker wins, so checks-green monitoring reports done while a later re-arm, failed-check, or issue marker returns the crew to working.
Only when no matching run exists does it fall back to the pane busy-signature and then a status-log event whose verb maps to a recognized run-state; a dead pane without a run reports unknown instead of trusting a stale log.
When no run lookup applies, or no matching run exists, it falls back to the pane busy-signature and then a status-log event whose verb maps to a recognized run-state; a dead pane without a run reports unknown instead of trusting a stale log.
Decision-only events such as `resolved` never become current state or leak their prose into the current-state detail.
In that status-log fallback, a declared external wait reports the distinct `paused` state with its reason.
A source that could not be consulted at all reports the distinct `degraded` state, never an ordinary `unknown`: the reader has no standing to make a claim about a crew it could not read, a busy pane and a declared pause are the only lower answers that survive without it, and the watcher reports that broken instrument on the pause cadence instead of climbing the wedge ladder against a reading nobody took.
Expand Down
67 changes: 67 additions & 0 deletions tests/fm-crew-state.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
# (g) no run + idle pane falls to the status-log verb -> status-log
# (h) dead pane: no run -> unknown/none; with a run -> run-step (not the shell)
# (i) kind=scout skips the run lookup -> pane/status-log
# (i2) kind=ship but mode=direct-PR/local-only also skips it -> pane/status-log
# (j) torn-down worktree / missing meta -> unknown/none
# (k) crew_is_provably_working end-to-end over the REAL helper (not a canned
# fake fm-crew-state.sh verdict): cross-branch attribution via the runs
Expand Down Expand Up @@ -1124,6 +1125,56 @@ test_scout_skips_run_lookup() {
pass "scout skips the run lookup"
}

# (i2) kind=ship but mode=direct-PR/local-only never starts a no-mistakes run
# either, exactly like kind=scout/secondmate: 2026-08-04/05 incident, confirmed
# 2 of 2 on two different harnesses. A direct-PR task pushes a branch and opens
# a PR itself; there is no run to reconcile against, so the lookup must be
# skipped rather than attempted and reported broken.
test_direct_pr_mode_skips_run_lookup() {
reset_fakes
local d out
d=$(new_case direct-pr-skip)
make_repo_on_branch "$d/wt" fm/feat-direct-pr
make_fakebin "$d" >/dev/null
fm_write_meta "$d/state/direct-pr.meta" "window=fm:fm-direct-pr" "worktree=$d/wt" "kind=ship" "mode=direct-PR"
# Even if a run happened to exist for this branch, a direct-PR task must not
# read it: no-mistakes never ran for it in the first place.
FM_FAKE_AXI_STATUS="$(run_running fm/feat-direct-pr)"
FM_FAKE_BUSY=1
out=$(run_crew_state "$d" direct-pr)
assert_not_contains "$out" "source: run-step" "direct-PR mode ignores no-mistakes run-step"
assert_contains "$out" "source: pane" "direct-PR mode reads pane busy-signature"
pass "mode=direct-PR skips the run lookup"
}

# Direct regression test for the reported defect: a finished direct-PR task
# whose worktree was never initialized for no-mistakes (so a run lookup would
# fail outright, exactly as the incident captured) must read a determinate
# done state from its own status log, never `degraded`.
test_direct_pr_finished_task_is_done_not_degraded() {
reset_fakes
local d out
d=$(new_case direct-pr-done)
make_repo_on_branch "$d/wt" fm/feat-direct-pr-done
make_fakebin "$d" >/dev/null
# A no-mistakes CLI that is present but broken: this is the exact shape of the
# incident (`the no-mistakes runs lookup ran and returned nothing`) if the
# mode check did not skip the call for this task.
cat > "$d/fakebin/no-mistakes" <<'SH'
#!/usr/bin/env bash
exit 7
SH
chmod +x "$d/fakebin/no-mistakes"
fm_write_meta "$d/state/direct-pr-done.meta" "window=fm:fm-direct-pr-done" "worktree=$d/wt" "kind=ship" "mode=direct-PR"
FM_FAKE_BUSY=0
printf 'done: PR https://github.com/example/repo/pull/1\n' > "$d/state/direct-pr-done.status"
out=$(run_crew_state "$d" direct-pr-done)
assert_contains "$out" "state: done" "a finished direct-PR task did not read as done"
assert_contains "$out" "source: status-log" "a finished direct-PR task's done state did not come from its status log"
assert_not_contains "$out" "degraded" "a finished direct-PR task with nothing to reconcile must not read as degraded"
pass "a finished direct-PR task reads done, not degraded"
}

# (j) torn-down worktree and missing meta are graceful (unknown/none, exit 0)
test_torn_down_worktree() {
reset_fakes
Expand Down Expand Up @@ -1456,6 +1507,19 @@ test_cause_kind_skips_run_lookup() {
pass "cause kind-skips-run-lookup"
}

test_cause_delivery_mode_skips_run_lookup() {
reset_fakes
local d out
d=$(new_case cause-mode-skips)
make_repo_on_branch "$d/wt" fm/feat-direct-pr-skip
make_fakebin "$d" >/dev/null
fm_write_meta "$d/state/direct-pr-skip.meta" "window=fm:fm-direct-pr-skip" "worktree=$d/wt" "kind=ship" "mode=direct-PR"
FM_FAKE_BUSY=0
out=$(run_crew_state "$d" direct-pr-skip)
assert_cause "$out" delivery-mode-skips-run-lookup "a delivery mode that never drives a run was collapsed with a crew whose run lookup found nothing"
pass "cause delivery-mode-skips-run-lookup"
}

test_cause_no_branch() {
reset_fakes
local d out
Expand Down Expand Up @@ -1784,6 +1848,8 @@ test_dead_window_still_reports_terminal_run_step
test_dead_window_still_reports_active_run_step
test_no_timeout_uses_perl_bound
test_scout_skips_run_lookup
test_direct_pr_mode_skips_run_lookup
test_direct_pr_finished_task_is_done_not_degraded
test_torn_down_worktree
test_missing_meta
test_provably_working_via_runs_list_fallback
Expand All @@ -1810,6 +1876,7 @@ test_cause_worktree_gone
test_cause_no_endpoint_recorded
test_cause_endpoint_unreadable
test_cause_kind_skips_run_lookup
test_cause_delivery_mode_skips_run_lookup
test_cause_no_branch
test_cause_no_run_attributed
test_cause_run_attribution_rejected
Expand Down
Loading