Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
4c7bf9d
docs: add implementation plan for agent-crash-resilience
danshapiro Jul 27, 2026
75985c0
docs(plan): harden agent-crash-resilience plan with load-bearing vali…
danshapiro Jul 27, 2026
626f876
docs(plan): fix fresh-eyes blocking issues — stale notice masking, cr…
danshapiro Jul 27, 2026
28ddbb0
docs(plan): fix fresh-eyes iteration-2 blocking issues — FAKE_CRASH_U…
danshapiro Jul 27, 2026
bf90aa2
docs(plan): fix fresh-eyes iteration-3 blocking issues — alias import…
danshapiro Jul 27, 2026
5abd30f
feat(ws): auto-resume policy — bounded schedule + crash eligibility d…
danshapiro Jul 27, 2026
afb87d4
feat(ws): send CrashEvent from PTY exit hook on genuine natural exits
danshapiro Jul 27, 2026
eae6952
feat(protocol): add terminal.replaced frame for server-driven crash a…
danshapiro Jul 27, 2026
4b417ab
feat(ws): respawn_agent_terminal seam — server-side resume generation…
danshapiro Jul 27, 2026
6379f24
feat(ws): auto-resume orchestrator — bounded retries, guards, recover…
danshapiro Jul 27, 2026
8a8da23
feat(client): terminal lifecycle slice + crash/replace/notice ws hand…
danshapiro Jul 27, 2026
7a9a3fa
feat(client): loud exited-pane error bar with one-click relaunch for …
danshapiro Jul 27, 2026
d119fd4
test(client): pin chime/status survival across server-driven terminal…
danshapiro Jul 27, 2026
ff0bb50
test(e2e): agent crash auto-resume, exhaustion banner, clean-exit qui…
danshapiro Jul 27, 2026
2431b7b
fix(client): terminalLifecycle selectors tolerate stores without the …
danshapiro Jul 27, 2026
013657c
fix(client): clear stale exit record on Relaunch so a failed relaunch…
danshapiro Jul 28, 2026
d103a2e
chore(ws,client): review hygiene — fail-loud logging + load-bearing c…
danshapiro Jul 28, 2026
552dd3c
merge: origin/main (83c5541f) into feat/agent-crash-resilience
danshapiro Jul 28, 2026
60a8a8f
fix(ws): integrate auto-resume respawn with relocated spawn gate (kat…
danshapiro Jul 28, 2026
41b20fa
style: cargo fmt
danshapiro Jul 28, 2026
60bfdca
fix(protocol,client): auto-resume notice carries typed fields — prose…
danshapiro Jul 28, 2026
2a4379c
fix(ws): supervise the auto-resume hub — a driver panic can no longer…
danshapiro Jul 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion crates/freshell-protocol/src/server_messages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ pub enum ServerMessage {
TerminalOutputBatch(TerminalOutputBatch),
#[serde(rename = "terminal.output.gap")]
TerminalOutputGap(TerminalOutputGap),
#[serde(rename = "terminal.replaced")]
TerminalReplaced(TerminalReplaced),
#[serde(rename = "terminal.session.associated")]
TerminalSessionAssociated(TerminalSessionAssociated),
#[serde(rename = "terminal.status")]
Expand All @@ -144,7 +146,7 @@ pub enum ServerMessage {

/// The exact `type` discriminants of every server→client message, in the frozen
/// inventory's order. This is the T0 conformance checklist.
pub const SERVER_MESSAGE_TYPES: [&str; 56] = [
pub const SERVER_MESSAGE_TYPES: [&str; 57] = [
"amplifier.activity.list.response",
"amplifier.activity.updated",
"claude.activity.list.response",
Expand Down Expand Up @@ -194,6 +196,7 @@ pub const SERVER_MESSAGE_TYPES: [&str; 56] = [
"terminal.output",
"terminal.output.batch",
"terminal.output.gap",
"terminal.replaced",
"terminal.session.associated",
"terminal.status",
"terminal.stream.changed",
Expand Down Expand Up @@ -1061,6 +1064,16 @@ pub struct TerminalOutputGap {
pub attach_request_id: Option<String>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TerminalReplaced {
pub old_terminal_id: String,
pub new_terminal_id: String,
pub exit_code: i64,
pub attempt: u32,
pub max_attempts: u32,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TerminalSessionAssociated {
Expand All @@ -1075,6 +1088,14 @@ pub struct TerminalStatus {
pub terminal_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub attempt: Option<i64>,
/// Auto-resume `recovering` frames only: the bounded retry budget. The
/// client renders attempt/maxAttempts from these FIELDS — `reason` prose
/// is purely presentational and must never be parsed (council 7w4h/xkhx).
#[serde(skip_serializing_if = "Option::is_none")]
pub max_attempts: Option<i64>,
/// Auto-resume `recovering` frames only: the crashed generation's exit code.
#[serde(skip_serializing_if = "Option::is_none")]
pub exit_code: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
}
Expand Down
25 changes: 19 additions & 6 deletions crates/freshell-protocol/tests/inventory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,27 +48,40 @@ fn server_types_match_inventory_exactly() {
let inv = inventory();
assert_eq!(
inv["serverToClient"]["count"].as_u64(),
Some(56),
"inventory declares 56 server→client types"
Some(57),
"inventory declares 57 server→client types"
);
let expected = json_type_set(&inv["serverToClient"]["types"]);
let actual: BTreeSet<String> = SERVER_MESSAGE_TYPES.iter().map(|s| s.to_string()).collect();
assert_eq!(actual.len(), 56, "crate declares 56 server types (no dups)");
assert_eq!(actual.len(), 57, "crate declares 57 server types (no dups)");
assert_eq!(
actual, expected,
"SERVER_MESSAGE_TYPES must equal the frozen inventory (no missing/extra)"
);
}

#[test]
fn combined_surface_is_85() {
fn combined_surface_is_86() {
let all = all_message_types();
assert_eq!(all.len(), 85, "29 client + 56 server = 85 discriminants");
assert_eq!(all.len(), 86, "29 client + 57 server = 86 discriminants");
// sorted + unique
let unique: BTreeSet<&str> = all.iter().copied().collect();
assert_eq!(
unique.len(),
85,
86,
"no discriminant collides across directions"
);
}

#[test]
fn terminal_replaced_roundtrips_camel_case() {
let json = r#"{"type":"terminal.replaced","oldTerminalId":"t-old","newTerminalId":"t-new","exitCode":1,"attempt":1,"maxAttempts":2}"#;
let msg: freshell_protocol::ServerMessage = serde_json::from_str(json).expect("parse");
let back = serde_json::to_string(&msg).expect("serialize");
let v: serde_json::Value = serde_json::from_str(&back).unwrap();
assert_eq!(v["type"], "terminal.replaced");
assert_eq!(v["oldTerminalId"], "t-old");
assert_eq!(v["newTerminalId"], "t-new");
assert_eq!(v["exitCode"], 1);
assert_eq!(v["maxAttempts"], 2);
}
13 changes: 13 additions & 0 deletions crates/freshell-protocol/tests/roundtrip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,19 @@ fn rich_server_messages() {
ServerMessage::TerminalTurnComplete(t) => assert_eq!(t.provider, AgentProvider::Opencode),
other => panic!("expected TerminalTurnComplete, got {other:?}"),
}

// terminal.status — auto-resume `recovering` with the typed retry-budget
// fields (council 7w4h/xkhx: attempt/maxAttempts/exitCode are FIELDS, the
// reason prose is purely presentational and never parsed by the client).
let wire = r#"{"type":"terminal.status","status":"recovering","terminalId":"t1","attempt":1,"maxAttempts":2,"exitCode":137,"reason":"claude crashed (exit 137) — auto-resuming, attempt 1/2"}"#;
match server_roundtrip(wire, "terminal.status") {
ServerMessage::TerminalStatus(s) => {
assert_eq!(s.attempt, Some(1));
assert_eq!(s.max_attempts, Some(2));
assert_eq!(s.exit_code, Some(137));
}
other => panic!("expected TerminalStatus, got {other:?}"),
}
}

#[test]
Expand Down
15 changes: 15 additions & 0 deletions crates/freshell-server/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,13 @@ async fn main() -> ExitCode {
fresh_opencode_state.set_identity_sink(fresh_agent_identity_sink.clone());
// opencode REST surface (Task 7's materialization site; V10 A13-N1)
fresh_agent_state.set_identity_sink(fresh_agent_identity_sink.clone());
// Lane D1: the crash-event channel for terminal auto-resume. The receiver
// is consumed by `auto_resume::spawn_auto_resume_hub`, spawned right
// after `ws_state` is assembled (the hub needs the full state).
let (auto_resume_tx, auto_resume_rx) =
tokio::sync::mpsc::unbounded_channel::<freshell_ws::auto_resume::CrashEvent>();
let ws_state = WsState {
auto_resume_tx,
activity: Some(activity_hub.clone()),
identity: terminal_identity.clone(),
amplifier_locator: amplifier_locator.clone(),
Expand Down Expand Up @@ -627,6 +633,15 @@ async fn main() -> ExitCode {
pane_ledger: std::sync::Arc::clone(&pane_ledger),
};

// Lane D1 (Task 5): the auto-resume hub — consumes the crash events the
// PTY exit hook sends and drives bounded respawns. A boot-time background
// task, same precedent as `spawn_idle_monitor` above. The handle is
// deliberately discarded: the hub SELF-SUPERVISES (council 7w4h/xkhx,
// crusty) — a driver panic is caught inside the task, logged ERROR, and
// the loop restarted with bounded escalating backoff, so the task only
// ever ends when the crash-event channel closes at shutdown.
freshell_ws::auto_resume::spawn_auto_resume_hub(ws_state.clone(), auto_resume_rx);

// P1.8 boot hygiene: quarantine, stale-marker sweep, supersession
// repair, GC. Tombstone deletion keys on the DIRECT stat
// (`transcript_definitively_absent`) — never on probe.exists()==Absent
Expand Down
1 change: 1 addition & 0 deletions crates/freshell-ws/src/amplifier_association.rs
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@ mod tests {
.unwrap(),
),
broadcast_tx: StdArc::clone(&broadcast_tx),
auto_resume_tx: tokio::sync::mpsc::unbounded_channel().0,
fresh_codex: freshell_freshagent::FreshCodexState::new(
StdArc::clone(&auth_token),
StdArc::clone(&broadcast_tx),
Expand Down
Loading
Loading