fix: recover wedged turns when provider event stream stalls (#46)

src/config.ts

@@ -215,8 +215,45 @@ const SessionSchema = z
   .object({
     defaultModel: z.string().optional(),
     fallbackModel: z.string().optional(),
+    /**
+     * Hard backstop against a wedged turn. If the provider event stream goes
+     * completely silent (no events at all) for this many ms while a turn is
+     * active, the turn is treated as stalled: the run is torn down, the
+     * subprocess reaped, status reset to idle, and a clear message shown.
+     * Generous by default — long-running tools still emit `tool_progress` /
+     * partial events, so true silence for this long is a reliable hang signal.
+     * Set to 0 to disable the watchdog.
+     */
+    turnStallTimeoutMs: z.number().min(0).default(300_000),
+    /**
+     * Per-call wall-clock timeout (ms) applied to external (user-configured)
+     * MCP servers, surfaced to the SDK as each server's `timeout`. A hung MCP
+     * tool call (e.g. an unresponsive HTTP gateway) then returns an SDK error
+     * the turn loop can act on, instead of going silent. Kept BELOW
+     * turnStallTimeoutMs so it fires first — the stall watchdog stays a coarse
+     * last-resort backstop. 0 = don't set (use the SDK default). Does not apply
+     * to codeoid's in-process memory server.
+     */
+    mcpToolTimeoutMs: z.number().min(0).default(120_000),
   })
-  .default({});
+  .default({ turnStallTimeoutMs: 300_000, mcpToolTimeoutMs: 120_000 })
+  // Enforce the "SDK signals first" contract across BOTH fields — not just the
+  // defaults. An env override / config file could otherwise set the MCP timeout
+  // at or above the stall timeout, so the coarse watchdog would force-recover
+  // before the SDK's clean per-tool error fires. Exempt the opt-out cases:
+  // turnStallTimeoutMs=0 (watchdog off → nothing to race) and mcpToolTimeoutMs=0
+  // (use SDK default → relationship is moot).
+  .refine(
+    (s) =>
+      s.turnStallTimeoutMs === 0 ||
+      s.mcpToolTimeoutMs === 0 ||
+      s.mcpToolTimeoutMs < s.turnStallTimeoutMs,
+    {
+      message:
+        "must be less than session.turnStallTimeoutMs so a hung MCP call surfaces an SDK error before the stall watchdog fires (set either to 0 to opt out)",
+      path: ["mcpToolTimeoutMs"],
+    },
+  );
 
 const AutoRotateSchema = z
   .object({
@@ -367,6 +404,10 @@ export interface CodeoidConfig {
   session: {
     defaultModel?: string;
     fallbackModel?: string;
+    /** Stall watchdog: ms of total event-stream silence before a turn is force-recovered (0 = off). Defaults to 300000 when omitted. */
+    turnStallTimeoutMs?: number;
+    /** Per-call timeout (ms) for external MCP servers, surfaced as the SDK's per-server `timeout`. 0 = use SDK default. Defaults to 120000 when omitted. */
+    mcpToolTimeoutMs?: number;
   };
 }
 
@@ -420,6 +461,8 @@ const ENV_OVERRIDES: readonly EnvOverride[] = [
   { env: "CODEOID_AUTO_ROTATE_MIN_TURNS", path: "autoRotate.minTurnsBeforeRotate", kind: "int" },
   { env: "CODEOID_DEFAULT_MODEL", path: "session.defaultModel", kind: "string" },
   { env: "CODEOID_FALLBACK_MODEL", path: "session.fallbackModel", kind: "string" },
+  { env: "CODEOID_TURN_STALL_TIMEOUT_MS", path: "session.turnStallTimeoutMs", kind: "int" },
+  { env: "CODEOID_MCP_TOOL_TIMEOUT_MS", path: "session.mcpToolTimeoutMs", kind: "int" },
 ];
 
 // ── Loading ──────────────────────────────────────────────────────────────
@@ -481,6 +524,23 @@ export function loadConfig(opts: LoadOptions = {}): CodeoidConfig {
     setByPath(parsed, ov.path, parseOverride(raw, ov.kind));
   }
 
+  // 3a. Re-validate after overrides. parseOverride() coerces strings to the
+  //     declared kind but does NOT enforce schema constraints (e.g. the
+  //     non-negative bound on session.turnStallTimeoutMs, or the 0..1 bounds on
+  //     the autoRotate percentages). Without this, CODEOID_TURN_STALL_TIMEOUT_MS=-1
+  //     would slip through and silently disable the stall watchdog. Re-running
+  //     RootSchema over the merged result fails fast on any out-of-range override.
+  const revalidated = RootSchema.safeParse(parsed);
+  if (!revalidated.success) {
+    const issues = revalidated.error.issues
+      .map((i) => `  ${i.path.join(".")}: ${i.message}`)
+      .join("\n");
+    throw new Error(
+      `Invalid config after applying environment overrides:\n${issues}\n(Check the corresponding CODEOID_* env vars.)`,
+    );
+  }
+  Object.assign(parsed, revalidated.data);
+
   // 3b. Resolve the ZeroID issuer (preset name or URL → concrete base URL) and
   //     pin the expected issuer claim. Every ZeroID deployment sets `iss` to
   //     its base URL, so defaulting auth.issuer to the resolved URL rejects

src/daemon/providers/claude/index.ts

@@ -236,8 +236,13 @@ export class ClaudeProvider implements SessionProvider {
       : { sessionId: this.#claudeCodeSessionId };
 
     // Merge user MCP servers with codeoid's in-process memory server.
+    // Apply a per-call wall-clock timeout to the external (user) servers so a
+    // hung MCP tool call surfaces as an SDK error instead of silently wedging
+    // the turn. Not applied to the in-process memory server. Kept below the
+    // session stall watchdog so the SDK signals first.
+    const mcpToolTimeoutMs = init.config?.session.mcpToolTimeoutMs ?? 120_000;
     const merged: Record<string, McpServerConfig> = {
-      ...loadUserMcpServers(opts.workdir),
+      ...withMcpToolTimeout(loadUserMcpServers(opts.workdir), mcpToolTimeoutMs),
       ...(init.memory
         ? {
             codeoid_memory: buildMemoryMcpServer(init.memory, {
@@ -621,6 +626,30 @@ export function translateSDKMessage(
 
 // ── Helpers ───────────────────────────────────────────────────────────────────
 
+/**
+ * Apply a per-call wall-clock `timeout` (ms) to external MCP servers so a hung
+ * tool call (e.g. an unresponsive HTTP gateway) returns an SDK error instead of
+ * silently stalling the turn. Only sets it when `ms > 0` and the server hasn't
+ * already declared its own `timeout`, so explicit per-server values still win.
+ * `timeout` is valid on every external (stdio/http/sse) McpServerConfig variant;
+ * these all come from JSON, so we re-cast at the same boundary parseMcpServerConfig uses.
+ */
+export function withMcpToolTimeout(
+  servers: Record<string, McpServerConfig>,
+  ms: number,
+): Record<string, McpServerConfig> {
+  if (ms <= 0) return servers;
+  const out: Record<string, McpServerConfig> = {};
+  for (const [name, cfg] of Object.entries(servers)) {
+    const obj = cfg as unknown as Record<string, unknown>;
+    out[name] =
+      typeof obj.timeout === "number"
+        ? cfg
+        : ({ ...obj, timeout: ms } as unknown as McpServerConfig);
+  }
+  return out;
+}
+
 function loadUserMcpServers(workdir: string): Record<string, McpServerConfig> {
   try {
     const raw = readFileSync(join(homedir(), ".claude.json"), "utf8");

src/daemon/providers/mock/session-provider.ts

@@ -44,18 +44,26 @@ export class MockSessionProvider implements SessionProvider {
   #backingSessionId: string;
   #hasQueried = false;
   #script: ProviderEvent[][];
+  /** When true, runTurn() emits its scripted events then leaves the queue OPEN
+   *  (never closes, never emits a terminal turn_done) — simulating a provider
+   *  whose stream has gone silent (hung tool / dead subprocess). The queue is
+   *  only closed by teardown(), mirroring ClaudeProvider. */
+  #stall: boolean;
+  /** Live turn queue, so teardown() can unblock a waiting consumer like the real provider. */
+  #currentQueue: AsyncQueue<ProviderEvent> | null = null;
 
   /** Every TurnOpts passed to runTurn() — inspect in tests. */
   readonly capturedOpts: TurnOpts[] = [];
 
   /** Incremented each time teardown() is called — useful for asserting cleanup. */
   teardownCount = 0;
 
-  constructor(id = "mock-session", script: ProviderEvent[][] = []) {
+  constructor(id = "mock-session", script: ProviderEvent[][] = [], opts: { stall?: boolean } = {}) {
     this.id = id;
     this.displayName = `MockSession(${id})`;
     this.#backingSessionId = `${id}-backing`;
     this.#script = script.map((s) => [...s]);
+    this.#stall = opts.stall ?? false;
   }
 
   get backingSessionId(): string { return this.#backingSessionId; }
@@ -74,6 +82,10 @@ export class MockSessionProvider implements SessionProvider {
   async teardown(): Promise<void> {
     this.teardownCount++;
     this.onRecoveryNeeded = undefined;
+    // Mirror ClaudeProvider: closing the live turn queue unblocks any consumer
+    // currently awaiting the next event (e.g. a stalled run being recovered).
+    this.#currentQueue?.close();
+    this.#currentQueue = null;
   }
 
   async dispose(): Promise<void> {
@@ -93,6 +105,7 @@ export class MockSessionProvider implements SessionProvider {
     ];
 
     const queue = new AsyncQueue<ProviderEvent>();
+    this.#currentQueue = queue;
 
     // Emit events asynchronously, calling canUseTool for each tool_start
     // to simulate the SDK's PreToolUse hook firing before the tool runs.
@@ -135,6 +148,11 @@ export class MockSessionProvider implements SessionProvider {
       }
     }
 
+    // Stall mode: emit the scripted events, then leave the queue OPEN (no
+    // terminal event, no close) so the consumer's next pull blocks — exactly
+    // what a hung provider stream looks like. Only teardown() closes it.
+    if (this.#stall) return;
+
     queue.close();
   }
 }