fix: fail over unhealthy Daytona sandbox hosts

apps/agent-worker/README.md

@@ -81,6 +81,11 @@ provider keys and sandbox capabilities are reacquired inside the active step and
 Workflow storage. A Worker isolate or Durable Object eviction therefore resumes from the last
 completed step instead of losing an in-memory coroutine. Transcript publication uses deterministic
 event keys and an atomic SQLite receipt, so Workflow step replay cannot duplicate visible parts.
+Preparation uses a multi-minute durable exponential-backoff window for transient provider
+failures. Daytona's explicit host-recovery start rejection is treated as a runtime failover signal:
+after the active-run lease and canonical volume mount are verified, the stopped container is
+replaced on the same isolated workspace-volume subpath. This preserves user files while avoiding
+an indefinite dependency on one unhealthy runner.
 There is no application step, token, duration, or cost ceiling; semantic completion ends the loop,
 while per-operation timeouts and the platform Workflow limit remain operational safeguards.
 The Worker pins Cloudflare's paid-plan maximum subrequest allowance because external provider,

apps/agent-worker/src/durable-objects/agent-run-workflow.ts

@@ -47,7 +47,10 @@ import {
   WorkflowToolStepResultSchema,
 } from "./agent-run-workflow-runtime";
 
-const PREPARE_STEP = stepConfig("10 minutes", 3);
+// A stopped Daytona sandbox can temporarily reject starts while its host recovers.
+// Keep that provider recovery inside the durable preparation step so a transient
+// host event does not become a user-visible failed run.
+const PREPARE_STEP = stepConfig("10 minutes", 6);
 const MODEL_STEP = stepConfig("5 minutes", 3);
 const TOOL_STEP = stepConfig("15 minutes", 2);
 const STATE_STEP = stepConfig("2 minutes", 5);

apps/agent-worker/src/durable-objects/project-sandbox-runtime-handle.ts

@@ -1,4 +1,8 @@
-import { DaytonaClient, type DaytonaSandbox } from "@cheatcode/agent-core/tools/code";
+import {
+  DaytonaClient,
+  type DaytonaSandbox,
+  isDaytonaHostRecoveryStartError,
+} from "@cheatcode/agent-core/tools/code";
 import { previewHostnameForWorker, resolveWorkerSecret } from "@cheatcode/env";
 import { APIError, createLogger } from "@cheatcode/observability";
 import { performAccountDeletion } from "./project-sandbox-account-deletion";
@@ -59,6 +63,8 @@ interface RuntimeState {
   workspaceState: ProjectSandboxWorkspaceState | undefined;
 }
 
+type RuntimeReplacementReason = "configuration_changed" | "daytona_host_recovery";
+
 interface SandboxLeaseRuntime {
   withCleanupSignal: <Result>(operation: () => Promise<Result>) => Promise<Result | undefined>;
   withOwnerRegistration: <Result>(
@@ -265,11 +271,21 @@ async function ensureSandbox(state: RuntimeState, startingRunId?: string): Promi
   if (state.cache.sandboxId && Date.now() - state.cache.startedVerifiedAtMs < STARTED_REVERIFY_MS) {
     return state.cache.sandboxId;
   }
-  const resolved = await inspectSandbox(state);
-  if (typeof resolved === "string") {
-    return resolved;
+  try {
+    const resolved = await inspectSandbox(state);
+    if (typeof resolved === "string") {
+      return resolved;
+    }
+    return replaceSandboxRuntime(state, startingRunId, "configuration_changed");
+  } catch (error) {
+    if (!isDaytonaHostRecoveryStartError(error)) {
+      throw error;
+    }
+    createLogger().warn("sandbox_host_recovery_failover_started", {
+      sandboxId: state.identity.sandboxName(),
+    });
+    return replaceSandboxRuntime(state, startingRunId, "daytona_host_recovery");
   }
-  return replaceSandboxRuntime(state, startingRunId);
 }
 
 async function restartSandboxForWorkspaceRecovery(
@@ -376,7 +392,11 @@ async function activateResolvedSandbox(
   return resolved.id;
 }
 
-async function replaceSandboxRuntime(state: RuntimeState, startingRunId?: string): Promise<string> {
+async function replaceSandboxRuntime(
+  state: RuntimeState,
+  startingRunId: string | undefined,
+  reason: RuntimeReplacementReason,
+): Promise<string> {
   if (state.isSandboxRuntimeUpdateInProgress) {
     throw sandboxRuntimeUpdatePending(state.env.DAYTONA_SANDBOX_SNAPSHOT);
   }
@@ -391,8 +411,8 @@ async function replaceSandboxRuntime(state: RuntimeState, startingRunId?: string
       let resolved: DaytonaSandbox;
       try {
         resolved = await state.provisioning.resolve(daytona);
-        if (!state.provisioning.isDesired(resolved)) {
-          resolved = await replaceSandboxRuntimeExclusive(state, daytona, resolved);
+        if (reason === "daytona_host_recovery" || !state.provisioning.isDesired(resolved)) {
+          resolved = await replaceSandboxRuntimeExclusive(state, daytona, resolved, reason);
         }
       } catch (error) {
         throw toUpstreamError(
@@ -412,6 +432,7 @@ async function replaceSandboxRuntimeExclusive(
   state: RuntimeState,
   daytona: DaytonaClient,
   current: DaytonaSandbox,
+  reason: RuntimeReplacementReason,
 ): Promise<DaytonaSandbox> {
   state.provisioning.assertRuntimeReplacementSafe(current);
   await prepareForSandboxReplacement(state);
@@ -421,6 +442,7 @@ async function replaceSandboxRuntimeExclusive(
     throw sandboxRuntimeUpdatePending(state.env.DAYTONA_SANDBOX_SNAPSHOT);
   }
   createLogger().info("sandbox_runtime_replaced", {
+    reason,
     sandboxId: state.identity.sandboxName(),
     snapshot: state.env.DAYTONA_SANDBOX_SNAPSHOT,
   });

packages/agent-core/src/tools/code/daytona-client.ts

@@ -34,6 +34,8 @@ const DAYTONA_FILE_LIST_MAX_ITEMS = 1_000;
 const DAYTONA_SANDBOX_PAGE_MAX_ITEMS = 100;
 const DAYTONA_SESSION_COMMAND_MAX_ITEMS = 1_000;
 const DAYTONA_VOLUME_NAME_MAX_CHARACTERS = 100;
+const DAYTONA_HOST_RECOVERY_START_MESSAGE =
+  "sandbox start is temporarily unavailable while the sandbox's host recovers";
 
 interface DaytonaClientConfig {
   apiKey: string;
@@ -66,6 +68,22 @@ export class DaytonaApiError extends Error {
   }
 }
 
+/** Identifies Daytona's host-local start rejection so callers can fail over safely. */
+export function isDaytonaHostRecoveryStartError(error: unknown): boolean {
+  let current = error;
+  for (let depth = 0; depth < 3; depth += 1) {
+    if (
+      current instanceof DaytonaApiError &&
+      current.status === 503 &&
+      current.message.toLowerCase().includes(DAYTONA_HOST_RECOVERY_START_MESSAGE)
+    ) {
+      return true;
+    }
+    current = current instanceof Error ? current.cause : undefined;
+  }
+  return false;
+}
+
 // ---------------------------------------------------------------------------
 // Response schemas project provider payloads down to fields used by the runtime.
 // ---------------------------------------------------------------------------

packages/agent-core/src/tools/code/index.ts

@@ -5,7 +5,11 @@ export type {
   DaytonaVolume,
   SandboxDestroyResult,
 } from "./daytona-client";
-export { DaytonaApiError, DaytonaClient } from "./daytona-client";
+export {
+  DaytonaApiError,
+  DaytonaClient,
+  isDaytonaHostRecoveryStartError,
+} from "./daytona-client";
 
 export {
   DeleteFileInputSchema,