diff --git a/apps/agent-worker/README.md b/apps/agent-worker/README.md index 89be442..becda76 100644 --- a/apps/agent-worker/README.md +++ b/apps/agent-worker/README.md @@ -81,6 +81,11 @@ provider keys and sandbox capabilities are reacquired inside the active step and Workflow storage. A Worker isolate or Durable Object eviction therefore resumes from the last completed step instead of losing an in-memory coroutine. Transcript publication uses deterministic event keys and an atomic SQLite receipt, so Workflow step replay cannot duplicate visible parts. +Preparation uses a multi-minute durable exponential-backoff window for transient provider +failures. Daytona's explicit host-recovery start rejection is treated as a runtime failover signal: +after the active-run lease and canonical volume mount are verified, the stopped container is +replaced on the same isolated workspace-volume subpath. This preserves user files while avoiding +an indefinite dependency on one unhealthy runner. There is no application step, token, duration, or cost ceiling; semantic completion ends the loop, while per-operation timeouts and the platform Workflow limit remain operational safeguards. The Worker pins Cloudflare's paid-plan maximum subrequest allowance because external provider, diff --git a/apps/agent-worker/src/durable-objects/agent-run-workflow.ts b/apps/agent-worker/src/durable-objects/agent-run-workflow.ts index 5fc2abd..fec0a61 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-workflow.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-workflow.ts @@ -47,7 +47,10 @@ import { WorkflowToolStepResultSchema, } from "./agent-run-workflow-runtime"; -const PREPARE_STEP = stepConfig("10 minutes", 3); +// A stopped Daytona sandbox can temporarily reject starts while its host recovers. +// Keep that provider recovery inside the durable preparation step so a transient +// host event does not become a user-visible failed run. +const PREPARE_STEP = stepConfig("10 minutes", 6); const MODEL_STEP = stepConfig("5 minutes", 3); const TOOL_STEP = stepConfig("15 minutes", 2); const STATE_STEP = stepConfig("2 minutes", 5); diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-runtime-handle.ts b/apps/agent-worker/src/durable-objects/project-sandbox-runtime-handle.ts index f69107e..f29bcb5 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-runtime-handle.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-runtime-handle.ts @@ -1,4 +1,8 @@ -import { DaytonaClient, type DaytonaSandbox } from "@cheatcode/agent-core/tools/code"; +import { + DaytonaClient, + type DaytonaSandbox, + isDaytonaHostRecoveryStartError, +} from "@cheatcode/agent-core/tools/code"; import { previewHostnameForWorker, resolveWorkerSecret } from "@cheatcode/env"; import { APIError, createLogger } from "@cheatcode/observability"; import { performAccountDeletion } from "./project-sandbox-account-deletion"; @@ -59,6 +63,8 @@ interface RuntimeState { workspaceState: ProjectSandboxWorkspaceState | undefined; } +type RuntimeReplacementReason = "configuration_changed" | "daytona_host_recovery"; + interface SandboxLeaseRuntime { withCleanupSignal: (operation: () => Promise) => Promise; withOwnerRegistration: ( @@ -265,11 +271,21 @@ async function ensureSandbox(state: RuntimeState, startingRunId?: string): Promi if (state.cache.sandboxId && Date.now() - state.cache.startedVerifiedAtMs < STARTED_REVERIFY_MS) { return state.cache.sandboxId; } - const resolved = await inspectSandbox(state); - if (typeof resolved === "string") { - return resolved; + try { + const resolved = await inspectSandbox(state); + if (typeof resolved === "string") { + return resolved; + } + return replaceSandboxRuntime(state, startingRunId, "configuration_changed"); + } catch (error) { + if (!isDaytonaHostRecoveryStartError(error)) { + throw error; + } + createLogger().warn("sandbox_host_recovery_failover_started", { + sandboxId: state.identity.sandboxName(), + }); + return replaceSandboxRuntime(state, startingRunId, "daytona_host_recovery"); } - return replaceSandboxRuntime(state, startingRunId); } async function restartSandboxForWorkspaceRecovery( @@ -376,7 +392,11 @@ async function activateResolvedSandbox( return resolved.id; } -async function replaceSandboxRuntime(state: RuntimeState, startingRunId?: string): Promise { +async function replaceSandboxRuntime( + state: RuntimeState, + startingRunId: string | undefined, + reason: RuntimeReplacementReason, +): Promise { if (state.isSandboxRuntimeUpdateInProgress) { throw sandboxRuntimeUpdatePending(state.env.DAYTONA_SANDBOX_SNAPSHOT); } @@ -391,8 +411,8 @@ async function replaceSandboxRuntime(state: RuntimeState, startingRunId?: string let resolved: DaytonaSandbox; try { resolved = await state.provisioning.resolve(daytona); - if (!state.provisioning.isDesired(resolved)) { - resolved = await replaceSandboxRuntimeExclusive(state, daytona, resolved); + if (reason === "daytona_host_recovery" || !state.provisioning.isDesired(resolved)) { + resolved = await replaceSandboxRuntimeExclusive(state, daytona, resolved, reason); } } catch (error) { throw toUpstreamError( @@ -412,6 +432,7 @@ async function replaceSandboxRuntimeExclusive( state: RuntimeState, daytona: DaytonaClient, current: DaytonaSandbox, + reason: RuntimeReplacementReason, ): Promise { state.provisioning.assertRuntimeReplacementSafe(current); await prepareForSandboxReplacement(state); @@ -421,6 +442,7 @@ async function replaceSandboxRuntimeExclusive( throw sandboxRuntimeUpdatePending(state.env.DAYTONA_SANDBOX_SNAPSHOT); } createLogger().info("sandbox_runtime_replaced", { + reason, sandboxId: state.identity.sandboxName(), snapshot: state.env.DAYTONA_SANDBOX_SNAPSHOT, }); diff --git a/packages/agent-core/src/tools/code/daytona-client.ts b/packages/agent-core/src/tools/code/daytona-client.ts index 9b6e075..2bc883e 100644 --- a/packages/agent-core/src/tools/code/daytona-client.ts +++ b/packages/agent-core/src/tools/code/daytona-client.ts @@ -34,6 +34,8 @@ const DAYTONA_FILE_LIST_MAX_ITEMS = 1_000; const DAYTONA_SANDBOX_PAGE_MAX_ITEMS = 100; const DAYTONA_SESSION_COMMAND_MAX_ITEMS = 1_000; const DAYTONA_VOLUME_NAME_MAX_CHARACTERS = 100; +const DAYTONA_HOST_RECOVERY_START_MESSAGE = + "sandbox start is temporarily unavailable while the sandbox's host recovers"; interface DaytonaClientConfig { apiKey: string; @@ -66,6 +68,22 @@ export class DaytonaApiError extends Error { } } +/** Identifies Daytona's host-local start rejection so callers can fail over safely. */ +export function isDaytonaHostRecoveryStartError(error: unknown): boolean { + let current = error; + for (let depth = 0; depth < 3; depth += 1) { + if ( + current instanceof DaytonaApiError && + current.status === 503 && + current.message.toLowerCase().includes(DAYTONA_HOST_RECOVERY_START_MESSAGE) + ) { + return true; + } + current = current instanceof Error ? current.cause : undefined; + } + return false; +} + // --------------------------------------------------------------------------- // Response schemas project provider payloads down to fields used by the runtime. // --------------------------------------------------------------------------- diff --git a/packages/agent-core/src/tools/code/index.ts b/packages/agent-core/src/tools/code/index.ts index 2978d62..a47b9f1 100644 --- a/packages/agent-core/src/tools/code/index.ts +++ b/packages/agent-core/src/tools/code/index.ts @@ -5,7 +5,11 @@ export type { DaytonaVolume, SandboxDestroyResult, } from "./daytona-client"; -export { DaytonaApiError, DaytonaClient } from "./daytona-client"; +export { + DaytonaApiError, + DaytonaClient, + isDaytonaHostRecoveryStartError, +} from "./daytona-client"; export { DeleteFileInputSchema,