feat(runtime): execute docker environment recipes

packages/core/src/evaluation/environment/docker.ts

@@ -0,0 +1,192 @@
+import { createHash } from 'node:crypto';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+import type {
+  DockerEnvironmentRecipe,
+  EnvironmentSetupConfig,
+} from '../loaders/environment-recipe.js';
+import type { TargetRuntimeConfig } from '../providers/sandbox-runner.js';
+import type { JsonObject } from '../types.js';
+import {
+  type CommandExecutor,
+  DefaultCommandExecutor,
+  DockerWorkspaceProvider,
+} from '../workspace/docker-workspace.js';
+
+export type DockerEnvironmentSetupStatus = 'skipped' | 'success' | 'failed';
+
+export interface DockerEnvironmentSetupResult {
+  readonly type: 'docker';
+  readonly image: string;
+  readonly workdir: string;
+  readonly status: DockerEnvironmentSetupStatus;
+  readonly command?: readonly string[] | string;
+  readonly cwd?: string;
+  readonly args?: JsonObject;
+  readonly stdout?: string;
+  readonly stderr?: string;
+  readonly exitCode?: number;
+  readonly targetRuntime: TargetRuntimeConfig;
+}
+
+export class DockerEnvironmentSetupError extends Error {
+  readonly result: DockerEnvironmentSetupResult;
+
+  constructor(message: string, result: DockerEnvironmentSetupResult) {
+    super(message);
+    this.name = 'DockerEnvironmentSetupError';
+    this.result = result;
+  }
+}
+
+function dockerImageTag(recipe: DockerEnvironmentRecipe): string {
+  const hash = createHash('sha256')
+    .update(recipe.context ?? '')
+    .update('\0')
+    .update(recipe.dockerfile ?? '')
+    .digest('hex')
+    .slice(0, 16);
+  return `agentv-environment:${hash}`;
+}
+
+function shellQuote(value: string): string {
+  return `'${value.replaceAll("'", "'\\''")}'`;
+}
+
+function setupPayload(recipe: DockerEnvironmentRecipe): JsonObject {
+  return {
+    args: recipe.setup?.args ?? {},
+    environment: {
+      type: 'docker',
+      workdir: recipe.workdir,
+      ...(recipe.image ? { image: recipe.image } : {}),
+      ...(recipe.context ? { context: recipe.context } : {}),
+      ...(recipe.dockerfile ? { dockerfile: recipe.dockerfile } : {}),
+    },
+    ...(recipe.setup?.env ? { env: recipe.setup.env } : {}),
+  };
+}
+
+function setupCommandLine(setup: EnvironmentSetupConfig, payload: string): string {
+  const envPrefix = Object.entries(setup.env ?? {})
+    .map(([key, value]) => `${key}=${shellQuote(value)}`)
+    .join(' ');
+  const command =
+    typeof setup.command === 'string'
+      ? setup.command
+      : setup.command.map((entry) => shellQuote(entry)).join(' ');
+  const setupCommand = envPrefix ? `${envPrefix} ${command}` : command;
+  return `printf %s ${shellQuote(payload)} | (${setupCommand})`;
+}
+
+function setupTimeoutSeconds(setup: EnvironmentSetupConfig | undefined): number | undefined {
+  return setup?.timeout_seconds;
+}
+
+function dockerRuntimeForRecipe(
+  recipe: DockerEnvironmentRecipe,
+  image: string,
+): TargetRuntimeConfig {
+  const setup = recipe.setup;
+  const payload = JSON.stringify(setupPayload({ ...recipe, image }), null, 2);
+  const tempDir = path.resolve(tmpdir());
+  const recipeMounts = (recipe.mounts ?? []).map((mount) => ({
+    source: mount.source,
+    target: mount.target,
+    access: mount.read_only === true ? 'ro' : (mount.access ?? 'rw'),
+  }));
+  const mounts = recipeMounts.some((mount) => mount.target === tempDir)
+    ? recipeMounts
+    : [...recipeMounts, { source: tempDir, target: tempDir, access: 'rw' }];
+  return {
+    mode: 'sandbox',
+    engine: 'docker',
+    image,
+    workdir: recipe.workdir,
+    host_cwd: recipe.sourceDir,
+    env: {
+      ...(recipe.env ?? {}),
+      AGENTV_ENVIRONMENT_WORKDIR: recipe.workdir,
+    },
+    ...(recipe.secrets !== undefined ? { secrets: recipe.secrets } : {}),
+    mounts,
+    ...(recipe.resources?.memory !== undefined ? { memory: recipe.resources.memory } : {}),
+    ...(recipe.resources?.cpus !== undefined ? { cpus: recipe.resources.cpus } : {}),
+    ...(setup !== undefined ? { setup: [setupCommandLine(setup, payload)] } : {}),
+    ...(setupTimeoutSeconds(setup) !== undefined
+      ? { setup_timeout: setupTimeoutSeconds(setup) }
+      : {}),
+  };
+}
+
+function formatDockerFailure(
+  action: string,
+  result: { stderr: string; stdout: string; exitCode: number },
+) {
+  const details = result.stderr.trim() || result.stdout.trim();
+  return details
+    ? `docker ${action} failed (exit ${result.exitCode}): ${details}`
+    : `docker ${action} failed (exit ${result.exitCode})`;
+}
+
+export async function prepareDockerEnvironment(
+  recipe: DockerEnvironmentRecipe,
+  executor: CommandExecutor = new DefaultCommandExecutor(),
+): Promise<DockerEnvironmentSetupResult> {
+  const image = recipe.context ? (recipe.image ?? dockerImageTag(recipe)) : recipe.image;
+  if (!image) {
+    throw new Error('Docker environment requires image when context is not set.');
+  }
+
+  const docker = new DockerWorkspaceProvider(
+    {
+      image,
+      ...(recipe.resources?.memory !== undefined ? { memory: recipe.resources.memory } : {}),
+      ...(recipe.resources?.cpus !== undefined ? { cpus: recipe.resources.cpus } : {}),
+    },
+    executor,
+  );
+
+  if (!(await docker.isDockerAvailable())) {
+    throw new Error(
+      'Docker environment configured but Docker CLI is not available. Install Docker and ensure it is running.',
+    );
+  }
+
+  if (recipe.context) {
+    const argv = ['docker', 'build', '-t', image];
+    if (recipe.dockerfile) {
+      argv.push('-f', recipe.dockerfile);
+    }
+    argv.push(recipe.context);
+    const result = await executor.exec(argv, { timeoutMs: 30 * 60 * 1000 });
+    if (result.exitCode !== 0) {
+      const failure: DockerEnvironmentSetupResult = {
+        type: 'docker',
+        image,
+        workdir: recipe.workdir,
+        status: 'failed',
+        stderr: result.stderr,
+        stdout: result.stdout,
+        exitCode: result.exitCode,
+        targetRuntime: dockerRuntimeForRecipe(recipe, image),
+      };
+      throw new DockerEnvironmentSetupError(formatDockerFailure('build', result), failure);
+    }
+  } else {
+    await docker.pullImage();
+  }
+
+  const runtime = dockerRuntimeForRecipe(recipe, image);
+  return {
+    type: 'docker',
+    image,
+    workdir: recipe.workdir,
+    status: recipe.setup ? 'success' : 'skipped',
+    ...(recipe.setup?.command !== undefined ? { command: recipe.setup.command } : {}),
+    cwd: recipe.sourceDir,
+    ...(recipe.setup?.args !== undefined ? { args: recipe.setup.args } : {}),
+    targetRuntime: runtime,
+  };
+}

packages/core/src/evaluation/orchestrator.ts

@@ -419,6 +419,7 @@ export interface EvaluationCache {
 export interface RunEvalCaseOptions {
   readonly evalCase: EvalTest;
   readonly provider: Provider;
+  readonly providerFactory?: (target: ResolvedTarget) => Provider;
   readonly target: ResolvedTarget;
   readonly evaluators: Partial<Record<string, Grader>> & { readonly 'llm-grader': Grader };
   readonly now?: () => Date;
@@ -1257,6 +1258,7 @@ export async function runEvaluation(
         const runCaseOptions: RunEvalCaseOptions = {
           evalCase: evalCase,
           provider: primaryProvider,
+          providerFactory,
           target,
           evaluators: evaluatorRegistry,
           maxRetries,
@@ -1874,12 +1876,6 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
   const promptInputs = await buildPromptInputs(evalCase, formattingMode);
   const typeRegistry = providedTypeRegistry ?? createBuiltinRegistry();
 
-  const cacheKey = useCache ? createCacheKey(provider, target, evalCase, promptInputs) : undefined;
-  let cachedResponse: ProviderResponse | undefined;
-  if (cacheKey && cache) {
-    cachedResponse = await cache.get(cacheKey);
-  }
-
   const nowFn = now ?? (() => new Date());
 
   let afterEachOutput: string | undefined;
@@ -1921,10 +1917,24 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
     beforeAllOutput,
     beforeEachOutput,
     baselineCommit,
+    targetCwd,
+    targetRuntime,
     isSharedWorkspace,
     caseWorkspaceFile,
     extensionState,
   } = workspaceSetup;
+  const effectiveTarget = targetRuntime ? { ...target, runtime: targetRuntime } : target;
+  const effectiveProvider =
+    targetRuntime !== undefined
+      ? (options.providerFactory ?? createProvider)(effectiveTarget)
+      : provider;
+  const cacheKey = useCache
+    ? createCacheKey(effectiveProvider, effectiveTarget, evalCase, promptInputs)
+    : undefined;
+  let cachedResponse: ProviderResponse | undefined;
+  if (cacheKey && cache) {
+    cachedResponse = await cache.get(cacheKey);
+  }
   const extensionMetadata = extensionState?.metadata;
   const providerMetadata = mergeMetadata(evalCase.metadata, extensionState?.providerContext);
   const resultMetadata = mergeMetadata(evalCase.metadata, extensionMetadata);
@@ -2032,15 +2042,15 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
   if (evalCase.mode === 'conversation' && evalCase.turns?.length) {
     let conversationResult = await runConversationMode({
       evalCase,
-      provider,
-      target,
+      provider: effectiveProvider,
+      target: effectiveTarget,
       evaluators,
       typeRegistry,
       graderProvider,
       promptInputs,
       nowFn,
       signal,
-      workspacePath,
+      workspacePath: targetCwd ?? workspacePath,
       caseWorkspaceFile,
       agentTimeoutMs,
       streamCallbacks: options.streamCallbacks,
@@ -2085,15 +2095,15 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
 
   while (!providerResponse && attempt < attemptBudget) {
     try {
-      providerResponse = await invokeProvider(provider, {
+      providerResponse = await invokeProvider(effectiveProvider, {
         evalCase: evalCase,
-        target,
+        target: effectiveTarget,
         promptInputs,
         attempt,
         evalFilePath,
         agentTimeoutMs,
         signal,
-        cwd: workspacePath,
+        cwd: targetCwd ?? workspacePath,
         workspaceFile: caseWorkspaceFile,
         captureFileChanges: !!baselineCommit,
         streamCallbacks: options.streamCallbacks,
@@ -2112,22 +2122,22 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
   }
 
   // Try fallback targets in order after exhausting retries on the primary
-  if (!providerResponse && target.fallbackTargets?.length && targetResolver) {
-    for (const fallbackName of target.fallbackTargets) {
+  if (!providerResponse && effectiveTarget.fallbackTargets?.length && targetResolver) {
+    for (const fallbackName of effectiveTarget.fallbackTargets) {
       const fallbackProvider = targetResolver(fallbackName);
       if (!fallbackProvider) {
         continue;
       }
       try {
         providerResponse = await invokeProvider(fallbackProvider, {
           evalCase: evalCase,
-          target,
+          target: effectiveTarget,
           promptInputs,
           attempt: 0,
           evalFilePath,
           agentTimeoutMs,
           signal,
-          cwd: workspacePath,
+          cwd: targetCwd ?? workspacePath,
           workspaceFile: caseWorkspaceFile,
           captureFileChanges: !!baselineCommit,
           streamCallbacks: options.streamCallbacks,
@@ -2145,11 +2155,11 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
   if (!providerResponse) {
     const errorResult = buildErrorResult(
       evalCase,
-      target.name,
+      effectiveTarget.name,
       nowFn(),
       lastError ?? new Error('Provider did not return a response'),
       promptInputs,
-      provider,
+      effectiveProvider,
       'agent',
       'provider_error',
       verbose,
@@ -2172,7 +2182,7 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
       evalCase,
       evalFilePath,
       repoRoot,
-      target: targetUsed ? { ...target, name: targetUsed } : target,
+      target: targetUsed ? { ...effectiveTarget, name: targetUsed } : effectiveTarget,
       attempt,
       response: providerResponse,
       nowFn,
@@ -2247,8 +2257,8 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
     const result = await evaluateCandidate({
       evalCase,
       candidate,
-      target,
-      provider,
+      target: effectiveTarget,
+      provider: effectiveProvider,
       evaluators,
       typeRegistry,
       promptInputs,

packages/core/src/evaluation/providers/sandbox-runner.ts

@@ -107,6 +107,10 @@ function dockerWorkdir(runtime: TargetRuntimeConfig): string | undefined {
   return asString(runtime.workdir) ?? asString(runtime.workspace);
 }
 
+function dockerHostCwd(runtime: TargetRuntimeConfig): string | undefined {
+  return asString(runtime.host_cwd);
+}
+
 function dockerNetwork(runtime: TargetRuntimeConfig): string {
   const network = asString(runtime.network);
   return network ?? 'none';
@@ -116,6 +120,14 @@ function dockerSetupCommands(runtime: TargetRuntimeConfig): readonly string[] {
   return asStringArray(runtime.setup);
 }
 
+function dockerMemory(runtime: TargetRuntimeConfig): string | undefined {
+  return asString(runtime.memory);
+}
+
+function dockerCpus(runtime: TargetRuntimeConfig): number | undefined {
+  return typeof runtime.cpus === 'number' && runtime.cpus > 0 ? runtime.cpus : undefined;
+}
+
 export async function runDockerSandboxCommand(
   command: string,
   options: SandboxCommandRunOptions,
@@ -159,6 +171,16 @@ export async function runDockerSandboxCommand(
     argv.push('--workdir', workdir);
   }
 
+  const memory = dockerMemory(options.runtime);
+  if (memory) {
+    argv.push('--memory', memory);
+  }
+
+  const cpus = dockerCpus(options.runtime);
+  if (cpus !== undefined) {
+    argv.push('--cpus', String(cpus));
+  }
+
   for (const [key, value] of Object.entries(dockerEnv(options.runtime))) {
     argv.push('--env', `${key}=${value}`);
   }
@@ -172,7 +194,7 @@ export async function runDockerSandboxCommand(
 
   return new Promise((resolve) => {
     const child = spawn('docker', argv, {
-      cwd: options.cwd,
+      cwd: dockerHostCwd(options.runtime) ?? options.cwd,
       env: process.env,
       windowsHide: true,
     });