Skip to content

fix(core): hard-deprecate runner batch config #1740

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
christso merged 1 commit into from
Jul 8, 2026
Merged

fix(core): hard-deprecate runner batch config #1740

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion apps/cli/src/commands/eval/task-bundle.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ const AUTHORING_TOP_LEVEL_TARGET_FIELDS = new Set([
'grader_target',
'max_budget_usd',
'workers',
'batch_requests',
'subagent_mode_allowed',
'max_retries',
'retry_initial_delay_ms',
Expand Down
9 changes: 5 additions & 4 deletions packages/core/src/evaluation/loaders/config-graph.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ const SUPPORTED_FILE_REF_FIELDS = new Set([
]);
const RUNTIME_MODES = new Set(['host', 'profile', 'sandbox']);
const AMBIGUOUS_PROVIDER_ALIASES = new Set(['codex', 'claude', 'copilot', 'pi']);
const REMOVED_RUNNER_BATCHING_MESSAGE =
'runner-level batching was removed; providers should implement internal queueing/batching behind per-request invocation.';
const REMOVED_TARGET_FIELDS = new Map([
['label', "target identity uses 'id'; remove 'label'."],
['name', "target identity uses 'id'; remove 'name'."],
Expand All @@ -44,10 +46,9 @@ const REMOVED_TARGET_FIELDS = new Map([
['install', 'install/setup steps belong in environment.setup, not under targets.'],
['grader_target', "grader selection belongs in 'defaults.grader' or evaluator config."],
['workers', "target-level 'workers' is not general run policy; use 'execution.max_concurrency'."],
[
'batch_requests',
"target-level 'batch_requests' is not part of the base config contract; keep provider batching under provider-specific config only when needed.",
],
['batch_requests', REMOVED_RUNNER_BATCHING_MESSAGE],
['provider_batching', REMOVED_RUNNER_BATCHING_MESSAGE],
['providerBatching', REMOVED_RUNNER_BATCHING_MESSAGE],
[
'subagent_mode_allowed',
"target-level 'subagent_mode_allowed' is not part of the base config contract.",
Expand Down
313 changes: 0 additions & 313 deletions packages/core/src/evaluation/orchestrator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -949,48 +949,7 @@ export async function runEvaluation(
// Discover custom providers from .agentv/providers/ directory
const providerRegistry = createBuiltinProviderRegistry();
await discoverProviders(providerRegistry, discoveryBaseDir);

const primaryProvider = getOrCreateProvider(target);
let providerSupportsBatch =
target.providerBatching === true &&
primaryProvider.supportsBatch === true &&
typeof primaryProvider.invokeBatch === 'function';
let batchingDisabledByRuntimePolicy = false;

// Disable batch mode when repeat count > 1 (batching is incompatible with separate attempts).
if (trials && trials.count > 1 && providerSupportsBatch) {
console.warn(
'Warning: Batch mode is disabled when evaluate_options.repeat.count > 1. Using per-attempt dispatch.',
);
providerSupportsBatch = false;
batchingDisabledByRuntimePolicy = true;
}

const requiresWorkspaceDispatch =
workspacePath !== undefined ||
legacyWorkspacePath !== undefined ||
target.environment !== undefined ||
filteredEvalCases.some(
(evalCase) => evalCase.workspace !== undefined || evalCase.environment !== undefined,
);
if (providerSupportsBatch && requiresWorkspaceDispatch) {
if (verbose) {
console.warn('Warning: Batch mode is disabled for workspace- or environment-enabled evals.');
}
providerSupportsBatch = false;
batchingDisabledByRuntimePolicy = true;
}

if (
target.providerBatching &&
!providerSupportsBatch &&
verbose &&
!batchingDisabledByRuntimePolicy
) {
console.warn(
`Request batching requested for target '${target.name}', but provider does not advertise batch support. Using per-attempt dispatch.`,
);
}

// Notify about total test count before starting
if (onProgress && filteredEvalCases.length > 0) {
Expand All @@ -1004,37 +963,6 @@ export async function runEvaluation(
}
}

if (providerSupportsBatch) {
try {
return await runBatchEvaluation({
evalCases: filteredEvalCases,
provider: primaryProvider,
target,
evaluatorRegistry,
typeRegistry,
nowFn: now ?? (() => new Date()),
onProgress,
onResult,
verbose,
resolveGraderProvider,
agentTimeoutMs,
targetResolver,
availableTargets,
threshold: scoreThreshold,
replayRecording,
evalFilePath,
repoRoot: repoRootPath,
});
} catch (error) {
if (verbose) {
const message = error instanceof Error ? error.message : String(error);
console.warn(
`Request batch execution failed, falling back to per-attempt dispatch: ${message}`,
);
}
}
}

const resolvedRetainOnSuccess = retainOnSuccess ?? (keepWorkspaces ? 'keep' : 'cleanup');
const resolvedRetainOnFailure = retainOnFailure ?? (cleanupWorkspaces ? 'cleanup' : 'keep');
const workers = options.maxConcurrency ?? target.workers ?? 1;
Expand Down Expand Up @@ -1630,247 +1558,6 @@ export async function runEvaluation(
}
}

async function runBatchEvaluation(options: {
readonly evalCases: readonly EvalTest[];
readonly provider: Provider;
readonly target: ResolvedProviderBackend;
readonly evaluatorRegistry: Partial<Record<string, Grader>> & {
readonly 'llm-grader': Grader;
};
readonly typeRegistry: import('./registry/grader-registry.js').GraderRegistry;
readonly nowFn: () => Date;
readonly onProgress?: (event: ProgressEvent) => MaybePromise<void>;
readonly onResult?: (result: EvaluationResult) => MaybePromise<void>;
readonly verbose?: boolean;
readonly resolveGraderProvider: (
target: ResolvedProviderBackend,
) => Promise<Provider | undefined>;
readonly agentTimeoutMs?: number;
readonly targetResolver?: (name: string) => Provider | undefined;
readonly availableTargets?: readonly string[];
readonly threshold?: number;
readonly replayRecording?: ReplayRecordingOptions;
readonly evalFilePath: string;
readonly repoRoot: string;
}): Promise<readonly EvaluationResult[]> {
const {
evalCases,
provider,
target,
evaluatorRegistry,
typeRegistry,
nowFn,
onProgress,
onResult,
verbose,
resolveGraderProvider,
agentTimeoutMs,
targetResolver,
availableTargets,
threshold: batchThreshold,
replayRecording,
evalFilePath,
repoRoot,
} = options;

// Prepare prompt inputs up front so we can reuse them for grading.
const promptInputsList: PromptInputs[] = [];
const formattingMode = usesFileReferencePrompt(provider) ? 'agent' : 'lm';

for (const evalCase of evalCases) {
const promptInputs = await buildPromptInputs(evalCase, formattingMode);
promptInputsList.push(promptInputs);
}

const batchRequests: ProviderRequest[] = evalCases.map((evalCase, index) => {
const promptInputs = promptInputsList[index];
return {
question: promptInputs.question,
systemPrompt: promptInputs.systemMessage,
inputFiles: evalCase.file_paths,
evalCaseId: evalCase.id,
suite: evalCase.suite,
evalFilePath,
};
});

const batchResponse = await provider.invokeBatch?.(batchRequests);
if (!Array.isArray(batchResponse)) {
throw new Error('Request batching failed: invokeBatch did not return an array');
}
if (batchResponse.length !== evalCases.length) {
throw new Error(
`Request batching failed: expected ${evalCases.length} responses, received ${batchResponse.length}`,
);
}

if (onProgress) {
const startedAt = Date.now();
for (let i = 0; i < evalCases.length; i++) {
await onProgress({
workerId: 1,
testId: evalCases[i].id,
status: 'running',
startedAt,
});
}
}

const results: EvaluationResult[] = [];
for (let i = 0; i < evalCases.length; i++) {
const evalCase = evalCases[i];
const promptInputs = promptInputsList[i];
const providerResponse = batchResponse[i];
await maybeRecordReplayFixture({
replayRecording,
evalCase,
evalFilePath,
repoRoot,
target,
attempt: 0,
response: providerResponse,
nowFn,
});

// Extract output from batch response
const output = providerResponse.output;
const hasExecutionMetrics =
providerResponse.tokenUsage !== undefined ||
providerResponse.costUsd !== undefined ||
providerResponse.durationMs !== undefined;

const computed = output
? computeTraceSummary(output)
: hasExecutionMetrics
? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } }
: undefined;
const merged = computed
? mergeExecutionMetrics(computed, {
tokenUsage: providerResponse.tokenUsage,
costUsd: providerResponse.costUsd,
durationMs: providerResponse.durationMs,
startTime: providerResponse.startTime,
endTime: providerResponse.endTime,
})
: undefined;
const trace = merged?.trace;
const costUsd = merged?.costUsd;
const durationMs = merged?.durationMs;
const tokenUsage = merged?.tokenUsage;
const startTime = merged?.startTime;
const endTime = merged?.endTime;
const rawProviderLogPath = extractProviderRawLogPath(providerResponse);
const providerResponseMetadata = buildProviderResponseTransformMetadata(providerResponse);

// Extract candidate from last assistant message in output
const candidate = extractLastAssistantContent(output);

const providerError = extractProviderError(providerResponse);

let result: EvaluationResult;
try {
result = await evaluateCandidate({
evalCase,
candidate,
target,
provider,
evaluators: evaluatorRegistry,
typeRegistry,
promptInputs,
nowFn,
attempt: 0,
sampleIndex: 0,
graderProvider: await resolveGraderProvider(target),
agentTimeoutMs,
output,
trace,
costUsd,
durationMs,
tokenUsage,
providerResponseMetadata,
startTime,
endTime,
rawProviderLogPath,
targetResolver,
availableTargets,
verbose,
threshold: evalCase.threshold ?? batchThreshold,
});

if (providerError) {
const failureReasonCode = targetExecutionFailureReasonCode(providerResponse);
result = {
...result,
targetExecution: providerResponse.targetExecution,
trace: appendErrorEventToTrace(result.trace, providerError, {
failure_stage: 'agent',
failure_reason_code: failureReasonCode,
}),
error: providerError,
executionStatus: 'execution_error' as const,
failureStage: 'agent' as const,
failureReasonCode: failureReasonCode,
executionError: { message: providerError, stage: 'agent' as const },
};
} else if (providerResponse.targetExecution) {
result = { ...result, targetExecution: providerResponse.targetExecution };
}
} catch (error) {
const errorResult = buildErrorResult(
evalCase,
target.name,
nowFn(),
error,
promptInputs,
provider,
'evaluator',
'evaluator_error',
verbose,
{ sampleIndex: 0, retryIndex: 0 },
);
results.push(errorResult);
if (onResult) {
await onResult(errorResult);
}
if (onProgress) {
await onProgress({
workerId: 1,
testId: evalCase.id,
status: 'failed',
completedAt: Date.now(),
error: error instanceof Error ? error.message : String(error),
score: errorResult.score,
executionStatus: errorResult.executionStatus,
evalRunDurationMs: errorResult.evalRun?.durationMs,
});
}
continue;
}

results.push(result);
if (onResult) {
await onResult(result);
}

if (onProgress) {
await onProgress({
workerId: 1,
testId: evalCase.id,
status: result.error ? 'failed' : 'completed',
startedAt: 0,
completedAt: Date.now(),
error: result.error,
score: result.score,
executionStatus: result.executionStatus,
durationMs: result.durationMs,
evalRunDurationMs: result.evalRun?.durationMs,
});
}
}

return results;
}

async function maybeRecordReplayFixture(options: {
readonly replayRecording?: ReplayRecordingOptions;
readonly evalCase: EvalTest;
Expand Down
2 changes: 0 additions & 2 deletions packages/core/src/evaluation/providers/claude-cli.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ export class ClaudeCliProvider implements Provider {
readonly id: string;
readonly kind = 'claude-cli' as const;
readonly targetName: string;
readonly supportsBatch = false;

private readonly config: ClaudeResolvedConfig;

constructor(targetName: string, config: ClaudeResolvedConfig) {
Expand Down
2 changes: 0 additions & 2 deletions packages/core/src/evaluation/providers/claude-sdk.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,6 @@ export class ClaudeSdkProvider implements Provider {
readonly id: string;
readonly kind = 'claude-sdk' as const;
readonly targetName: string;
readonly supportsBatch = false;

private readonly config: ClaudeResolvedConfig;

constructor(targetName: string, config: ClaudeResolvedConfig) {
Expand Down
2 changes: 0 additions & 2 deletions packages/core/src/evaluation/providers/claude.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,6 @@ export class ClaudeProvider implements Provider {
readonly id: string;
readonly kind = 'claude-sdk' as const;
readonly targetName: string;
readonly supportsBatch = false;

private readonly config: ClaudeResolvedConfig;

constructor(targetName: string, config: ClaudeResolvedConfig) {
Expand Down
Loading
Loading