From 3ddff4f44f4070957493e00cf7a50a3a6088fbf4 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 9 Jul 2026 09:00:32 -0400 Subject: [PATCH 1/5] feat(evals): cursor_sdk full-harness adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registers --harness cursor_sdk via @cursor/sdk (Agent.create → send → stream/wait against Cursor's managed local agent). Classification: FULL harness, on the smart tier next to claude_code/codex — 'the same runtime, harness, and models that power Cursor' — not the bare tier, despite being a new SDK. The runner exposes exactly one custom tool (browse, same allowed-command gate as every other external harness) and prompts the agent to use only that tool; the SDK exposes no tool allow-list, so browse-only discipline is prompt + custom-tool based rather than a canUseTool hard gate (recorded in the design doc). cursorAdapter reverse-maps the SDKMessage stream (tool_call/assistant/thinking/usage) into a verifier Trajectory, collapsing running→terminal duplicate tool_call events by call_id. Auth: CURSOR_API_KEY. Live validation pending an API key — implemented and unit-tested against a mocked SDK. Co-Authored-By: Claude Fable 5 --- packages/evals/framework/benchHarness.ts | 6 + packages/evals/framework/benchTypes.ts | 1 + packages/evals/framework/cursorSdkRunner.ts | 319 ++++++++++++++++++ .../framework/harnesses/cursorAdapter.ts | 182 ++++++++++ packages/evals/package.json | 1 + .../tests/framework/benchHarness.test.ts | 13 + .../tests/framework/cursorSdkRunner.test.ts | 268 +++++++++++++++ pnpm-lock.yaml | 117 +++++++ 8 files changed, 907 insertions(+) create mode 100644 packages/evals/framework/cursorSdkRunner.ts create mode 100644 packages/evals/framework/harnesses/cursorAdapter.ts create mode 100644 packages/evals/tests/framework/cursorSdkRunner.test.ts diff --git a/packages/evals/framework/benchHarness.ts b/packages/evals/framework/benchHarness.ts index 24dbdc09a..15fd2da27 100644 --- a/packages/evals/framework/benchHarness.ts +++ b/packages/evals/framework/benchHarness.ts @@ -34,6 +34,7 @@ import { import { runVercelAiSdkAgent } from "./vercelAiSdkRunner.js"; import { runAnthropicSdkAgent } from "./anthropicSdkRunner.js"; import { runOpenAiAgentsSdkAgent } from "./openaiAgentsSdkRunner.js"; +import { runCursorSdkAgent } from "./cursorSdkRunner.js"; import type { DiscoveredTask, TaskResult } from "./types.js"; import type { AdapterBackedHarness, @@ -454,6 +455,10 @@ export const openaiAgentsSdkHarness = buildAdapterBackedHarness( "openai_agents_sdk", runOpenAiAgentsSdkAgent, ); +export const cursorSdkHarness = buildAdapterBackedHarness( + "cursor_sdk", + runCursorSdkAgent, +); const harnessRegistry = new Map([ ["stagehand", stagehandHarness], @@ -462,6 +467,7 @@ const harnessRegistry = new Map([ ["vercel_ai_sdk", vercelAiSdkHarness], ["anthropic_sdk", anthropicSdkHarness], ["openai_agents_sdk", openaiAgentsSdkHarness], + ["cursor_sdk", cursorSdkHarness], ]); export function getBenchHarness(harness: Harness): BenchHarness { diff --git a/packages/evals/framework/benchTypes.ts b/packages/evals/framework/benchTypes.ts index e6c50058c..e8f44c0f6 100644 --- a/packages/evals/framework/benchTypes.ts +++ b/packages/evals/framework/benchTypes.ts @@ -34,6 +34,7 @@ export const EXECUTABLE_BENCH_HARNESSES = [ "vercel_ai_sdk", "anthropic_sdk", "openai_agents_sdk", + "cursor_sdk", ] as const satisfies readonly Harness[]; /** diff --git a/packages/evals/framework/cursorSdkRunner.ts b/packages/evals/framework/cursorSdkRunner.ts new file mode 100644 index 000000000..723b4d9b0 --- /dev/null +++ b/packages/evals/framework/cursorSdkRunner.ts @@ -0,0 +1,319 @@ +/** + * cursorSdkRunner — FULL-harness runner via the Cursor SDK (`@cursor/sdk`). + * + * Classification: this sits on the smart tier next to claude_code/codex, NOT + * the bare tier — "the same runtime, harness, and models that power Cursor" + * is the product's own description. The SDK runs Cursor's managed local agent + * (its own loop, planning, retries, shell/file tools); we expose exactly one + * custom tool (`browse`, same allowed-command gate as every other external + * harness) and prompt the agent to use only that tool. The SDK does not + * expose an allow-list to hard-disable its native shell tools, so the browse + * gating here is prompt + custom-tool discipline rather than the Claude Code + * canUseTool hard gate — recorded as a known limitation in the design doc. + * + * Auth: CURSOR_API_KEY (the SDK's own default env var). Model ids come from + * Cursor's catalog (e.g. "composer-2.5" / "cursor/composer-2.5" via -m). + * + * Testability: pass `sdk` with a mocked { Agent } (Agent.create → send → + * stream/wait). + */ +import type { AvailableModel } from "@browserbasehq/stagehand"; +import { EvalsError } from "../errors.js"; +import type { EvalLogger } from "../logger.js"; +import type { TaskResult } from "./types.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import type { PreparedExternalHarnessAdapter } from "./externalHarnessToolAdapter.js"; +import { runBareBrowseCommand } from "./externalHarnessToolAdapter.js"; +import { + BROWSE_TOOL_DESCRIPTION, + BROWSE_TOOL_NAME, + buildBareLoopUserPrompt, + stringifyLoopError, + stripProviderPrefix, +} from "./bareLoopRunner.js"; +import { parseEvalResultText } from "./evalResultParser.js"; +import { cursorAdapter } from "./harnesses/cursorAdapter.js"; +import { + gradeExternalTrajectory, + type ExternalHarnessVerifierConfig, +} from "./verifierAdapter.js"; + +type MetricValue = { count: number; value: number }; +type CursorSdkMessage = Record; + +const HARNESS = "cursor_sdk"; +export const DEFAULT_CURSOR_MODEL = "composer-2.5"; + +export interface CursorRunHandle { + stream(): AsyncGenerator; + wait(): Promise<{ + status: string; + result?: string; + error?: { message: string; code?: string }; + usage?: { + inputTokens?: number; + outputTokens?: number; + cacheReadTokens?: number; + }; + }>; +} + +export interface CursorSdkAgentHandle { + send(message: string): Promise; + close(): void; +} + +export interface CursorSdk { + Agent: { + create(options: Record): Promise; + }; +} + +export interface CursorSdkRunnerInput { + plan: ExternalHarnessTaskPlan; + model: AvailableModel; + logger: EvalLogger; + toolAdapter: PreparedExternalHarnessAdapter; + signal?: AbortSignal; + /** Injectable for unit tests; defaults to the real @cursor/sdk Agent. */ + sdk?: CursorSdk; + verifier?: ExternalHarnessVerifierConfig; +} + +export function normalizeCursorModel(model: AvailableModel): string { + if (model === ("cursor/default" as AvailableModel)) { + return DEFAULT_CURSOR_MODEL; + } + if (model.includes("/") && !model.startsWith("cursor/")) { + throw new EvalsError( + `cursor_sdk harness only accepts cursor models (e.g. cursor/${DEFAULT_CURSOR_MODEL}); received "${model}".`, + ); + } + return stripProviderPrefix(model); +} + +export function buildCursorPrompt( + plan: ExternalHarnessTaskPlan, + systemPromptAddendum: string, +): string { + // Cursor's SDK has no separate system-prompt channel for local agents; the + // skill-arm text rides at the top of the single prompt instead. + return [ + systemPromptAddendum, + `Use ONLY the custom "${BROWSE_TOOL_NAME}" tool for browser work — do not use your shell for browsing, and do not edit repository files.`, + "", + buildBareLoopUserPrompt(plan), + ].join("\n"); +} + +export async function runCursorSdkAgent( + input: CursorSdkRunnerInput, +): Promise { + const sdk = input.sdk ?? (await loadCursorSdk()); + if (!input.sdk && !process.env.CURSOR_API_KEY) { + throw new EvalsError( + "cursor_sdk harness requires CURSOR_API_KEY in the environment.", + ); + } + + const messages: CursorSdkMessage[] = []; + let finalText = ""; + let runStatus = "error"; + let stopReason: string | undefined; + let usage: + | { inputTokens?: number; outputTokens?: number; cacheReadTokens?: number } + | undefined; + let iterationError: unknown; + let agent: CursorSdkAgentHandle | undefined; + + try { + agent = await sdk.Agent.create({ + model: { id: normalizeCursorModel(input.model) }, + name: `stagehand-evals-${input.plan.dataset}-${input.plan.taskId ?? "task"}`, + local: { + cwd: input.toolAdapter.cwd, + customTools: { + [BROWSE_TOOL_NAME]: { + description: BROWSE_TOOL_DESCRIPTION, + inputSchema: { + type: "object", + properties: { + args: { + type: "string", + description: + 'Everything after "browse", e.g. "open https://example.com".', + }, + }, + required: ["args"], + }, + execute: async (args: Record) => { + const { output } = await runBareBrowseCommand( + input.toolAdapter, + String(args.args ?? ""), + ); + return output; + }, + }, + }, + }, + }); + + const run = await agent.send( + buildCursorPrompt(input.plan, input.toolAdapter.systemPromptAddendum), + ); + + for await (const message of run.stream()) { + if (input.signal?.aborted) { + throw new EvalsError("cursor_sdk run aborted"); + } + messages.push(message); + logCursorMessage(input.logger, message); + } + + const result = await run.wait(); + runStatus = result.status; + finalText = result.result ?? ""; + usage = result.usage; + if (result.error) { + stopReason = result.error.message; + } + } catch (error) { + iterationError = error; + input.logger.warn({ + category: HARNESS, + message: `Cursor stopped before a normal result: ${stringifyLoopError(error)}`, + level: 0, + }); + } finally { + try { + agent?.close(); + } catch { + // best-effort only + } + } + + const parsed = parseEvalResultText(finalText); + const completed = runStatus === "finished" && !iterationError; + const errorMessage = + parsed.summary ?? + stopReason ?? + (stringifyLoopError(iterationError) || + finalText || + "Cursor did not report success"); + + const baseResult: TaskResult = { + _success: parsed.success, + error: !parsed.success ? errorMessage : undefined, + reasoning: parsed.summary, + finalAnswer: parsed.finalAnswer, + rawResult: parsed.raw, + cursorStatus: completed ? "completed" : "sdk_error", + ...(stopReason && { cursorStopReason: stopReason }), + logs: input.logger.getLogs(), + metrics: buildCursorMetrics(usage, messages), + }; + + if (!input.verifier) { + return baseResult; + } + + const verifier = input.verifier; + return gradeExternalTrajectory({ + buildTrajectory: () => + cursorAdapter.fromHarnessResult( + { + messages, + finalAnswer: parsed.finalAnswer ?? finalText, + status: completed ? "complete" : "error", + ...(usage && { + usage: { + input_tokens: toFinite(usage.inputTokens), + output_tokens: toFinite(usage.outputTokens), + ...(usage.cacheReadTokens !== undefined && { + cached_input_tokens: toFinite(usage.cacheReadTokens), + }), + }, + }), + }, + verifier.taskSpec, + ), + verifier, + baseResult, + errorMessage, + category: HARNESS, + logger: input.logger, + }); +} + +function buildCursorMetrics( + usage: + | { inputTokens?: number; outputTokens?: number; cacheReadTokens?: number } + | undefined, + messages: CursorSdkMessage[], +): Record { + const toolCalls = messages.filter( + (message) => message.type === "tool_call" && message.status !== "running", + ).length; + return { + cursor_tool_calls: metricValue(toolCalls), + cursor_input_tokens: metricValue(usage?.inputTokens), + cursor_output_tokens: metricValue(usage?.outputTokens), + cursor_cache_read_tokens: metricValue(usage?.cacheReadTokens), + cursor_total_tokens: metricValue( + toFinite(usage?.inputTokens) + toFinite(usage?.outputTokens), + ), + }; +} + +function logCursorMessage(logger: EvalLogger, message: CursorSdkMessage): void { + const type = String(message.type ?? "unknown"); + let summary = `${type} message`; + if (type === "tool_call") { + summary = + `tool: ${String(message.name ?? "")} ${String(message.status ?? "")}`.trim(); + } else if (type === "status") { + summary = `status: ${String(message.status ?? "")}`; + } else if (type === "thinking" && typeof message.text === "string") { + summary = `thinking: ${clip(message.text, 200)}`; + } else if (type === "assistant") { + summary = "assistant message"; + } + logger.log({ + category: HARNESS, + message: summary, + level: 1, + auxiliary: { + type: { value: type, type: "string" }, + }, + }); +} + +function metricValue(value: unknown): MetricValue { + return { count: 1, value: toFinite(value) }; +} + +function toFinite(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) ? value : 0; +} + +function clip(value: string, maxLength: number): string { + return value.length <= maxLength + ? value + : `${value.slice(0, maxLength - 1)}…`; +} + +async function loadCursorSdk(): Promise { + try { + const mod = (await import("@cursor/sdk")) as { Agent?: CursorSdk["Agent"] }; + if (!mod.Agent || typeof mod.Agent.create !== "function") { + throw new Error("Agent export missing"); + } + return { Agent: mod.Agent }; + } catch (error) { + throw new EvalsError( + `cursor_sdk harness requires @cursor/sdk. Install it in packages/evals before running --harness cursor_sdk. ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} diff --git a/packages/evals/framework/harnesses/cursorAdapter.ts b/packages/evals/framework/harnesses/cursorAdapter.ts new file mode 100644 index 000000000..4916c7e6f --- /dev/null +++ b/packages/evals/framework/harnesses/cursorAdapter.ts @@ -0,0 +1,182 @@ +/** + * cursorAdapter — converts a Cursor SDK (`@cursor/sdk`) run into a + * `Trajectory` the verifier can consume. + * + * Input shape: the SDK's `run.stream()` yields `SDKMessage` events — + * `assistant` (text + tool_use blocks), `tool_call` (name/args/result with a + * running→completed|error status), `thinking`, `usage` (per-turn token + * counts), and `status`. We accumulate the stream upstream in + * `runCursorSdkAgent` and hand the full list here. + * + * Mapping: + * - Each terminal `tool_call` event (status completed|error) becomes one + * normalized tool call. Duplicate events for the same call_id collapse to + * the last (terminal) one. + * - `assistant` text blocks and `thinking` text buffered since the previous + * tool call fold into the next tool call's `reasoning`; trailing text + * becomes the finalAnswer fallback. + * - `usage` events sum into the trajectory usage. + * + * Like claude_code/codex, Cursor is a full harness: the runner does not own + * the loop, so this adapter reverse-maps the harness's event stream. + */ +import type { TaskSpec, Trajectory } from "@browserbasehq/stagehand"; +import { + buildTrajectory, + type NormalizedToolCall, + type TrajectoryAdapter, +} from "./trajectoryAdapter.js"; + +export interface CursorRunResult { + /** Raw SDKMessage stream collected during execution, in arrival order. */ + messages: Array>; + /** Final result text from run.wait() (falls back to trailing assistant text). */ + finalAnswer?: string; + /** Trajectory-level status. Defaults to "complete". */ + status?: Trajectory["status"]; + /** Optional usage to fold into Trajectory.usage. */ + usage?: Partial; +} + +export class CursorTrajectoryAdapter + implements TrajectoryAdapter +{ + fromHarnessResult(result: CursorRunResult, taskSpec: TaskSpec): Trajectory { + const toolCalls: NormalizedToolCall[] = []; + // call_id → index into toolCalls, so a completed event replaces its + // earlier "running" placeholder instead of duplicating the call. + const callIndexById = new Map(); + let pendingReasoning = ""; + const trailingTextParts: string[] = []; + let usageTotals = { input_tokens: 0, output_tokens: 0, cached: 0 }; + let sawUsage = false; + + for (const message of result.messages) { + const type = String(message.type ?? ""); + + if (type === "assistant") { + const text = extractAssistantText(message); + if (text) { + pendingReasoning = appendText(pendingReasoning, text); + trailingTextParts.push(text); + } + continue; + } + + if (type === "thinking" && typeof message.text === "string") { + pendingReasoning = appendText(pendingReasoning, message.text); + continue; + } + + if (type === "tool_call") { + const callId = + typeof message.call_id === "string" ? message.call_id : ""; + const status = String(message.status ?? ""); + const call: NormalizedToolCall = { + name: typeof message.name === "string" ? message.name : "tool", + args: isRecord(message.args) + ? (message.args as Record) + : { input: message.args }, + result: message.result ?? "", + ok: status !== "error", + ...(status === "error" && { + error: stringifyResult(message.result) || "tool_call error", + }), + reasoning: pendingReasoning.trim() || undefined, + }; + const existing = callId ? callIndexById.get(callId) : undefined; + if (existing !== undefined) { + // Preserve the reasoning captured with the first (running) event. + call.reasoning = toolCalls[existing].reasoning ?? call.reasoning; + toolCalls[existing] = call; + } else { + toolCalls.push(call); + if (callId) callIndexById.set(callId, toolCalls.length - 1); + pendingReasoning = ""; + trailingTextParts.length = 0; + } + continue; + } + + if (type === "usage" && isRecord(message.usage)) { + sawUsage = true; + usageTotals = { + input_tokens: + usageTotals.input_tokens + toFinite(message.usage.inputTokens), + output_tokens: + usageTotals.output_tokens + toFinite(message.usage.outputTokens), + cached: usageTotals.cached + toFinite(message.usage.cacheReadTokens), + }; + continue; + } + } + + const trailing = trailingTextParts.join("\n").trim(); + const finalAnswer = + result.finalAnswer ?? (trailing.length > 0 ? trailing : undefined); + + return buildTrajectory({ + taskSpec, + toolCalls, + finalAnswer, + status: result.status ?? "complete", + usage: + result.usage ?? + (sawUsage + ? { + input_tokens: usageTotals.input_tokens, + output_tokens: usageTotals.output_tokens, + ...(usageTotals.cached > 0 && { + cached_input_tokens: usageTotals.cached, + }), + } + : undefined), + }); + } +} + +export const cursorAdapter = new CursorTrajectoryAdapter(); + +function extractAssistantText( + message: Record, +): string | undefined { + const inner = message.message; + if (!isRecord(inner)) return undefined; + const content = inner.content; + if (!Array.isArray(content)) return undefined; + const parts: string[] = []; + for (const block of content) { + if ( + isRecord(block) && + block.type === "text" && + typeof block.text === "string" + ) { + parts.push(block.text); + } + } + return parts.length > 0 ? parts.join("\n") : undefined; +} + +function appendText(buffer: string, addition: string): string { + if (!addition) return buffer; + if (!buffer) return addition; + return `${buffer}\n${addition}`; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function toFinite(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) ? value : 0; +} + +function stringifyResult(value: unknown): string { + if (typeof value === "string") return value; + if (value === undefined || value === null) return ""; + try { + return JSON.stringify(value) ?? ""; + } catch { + return String(value); + } +} diff --git a/packages/evals/package.json b/packages/evals/package.json index 3f68d925c..0b8d21f71 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -26,6 +26,7 @@ "@anthropic-ai/claude-agent-sdk": "^0.2.141", "@anthropic-ai/sdk": "0.39.0", "@browserbasehq/stagehand": "workspace:*", + "@cursor/sdk": "1.0.23", "@openai/agents": "0.13.0", "@openai/codex-sdk": "0.125.0", "ai": "^5.0.133", diff --git a/packages/evals/tests/framework/benchHarness.test.ts b/packages/evals/tests/framework/benchHarness.test.ts index b0853b2b3..242cf5e9b 100644 --- a/packages/evals/tests/framework/benchHarness.test.ts +++ b/packages/evals/tests/framework/benchHarness.test.ts @@ -3,6 +3,7 @@ import { anthropicSdkHarness, claudeCodeHarness, codexHarness, + cursorSdkHarness, getBenchHarness, openaiAgentsSdkHarness, vercelAiSdkHarness, @@ -65,4 +66,16 @@ describe("bench harness registry", () => { /external harness execute path/, ); }); + + it("registers cursor_sdk as a concrete executable harness", async () => { + const harness = getBenchHarness("cursor_sdk"); + + expect(harness).toBe(cursorSdkHarness); + expect(harness.supportedTaskKinds).toEqual(["agent", "suite"]); + expect(harness.supportsApi).toBe(false); + expect(harness.execute).toBeDefined(); + await expect(harness.start({} as never)).rejects.toThrow( + /external harness execute path/, + ); + }); }); diff --git a/packages/evals/tests/framework/cursorSdkRunner.test.ts b/packages/evals/tests/framework/cursorSdkRunner.test.ts new file mode 100644 index 000000000..45d862adf --- /dev/null +++ b/packages/evals/tests/framework/cursorSdkRunner.test.ts @@ -0,0 +1,268 @@ +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { AvailableModel, TaskSpec } from "@browserbasehq/stagehand"; +import { EvalLogger } from "../../logger.js"; +import type { ExternalHarnessTaskPlan } from "../../framework/externalHarnessPlan.js"; +import type { PreparedExternalHarnessAdapter } from "../../framework/externalHarnessToolAdapter.js"; +import { BARE_LOOP_DEFAULT_SYSTEM_PROMPT } from "../../framework/externalHarnessToolAdapter.js"; +import { + normalizeCursorModel, + runCursorSdkAgent, + type CursorSdk, +} from "../../framework/cursorSdkRunner.js"; +import { cursorAdapter } from "../../framework/harnesses/cursorAdapter.js"; + +const plan: ExternalHarnessTaskPlan = { + dataset: "webtailbench", + taskId: "wtb-1", + startUrl: "https://example.com", + instruction: "Find the checkout button", +}; + +let tempDir: string | undefined; + +afterEach(async () => { + if (tempDir) { + await fsp.rm(tempDir, { recursive: true, force: true }); + tempDir = undefined; + } +}); + +async function makeAdapter(): Promise { + tempDir = await fsp.mkdtemp( + path.join(os.tmpdir(), "stagehand-evals-cursor-test-"), + ); + const bin = path.join(tempDir, "browse"); + await fsp.writeFile(bin, '#!/usr/bin/env bash\necho "browse-output:$@"\n', { + mode: 0o755, + }); + return { + cwd: tempDir, + env: { ...process.env } as Record, + browseBinPath: bin, + skillMode: "none", + systemPromptAddendum: BARE_LOOP_DEFAULT_SYSTEM_PROMPT, + metadata: { toolCommand: "browse", browseCliEntrypoint: bin }, + cleanup: async () => {}, + }; +} + +describe("cursor_sdk runner", () => { + it("normalizes cursor-prefixed models and rejects other providers", () => { + expect(normalizeCursorModel("cursor/composer-2.5" as AvailableModel)).toBe( + "composer-2.5", + ); + expect(normalizeCursorModel("composer-2.5" as AvailableModel)).toBe( + "composer-2.5", + ); + expect(normalizeCursorModel("cursor/default" as AvailableModel)).toBe( + "composer-2.5", + ); + expect(() => + normalizeCursorModel("openai/gpt-5.4" as AvailableModel), + ).toThrow(/only accepts cursor models/); + }); + + it("creates a local agent with the gated browse custom tool and collects the stream", async () => { + const adapter = await makeAdapter(); + let capturedCreateOptions: Record | undefined; + let capturedPrompt: string | undefined; + let closed = false; + + const sdk: CursorSdk = { + Agent: { + create: async (options) => { + capturedCreateOptions = options; + return { + send: async (message: string) => { + capturedPrompt = message; + const local = options.local as { + customTools: Record< + string, + { + execute: ( + args: Record, + ) => Promise; + } + >; + }; + const browseOutput = await local.customTools.browse.execute({ + args: "--help", + }); + return { + stream: async function* () { + yield { + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: "Checking the CLI." }], + }, + }; + yield { + type: "tool_call", + call_id: "c1", + name: "browse", + status: "completed", + args: { args: "--help" }, + result: browseOutput, + }; + yield { + type: "usage", + usage: { + inputTokens: 200, + outputTokens: 40, + cacheReadTokens: 10, + }, + }; + }, + wait: async () => ({ + status: "finished", + result: + 'EVAL_RESULT: {"success":true,"summary":"done","finalAnswer":"checkout"}', + usage: { + inputTokens: 200, + outputTokens: 40, + cacheReadTokens: 10, + }, + }), + }; + }, + close: () => { + closed = true; + }, + }; + }, + }, + }; + + const result = await runCursorSdkAgent({ + plan, + model: "cursor/composer-2.5" as AvailableModel, + logger: new EvalLogger(false), + toolAdapter: adapter, + sdk, + }); + + expect((capturedCreateOptions?.model as { id: string }).id).toBe( + "composer-2.5", + ); + const local = capturedCreateOptions?.local as Record; + expect(local.cwd).toBe(adapter.cwd); + expect(capturedPrompt).toContain(BARE_LOOP_DEFAULT_SYSTEM_PROMPT); + expect(capturedPrompt).toContain("Find the checkout button"); + expect(capturedPrompt).toContain('Use ONLY the custom "browse" tool'); + expect(closed).toBe(true); + expect(result._success).toBe(true); + expect(result.finalAnswer).toBe("checkout"); + expect(result.cursorStatus).toBe("completed"); + const metrics = result.metrics as Record; + expect(metrics.cursor_tool_calls.value).toBe(1); + expect(metrics.cursor_input_tokens.value).toBe(200); + expect(metrics.cursor_output_tokens.value).toBe(40); + expect(metrics.cursor_total_tokens.value).toBe(240); + }); + + it("returns a failed task result instead of throwing on SDK errors", async () => { + const adapter = await makeAdapter(); + const sdk: CursorSdk = { + Agent: { + create: async () => { + throw new Error("cursor exploded"); + }, + }, + }; + + const result = await runCursorSdkAgent({ + plan, + model: "cursor/composer-2.5" as AvailableModel, + logger: new EvalLogger(false), + toolAdapter: adapter, + sdk, + }); + + expect(result._success).toBe(false); + expect(result.cursorStatus).toBe("sdk_error"); + expect(result.error).toContain("cursor exploded"); + }); +}); + +describe("cursor trajectory adapter", () => { + const taskSpec: TaskSpec = { + id: "wtb-1", + instruction: "Find the checkout button", + initUrl: "https://example.com", + }; + + it("maps tool_call messages to steps with buffered reasoning and sums usage", () => { + const trajectory = cursorAdapter.fromHarnessResult( + { + messages: [ + { + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: "First I will read the help." }], + }, + }, + { type: "thinking", text: "The CLI is unfamiliar." }, + { + type: "tool_call", + call_id: "c1", + name: "browse", + status: "running", + args: { args: "--help" }, + }, + { + type: "tool_call", + call_id: "c1", + name: "browse", + status: "completed", + args: { args: "--help" }, + result: "usage: browse ...", + }, + { + type: "tool_call", + call_id: "c2", + name: "browse", + status: "error", + args: { args: "bogus" }, + result: "unknown command", + }, + { + type: "usage", + usage: { inputTokens: 100, outputTokens: 20, cacheReadTokens: 5 }, + }, + { + type: "usage", + usage: { inputTokens: 50, outputTokens: 10, cacheReadTokens: 0 }, + }, + { + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: "All done." }], + }, + }, + ], + status: "complete", + }, + taskSpec, + ); + + expect(trajectory.steps).toHaveLength(2); + expect(trajectory.steps[0].actionName).toBe("browse"); + expect(trajectory.steps[0].reasoning).toContain( + "First I will read the help.", + ); + expect(trajectory.steps[0].reasoning).toContain("The CLI is unfamiliar."); + expect(trajectory.steps[0].toolOutput?.ok).toBe(true); + expect(trajectory.steps[0].toolOutput?.result).toBe("usage: browse ..."); + expect(trajectory.steps[1].toolOutput?.ok).toBe(false); + expect(trajectory.finalAnswer).toBe("All done."); + expect(trajectory.usage.input_tokens).toBe(150); + expect(trajectory.usage.output_tokens).toBe(30); + expect(trajectory.usage.cached_input_tokens).toBe(5); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 492603ed4..8d7e49a4a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -344,6 +344,9 @@ importers: '@browserbasehq/stagehand': specifier: workspace:* version: link:../core + '@cursor/sdk': + specifier: 1.0.23 + version: 1.0.23 '@openai/agents': specifier: 0.13.0 version: 0.13.0(@aws-sdk/credential-provider-node@3.972.49)(@cfworker/json-schema@4.1.1)(@smithy/signature-v4@5.4.6)(bufferutil@4.0.9)(ws@8.21.0(bufferutil@4.0.9))(zod@4.2.1) @@ -906,6 +909,9 @@ packages: '@browserbasehq/sdk@2.14.1': resolution: {integrity: sha512-Ahmzb5+82ePgSGwouKEoAhiGy8mJ3RbCl+uigfGfVc60zseYIftcMiDPXSmrTQr1NHaATRx4cJlynP8ZUweXtg==} + '@bufbuild/protobuf@1.10.0': + resolution: {integrity: sha512-QDdVFLoN93Zjg36NoQPZfsVH9tZew7wKDKyV5qRdj8ntT4wQCOradQjRaTdwMhWUYsgKsvCINKKm87FdEk96Ag==} + '@canvas/image-data@1.1.0': resolution: {integrity: sha512-QdObRRjRbcXGmM1tmJ+MrHcaz1MftF2+W7YI+MsphnsCrmtyfS0d5qJbk0MeSbUeyM/jCb0hmnkXPsy026L7dA==} @@ -973,6 +979,24 @@ packages: '@changesets/write@0.4.0': resolution: {integrity: sha512-CdTLvIOPiCNuH71pyDu3rA+Q0n65cmAbXnwWH84rKGiFumFzkmHNT8KHTMEchcxN+Kl8I54xGUhJ7l3E7X396Q==} + '@connectrpc/connect-node@1.7.0': + resolution: {integrity: sha512-6vaPIkG/NyhxlYgytLoR9KYbPhczEboFB2OYWkA9qvUz1K7efXfeGrlRxoLtpa+r8VxyIOw73w5ktNe743nD+A==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@bufbuild/protobuf': ^1.10.0 + '@connectrpc/connect': 1.7.0 + + '@connectrpc/connect-web@1.7.0': + resolution: {integrity: sha512-qyP0YOnUPRWwCc/VfsoydMJvkb7EyUPr2q9sHgBuJzbADjiqck1gKH5V5ZPzPhTLBvmz5UvG+wiZ5sMRQHU1MQ==} + peerDependencies: + '@bufbuild/protobuf': ^1.10.0 + '@connectrpc/connect': 1.7.0 + + '@connectrpc/connect@1.7.0': + resolution: {integrity: sha512-iNKdJRi69YP3mq6AePRT8F/HrxWCewrhxnLMNm0vpqXAR8biwzRtO6Hjx80C6UvtKJ5sFmffQT7I4Baecz389w==} + peerDependencies: + '@bufbuild/protobuf': ^1.10.0 + '@csstools/color-helpers@6.0.2': resolution: {integrity: sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==} engines: {node: '>=20.19.0'} @@ -1009,6 +1033,40 @@ packages: resolution: {integrity: sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==} engines: {node: '>=20.19.0'} + '@cursor/sdk-darwin-arm64@1.0.23': + resolution: {integrity: sha512-HpltGkIDFG+XgsETJho0OiOvGLyMKqZKhh4uXtD3ZKZcgTtLvKgbbU8jVgEMa3qodtfwPz5lpwAjs5jM1zOt6w==} + cpu: [arm64] + os: [darwin] + hasBin: true + + '@cursor/sdk-darwin-x64@1.0.23': + resolution: {integrity: sha512-sjRQVhU7UL3kYSR9DzY+BAwa/iFdnPvTI3E62mIhj/2ey3LEnhIhotZQ9ymNJ0wA5BfhC7Be+JYcSMDyHxxr1g==} + cpu: [x64] + os: [darwin] + hasBin: true + + '@cursor/sdk-linux-arm64@1.0.23': + resolution: {integrity: sha512-RizTwZ2Hhhaq8bnF3yGUZWBUvLA+/lExlTAg/5Levc9f7hNoDZViIP3XaVWY/8yzPAxypeds8tWacVIaAA2/Ig==} + cpu: [arm64] + os: [linux] + hasBin: true + + '@cursor/sdk-linux-x64@1.0.23': + resolution: {integrity: sha512-s93WBbi4hrE/uisTrEfNNH1m18bwfO6wmmsnIYQ3W3gbGBriFM1ZeroHxj1F6paMDE7o/dfUeW22lQ72rRZrSA==} + cpu: [x64] + os: [linux] + hasBin: true + + '@cursor/sdk-win32-x64@1.0.23': + resolution: {integrity: sha512-d1p1+tRJbNZTUqw8SPEW9OYtU02ynZysASDDvhpaEFtbPwGFUrez9sVbFHpbuXeOXL//H+faiHJanVFsUNfIZw==} + cpu: [x64] + os: [win32] + hasBin: true + + '@cursor/sdk@1.0.23': + resolution: {integrity: sha512-VIh8oW89XXACUkQqB2N8TaU3A/y2jQieF++VC6QqIqKhKRKj7kd+pzeh43MXusuPpebtxtEzqvjaCLlc1xbYTQ==} + engines: {node: '>=22.13'} + '@emnapi/runtime@1.7.1': resolution: {integrity: sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==} @@ -2482,6 +2540,12 @@ packages: '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + '@statsig/client-core@3.31.0': + resolution: {integrity: sha512-SuxQD6TmVszPG7FoMKwTk/uyBuVFk7XnxI3T/E0uyb7PL7GNjONtfsoh+NqBBVUJVse0CUeSFfgJPoZy1ZOslQ==} + + '@statsig/js-client@3.31.0': + resolution: {integrity: sha512-LFa5E0LjT6sTfZv3sNGoyRLSZ1078+agdgOA+Vm1ecjG+KbSOfBLTW7hMwimrJ29slRwbYDzbtKaPJo/R37N2g==} + '@stoplight/better-ajv-errors@1.0.3': resolution: {integrity: sha512-0p9uXkuB22qGdNfy3VeEhxkU5uwvp/KrBTAbrLBURv6ilxIVwanKwjMc41lQfIVgPGcOkmLbTolfFrSsueu7zA==} engines: {node: ^12.20 || >= 14.13} @@ -8247,6 +8311,8 @@ snapshots: transitivePeerDependencies: - encoding + '@bufbuild/protobuf@1.10.0': {} + '@canvas/image-data@1.1.0': {} '@cfworker/json-schema@4.1.1': @@ -8410,6 +8476,21 @@ snapshots: human-id: 4.1.3 prettier: 2.8.8 + '@connectrpc/connect-node@1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0))': + dependencies: + '@bufbuild/protobuf': 1.10.0 + '@connectrpc/connect': 1.7.0(@bufbuild/protobuf@1.10.0) + undici: 7.28.0 + + '@connectrpc/connect-web@1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0))': + dependencies: + '@bufbuild/protobuf': 1.10.0 + '@connectrpc/connect': 1.7.0(@bufbuild/protobuf@1.10.0) + + '@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0)': + dependencies: + '@bufbuild/protobuf': 1.10.0 + '@csstools/color-helpers@6.0.2': {} '@csstools/css-calc@3.2.0(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0))(@csstools/css-tokenizer@4.0.0)': @@ -8434,6 +8515,36 @@ snapshots: '@csstools/css-tokenizer@4.0.0': {} + '@cursor/sdk-darwin-arm64@1.0.23': + optional: true + + '@cursor/sdk-darwin-x64@1.0.23': + optional: true + + '@cursor/sdk-linux-arm64@1.0.23': + optional: true + + '@cursor/sdk-linux-x64@1.0.23': + optional: true + + '@cursor/sdk-win32-x64@1.0.23': + optional: true + + '@cursor/sdk@1.0.23': + dependencies: + '@bufbuild/protobuf': 1.10.0 + '@connectrpc/connect': 1.7.0(@bufbuild/protobuf@1.10.0) + '@connectrpc/connect-node': 1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0)) + '@connectrpc/connect-web': 1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0)) + '@statsig/js-client': 3.31.0 + zod: 3.25.76 + optionalDependencies: + '@cursor/sdk-darwin-arm64': 1.0.23 + '@cursor/sdk-darwin-x64': 1.0.23 + '@cursor/sdk-linux-arm64': 1.0.23 + '@cursor/sdk-linux-x64': 1.0.23 + '@cursor/sdk-win32-x64': 1.0.23 + '@emnapi/runtime@1.7.1': dependencies: tslib: 2.8.1 @@ -10256,6 +10367,12 @@ snapshots: '@standard-schema/spec@1.1.0': {} + '@statsig/client-core@3.31.0': {} + + '@statsig/js-client@3.31.0': + dependencies: + '@statsig/client-core': 3.31.0 + '@stoplight/better-ajv-errors@1.0.3(ajv@8.20.0)': dependencies: ajv: 8.20.0 From 161cd410b9ab1a0fec51a480758125180edb0126 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 9 Jul 2026 10:05:31 -0400 Subject: [PATCH 2/5] docs(evals): drop stale design-doc references from docblocks Co-Authored-By: Claude Fable 5 --- packages/evals/framework/cursorSdkRunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/evals/framework/cursorSdkRunner.ts b/packages/evals/framework/cursorSdkRunner.ts index 723b4d9b0..2a0128e59 100644 --- a/packages/evals/framework/cursorSdkRunner.ts +++ b/packages/evals/framework/cursorSdkRunner.ts @@ -9,7 +9,7 @@ * harness) and prompt the agent to use only that tool. The SDK does not * expose an allow-list to hard-disable its native shell tools, so the browse * gating here is prompt + custom-tool discipline rather than the Claude Code - * canUseTool hard gate — recorded as a known limitation in the design doc. + * canUseTool hard gate (see packages/evals/README.md#external-harnesses). * * Auth: CURSOR_API_KEY (the SDK's own default env var). Model ids come from * Cursor's catalog (e.g. "composer-2.5" / "cursor/composer-2.5" via -m). From 8655910d8b005ea196fb857b948006a6a2d8a021 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 9 Jul 2026 11:48:50 -0400 Subject: [PATCH 3/5] fix(evals): propagate tool failures through cursor_sdk's custom-tool result + clip tool output Two comparability bugs in the cursor_sdk custom tool: - execute() returned the bare output string and discarded `ok` from runBareBrowseCommand. Since runBareBrowseCommand never throws, the SDK had no signal a browse command failed, so every tool call read back as "completed" and cursorAdapter (which derives ok from that status) recorded every call as ok:true. Return the SDK's own SDKCustomToolResult shape ({content, isError}, per @cursor/sdk's options.d.ts) with isError: !ok instead. - The tool ran runBareBrowseCommand directly, bypassing the 20k output clip every other harness gets via createBareLoopToolRecorder's readToolOutputLimit. Apply the same clip here so tool-output sizes are comparable across all five harnesses. Also strengthens the test: the previous mock hardcoded tool_call status "completed" regardless of the tool result. Added a case that fails the underlying browse command and asserts isError:true out of the custom tool AND ok:false once that isError flows through a terminal tool_call event into the recorded trajectory. Co-Authored-By: Claude Fable 5 --- packages/evals/framework/cursorSdkRunner.ts | 29 ++++- .../tests/framework/cursorSdkRunner.test.ts | 102 ++++++++++++++++++ 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/packages/evals/framework/cursorSdkRunner.ts b/packages/evals/framework/cursorSdkRunner.ts index 2a0128e59..87fb39267 100644 --- a/packages/evals/framework/cursorSdkRunner.ts +++ b/packages/evals/framework/cursorSdkRunner.ts @@ -28,6 +28,7 @@ import { BROWSE_TOOL_DESCRIPTION, BROWSE_TOOL_NAME, buildBareLoopUserPrompt, + readToolOutputLimit, stringifyLoopError, stripProviderPrefix, } from "./bareLoopRunner.js"; @@ -41,6 +42,17 @@ import { type MetricValue = { count: number; value: number }; type CursorSdkMessage = Record; +/** + * Shape of `@cursor/sdk`'s `SDKCustomToolResult` (see options.d.ts) that we + * actually use: text content plus the `isError` flag. Returning the bare + * output string (as before) gives the SDK no way to know the browse command + * failed, so every call reads back as "completed" regardless of `ok`. + */ +type BrowseCustomToolResult = { + content: [{ type: "text"; text: string }]; + isError: boolean; +}; + const HARNESS = "cursor_sdk"; export const DEFAULT_CURSOR_MODEL = "composer-2.5"; @@ -146,12 +158,23 @@ export async function runCursorSdkAgent( }, required: ["args"], }, - execute: async (args: Record) => { - const { output } = await runBareBrowseCommand( + execute: async ( + args: Record, + ): Promise => { + const { ok, output } = await runBareBrowseCommand( input.toolAdapter, String(args.args ?? ""), ); - return output; + // Same 20k clip the bare-loop recorder applies (bareLoopRunner's + // readToolOutputLimit) — Cursor bypasses createBareLoopToolRecorder + // entirely (it's a full harness, not a bare loop), so this is + // applied directly here to keep tool-output sizes comparable + // across all external harnesses. + const clipped = clip(output, readToolOutputLimit()); + return { + content: [{ type: "text", text: clipped }], + isError: !ok, + }; }, }, }, diff --git a/packages/evals/tests/framework/cursorSdkRunner.test.ts b/packages/evals/tests/framework/cursorSdkRunner.test.ts index 45d862adf..368fe2cd1 100644 --- a/packages/evals/tests/framework/cursorSdkRunner.test.ts +++ b/packages/evals/tests/framework/cursorSdkRunner.test.ts @@ -164,6 +164,108 @@ describe("cursor_sdk runner", () => { expect(metrics.cursor_total_tokens.value).toBe(240); }); + it("propagates a failing browse command's isError through the recorded trajectory", async () => { + const adapter = await makeAdapter(); + // Overwrite the stub `browse` binary to fail, so runBareBrowseCommand + // returns ok:false. + await fsp.writeFile( + adapter.browseBinPath, + '#!/usr/bin/env bash\necho "boom" >&2\nexit 1\n', + { mode: 0o755 }, + ); + + let capturedToolResult: unknown; + const sdk: CursorSdk = { + Agent: { + create: async (options) => { + return { + send: async () => { + const local = options.local as { + customTools: Record< + string, + { + execute: (args: Record) => Promise<{ + content: Array<{ type: string; text: string }>; + isError: boolean; + }>; + } + >; + }; + capturedToolResult = await local.customTools.browse.execute({ + args: "open https://example.com", + }); + const { isError, content } = capturedToolResult as { + isError: boolean; + content: Array<{ type: string; text: string }>; + }; + return { + // Mirrors what the real SDK does with a custom tool's + // isError/content: a terminal tool_call event whose status + // reflects isError, carrying the same content as `result`. + stream: async function* () { + yield { + type: "tool_call", + call_id: "c1", + name: "browse", + status: isError ? "error" : "completed", + args: { args: "open https://example.com" }, + result: content[0]?.text ?? "", + }; + }, + wait: async () => ({ + status: "finished", + result: + 'EVAL_RESULT: {"success":false,"summary":"browse command failed"}', + }), + }; + }, + close: () => {}, + }; + }, + }, + }; + + await runCursorSdkAgent({ + plan, + model: "cursor/composer-2.5" as AvailableModel, + logger: new EvalLogger(false), + toolAdapter: adapter, + sdk, + }); + + // The custom tool itself must report isError:true — not just a bare + // output string the SDK has no way to read as a failure. + expect(capturedToolResult).toMatchObject({ isError: true }); + const toolContent = ( + capturedToolResult as { content: Array<{ text: string }> } + ).content[0].text; + expect(toolContent).toContain("boom"); + + // And once that isError flows through a terminal tool_call event (as the + // real SDK does), the trajectory adapter must land it as ok:false. + const trajectory = cursorAdapter.fromHarnessResult( + { + messages: [ + { + type: "tool_call", + call_id: "c1", + name: "browse", + status: "error", + args: { args: "open https://example.com" }, + result: toolContent, + }, + ], + status: "error", + }, + { + id: "wtb-1", + instruction: "Find the checkout button", + initUrl: "https://example.com", + }, + ); + expect(trajectory.steps[0].toolOutput?.ok).toBe(false); + }); + it("returns a failed task result instead of throwing on SDK errors", async () => { const adapter = await makeAdapter(); const sdk: CursorSdk = { From 31d69846a826f3b5cb24f03d7032acc721cdb84a Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 9 Jul 2026 12:17:42 -0400 Subject: [PATCH 4/5] docs(evals): present-tense phrasing in cursor_sdk runner comments Co-Authored-By: Claude Fable 5 --- packages/evals/framework/cursorSdkRunner.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/evals/framework/cursorSdkRunner.ts b/packages/evals/framework/cursorSdkRunner.ts index 87fb39267..c5db820be 100644 --- a/packages/evals/framework/cursorSdkRunner.ts +++ b/packages/evals/framework/cursorSdkRunner.ts @@ -44,9 +44,9 @@ type CursorSdkMessage = Record; /** * Shape of `@cursor/sdk`'s `SDKCustomToolResult` (see options.d.ts) that we - * actually use: text content plus the `isError` flag. Returning the bare - * output string (as before) gives the SDK no way to know the browse command - * failed, so every call reads back as "completed" regardless of `ok`. + * actually use: text content plus the `isError` flag. Returning a bare + * output string would give the SDK no way to know the browse command + * failed — every call would read back as "completed" regardless of `ok`. */ type BrowseCustomToolResult = { content: [{ type: "text"; text: string }]; @@ -109,7 +109,7 @@ export function buildCursorPrompt( systemPromptAddendum: string, ): string { // Cursor's SDK has no separate system-prompt channel for local agents; the - // skill-arm text rides at the top of the single prompt instead. + // skill-mode text rides at the top of the single prompt instead. return [ systemPromptAddendum, `Use ONLY the custom "${BROWSE_TOOL_NAME}" tool for browser work — do not use your shell for browsing, and do not edit repository files.`, From ab81880a72f5c37f00577ba0918bfbd0b7be5bf3 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 9 Jul 2026 13:23:24 -0400 Subject: [PATCH 5/5] fix(evals): only record terminal cursor tool_call events; dedup tool-call metric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cursorAdapter: non-terminal (running) tool_call events no longer become trajectory steps — an interrupted run must not surface an in-flight call as a successful step. Reasoning buffered before a running event still carries over to its terminal event. - cursorSdkRunner: cursor_tool_calls counts distinct terminal call_ids, matching the adapter's duplicate collapsing. Co-Authored-By: Claude Fable 5 --- packages/evals/framework/cursorSdkRunner.ts | 20 ++++++++-- .../framework/harnesses/cursorAdapter.ts | 32 ++++++++++++--- .../tests/framework/cursorSdkRunner.test.ts | 40 +++++++++++++++++++ 3 files changed, 83 insertions(+), 9 deletions(-) diff --git a/packages/evals/framework/cursorSdkRunner.ts b/packages/evals/framework/cursorSdkRunner.ts index c5db820be..4cbe4e7d4 100644 --- a/packages/evals/framework/cursorSdkRunner.ts +++ b/packages/evals/framework/cursorSdkRunner.ts @@ -274,9 +274,23 @@ function buildCursorMetrics( | undefined, messages: CursorSdkMessage[], ): Record { - const toolCalls = messages.filter( - (message) => message.type === "tool_call" && message.status !== "running", - ).length; + // Count distinct terminal tool calls: the SDK can emit more than one + // terminal event for a call_id, and the trajectory adapter collapses those + // duplicates — the metric should agree with the recorded trajectory. + const seenCallIds = new Set(); + let anonymousCalls = 0; + for (const message of messages) { + if (message.type !== "tool_call") continue; + const status = String(message.status ?? ""); + if (status !== "completed" && status !== "error") continue; + const callId = typeof message.call_id === "string" ? message.call_id : ""; + if (callId) { + seenCallIds.add(callId); + } else { + anonymousCalls += 1; + } + } + const toolCalls = seenCallIds.size + anonymousCalls; return { cursor_tool_calls: metricValue(toolCalls), cursor_input_tokens: metricValue(usage?.inputTokens), diff --git a/packages/evals/framework/harnesses/cursorAdapter.ts b/packages/evals/framework/harnesses/cursorAdapter.ts index 4916c7e6f..82253d0c8 100644 --- a/packages/evals/framework/harnesses/cursorAdapter.ts +++ b/packages/evals/framework/harnesses/cursorAdapter.ts @@ -10,8 +10,11 @@ * * Mapping: * - Each terminal `tool_call` event (status completed|error) becomes one - * normalized tool call. Duplicate events for the same call_id collapse to - * the last (terminal) one. + * normalized tool call. Non-terminal events (e.g. "running") are progress + * markers, not results — an interrupted run must not record an in-flight + * call as a step — but the reasoning buffered before them carries over to + * the terminal event. Duplicate terminal events for one call_id collapse + * to the last. * - `assistant` text blocks and `thinking` text buffered since the previous * tool call fold into the next tool call's `reasoning`; trailing text * becomes the finalAnswer fallback. @@ -43,9 +46,12 @@ export class CursorTrajectoryAdapter { fromHarnessResult(result: CursorRunResult, taskSpec: TaskSpec): Trajectory { const toolCalls: NormalizedToolCall[] = []; - // call_id → index into toolCalls, so a completed event replaces its - // earlier "running" placeholder instead of duplicating the call. + // call_id → index into toolCalls, so duplicate terminal events for the + // same call replace their predecessor instead of duplicating the call. const callIndexById = new Map(); + // Reasoning buffered when a non-terminal ("running") event arrives, keyed + // by call_id so the terminal event still picks it up. + const parkedReasoningById = new Map(); let pendingReasoning = ""; const trailingTextParts: string[] = []; let usageTotals = { input_tokens: 0, output_tokens: 0, cached: 0 }; @@ -72,6 +78,17 @@ export class CursorTrajectoryAdapter const callId = typeof message.call_id === "string" ? message.call_id : ""; const status = String(message.status ?? ""); + if (status !== "completed" && status !== "error") { + // In-flight event: park the buffered reasoning for the terminal + // event and record nothing — an interrupted run must not surface + // an unfinished call as a (successful) step. + if (callId && pendingReasoning.trim()) { + parkedReasoningById.set(callId, pendingReasoning.trim()); + } + pendingReasoning = ""; + trailingTextParts.length = 0; + continue; + } const call: NormalizedToolCall = { name: typeof message.name === "string" ? message.name : "tool", args: isRecord(message.args) @@ -82,11 +99,14 @@ export class CursorTrajectoryAdapter ...(status === "error" && { error: stringifyResult(message.result) || "tool_call error", }), - reasoning: pendingReasoning.trim() || undefined, + reasoning: + (callId ? parkedReasoningById.get(callId) : undefined) ?? + (pendingReasoning.trim() || undefined), }; const existing = callId ? callIndexById.get(callId) : undefined; if (existing !== undefined) { - // Preserve the reasoning captured with the first (running) event. + // Duplicate terminal event: keep the last, preserving the first + // recorded reasoning. call.reasoning = toolCalls[existing].reasoning ?? call.reasoning; toolCalls[existing] = call; } else { diff --git a/packages/evals/tests/framework/cursorSdkRunner.test.ts b/packages/evals/tests/framework/cursorSdkRunner.test.ts index 368fe2cd1..f774edda5 100644 --- a/packages/evals/tests/framework/cursorSdkRunner.test.ts +++ b/packages/evals/tests/framework/cursorSdkRunner.test.ts @@ -108,6 +108,16 @@ describe("cursor_sdk runner", () => { args: { args: "--help" }, result: browseOutput, }; + // Duplicate terminal event for the same call_id: metrics + // must count it once, matching the trajectory adapter. + yield { + type: "tool_call", + call_id: "c1", + name: "browse", + status: "completed", + args: { args: "--help" }, + result: browseOutput, + }; yield { type: "usage", usage: { @@ -367,4 +377,34 @@ describe("cursor trajectory adapter", () => { expect(trajectory.usage.output_tokens).toBe(30); expect(trajectory.usage.cached_input_tokens).toBe(5); }); + + it("does not record an in-flight (running-only) tool call as a step", () => { + const trajectory = cursorAdapter.fromHarnessResult( + { + messages: [ + { + type: "tool_call", + call_id: "c1", + name: "browse", + status: "completed", + args: { args: "--help" }, + result: "usage: browse ...", + }, + { + // Interrupted mid-call: no terminal event ever arrives for c2. + type: "tool_call", + call_id: "c2", + name: "browse", + status: "running", + args: { args: "open https://example.com" }, + }, + ], + status: "error", + }, + taskSpec, + ); + + expect(trajectory.steps).toHaveLength(1); + expect(trajectory.steps[0].toolOutput?.ok).toBe(true); + }); });