From 6ec3fec9471f0c1eb19c090262766627016f1e1f Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Wed, 24 Jun 2026 11:18:04 +0100 Subject: [PATCH 1/2] feat(eval): add OpenCode agent harness Adds `opencode` (OpenCode CLI) as an eval agent on the agents framework: an agents/opencode module (runner + parser + factory + registry definition) plus experiments. Orchestration is unchanged; opencode's transcript is parsed into the same surface scorers use. Runs in both modes, like Claude Code / Codex. - runner.ts: install `opencode-ai`, then `opencode run --format json --dangerously-skip-permissions < /dev/null` (opencode blocks on stdin otherwise). Multi-provider: `provider/model` ids typed from the vendor SDKs (anthropic/openai/google); `apiKeyEnvVar` + `modelProvider` resolved from the prefix. MCP written to an OPENCODE_CONFIG file in scratch. - parser.ts: the 1.15 JSONL schema (text / tool_use / reasoning / error / step_*), paired tool_call + tool_result by callID. - engine.ts + types.ts: multi-provider CLIs carry an optional `modelProvider`; the engine prefers it over the per-agent-id mapping. `requireApiKey` uses the shared `requireEnv`. - eval-metadata: `opencode` harness id; `google` model provider. - Experiments: opencode-claude-sonnet-5, opencode-gpt-5.4-mini, opencode-gemini-flash. Adds @ai-sdk/google for the Gemini model-id types. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 3 + apps/web/src/App.tsx | 9 +- experiments/opencode-claude-sonnet-5.ts | 22 ++ experiments/opencode-gemini-flash.ts | 24 ++ experiments/opencode-gpt-5.4-mini.ts | 21 ++ packages/core/package.json | 3 +- packages/core/src/agents/engine.ts | 18 +- packages/core/src/agents/opencode/index.ts | 42 ++++ .../core/src/agents/opencode/parser.test.ts | 193 +++++++++++++++ packages/core/src/agents/opencode/parser.ts | 230 ++++++++++++++++++ .../core/src/agents/opencode/runner.test.ts | 82 +++++++ packages/core/src/agents/opencode/runner.ts | 211 ++++++++++++++++ packages/core/src/agents/registry.ts | 7 +- packages/core/src/agents/shared.test.ts | 28 +++ packages/core/src/agents/shared.ts | 29 ++- packages/core/src/agents/types.ts | 6 + packages/core/src/eval-metadata.ts | 9 +- packages/core/src/index.ts | 3 +- pnpm-lock.yaml | 39 +++ pnpm-workspace.yaml | 1 + 20 files changed, 962 insertions(+), 18 deletions(-) create mode 100644 experiments/opencode-claude-sonnet-5.ts create mode 100644 experiments/opencode-gemini-flash.ts create mode 100644 experiments/opencode-gpt-5.4-mini.ts create mode 100644 packages/core/src/agents/opencode/index.ts create mode 100644 packages/core/src/agents/opencode/parser.test.ts create mode 100644 packages/core/src/agents/opencode/parser.ts create mode 100644 packages/core/src/agents/opencode/runner.test.ts create mode 100644 packages/core/src/agents/opencode/runner.ts create mode 100644 packages/core/src/agents/shared.test.ts diff --git a/.env.example b/.env.example index 19a0696f..ff3d14b2 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,6 @@ # AI SDK Core direct-provider credentials. ANTHROPIC_API_KEY= OPENAI_API_KEY= + +# Opencode Gemini API key +GOOGLE_GENERATIVE_AI_API_KEY= diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index a891a817..d428fd2e 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -198,6 +198,7 @@ const AGENT_LABELS = { "ai-sdk": "AI SDK", "claude-code": "Claude Code", codex: "Codex", + opencode: "OpenCode", } satisfies Record const EXPERIMENT_SUITES = ["benchmark", "no-skills"] as const @@ -313,11 +314,15 @@ function formatOpenAiModel(modelId: string) { } function formatModel(display: ExperimentDisplay) { + // opencode model ids are `provider/model`; strip the prefix for display. + const modelId = display.modelId.replace(`${display.modelProvider}/`, "") switch (display.modelProvider) { case "anthropic": - return formatAnthropicModel(display.modelId) + return formatAnthropicModel(modelId) case "openai": - return formatOpenAiModel(display.modelId) + return formatOpenAiModel(modelId) + case "google": + return modelId } } diff --git a/experiments/opencode-claude-sonnet-5.ts b/experiments/opencode-claude-sonnet-5.ts new file mode 100644 index 00000000..89e9d10c --- /dev/null +++ b/experiments/opencode-claude-sonnet-5.ts @@ -0,0 +1,22 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +// OpenCode is a CLI agent driving Claude Sonnet 5. Like Claude Code / Codex it +// runs in both modes: `runtime` supplies the MCP servers for tools-mode evals +// (written into opencode's config) and `localStack` drives local-stack evals. +// Which mode an eval uses is a property of the eval, not the agent. +export default defineExperiment({ + agent: opencodeAgent({ + model: 'anthropic/claude-sonnet-5', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ['supabase', 'supabase-postgres-best-practices'], +}); diff --git a/experiments/opencode-gemini-flash.ts b/experiments/opencode-gemini-flash.ts new file mode 100644 index 00000000..f2f68c8d --- /dev/null +++ b/experiments/opencode-gemini-flash.ts @@ -0,0 +1,24 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +// OpenCode driving Google's latest Gemini Flash (cheapest tier). Runs in both +// modes (see opencode-claude-sonnet-5.ts); the `google/` prefix selects the +// GOOGLE_GENERATIVE_AI_API_KEY credential (Google AI Studio, not Vertex). +// `gemini-flash-latest` tracks the newest Flash — the only Gemini Flash id that +// the AI-Studio key serves end-to-end (pinned 2.5/3.x-flash ids returned no +// output via opencode 1.15.7). +export default defineExperiment({ + agent: opencodeAgent({ + model: 'google/gemini-flash-latest', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ['supabase', 'supabase-postgres-best-practices'], +}); diff --git a/experiments/opencode-gpt-5.4-mini.ts b/experiments/opencode-gpt-5.4-mini.ts new file mode 100644 index 00000000..4937744c --- /dev/null +++ b/experiments/opencode-gpt-5.4-mini.ts @@ -0,0 +1,21 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +// OpenCode driving OpenAI GPT-5.4 mini. Runs in both modes (see opencode-claude- +// sonnet-5.ts); the `openai/` model prefix selects the OPENAI_API_KEY +// credential. +export default defineExperiment({ + agent: opencodeAgent({ + model: 'openai/gpt-5.4-mini', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ['supabase', 'supabase-postgres-best-practices'], +}); diff --git a/packages/core/package.json b/packages/core/package.json index 4cce8026..bf84b15f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -18,7 +18,7 @@ }, "dependencies": { "@anthropic-ai/sdk": "catalog:", - "openai": "catalog:", + "@ai-sdk/google": "catalog:", "@ai-sdk/mcp": "catalog:", "@ai-sdk/openai": "catalog:", "@supabase-evals/platform-lite": "workspace:*", @@ -26,6 +26,7 @@ "ai": "catalog:", "executor": "1.4.29", "gray-matter": "^4.0.3", + "openai": "catalog:", "typescript": "catalog:", "zod": "catalog:" } diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index 36dffe39..e5f17129 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -29,6 +29,7 @@ import { SYSTEM_PROMPT_PATH, USER_PROMPT_PATH, processStopReason, + requireEnv, rewriteLoopback, writeSandboxFile, } from './shared.js'; @@ -39,6 +40,10 @@ function modelProviderForAgent(id: AgentRunner['id']): ModelProvider { return 'anthropic'; case 'codex': return 'openai'; + case 'opencode': + throw new Error( + 'opencode is multi-provider; its runner sets `modelProvider` from the model id' + ); case 'ai-sdk': throw new Error('ai-sdk agents are not created through createCliAgent'); } @@ -60,7 +65,7 @@ export function createCliAgent( modelId: options.model, metadata: { agent: runner.id, - modelProvider: modelProviderForAgent(runner.id), + modelProvider: runner.modelProvider ?? modelProviderForAgent(runner.id), modelId: options.model, ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } @@ -117,11 +122,8 @@ export function createCliAgent( } function requireApiKey(runner: AgentRunner): string { - const apiKey = process.env[runner.apiKeyEnvVar]; - if (!apiKey) { - throw new Error( - `Missing ${runner.displayName} credentials. Set ${runner.apiKeyEnvVar} before running ${runner.id} evals.` - ); - } - return apiKey; + return requireEnv( + runner.apiKeyEnvVar, + `Set it to run ${runner.displayName} (${runner.id}) evals.` + ); } diff --git a/packages/core/src/agents/opencode/index.ts b/packages/core/src/agents/opencode/index.ts new file mode 100644 index 00000000..10c33080 --- /dev/null +++ b/packages/core/src/agents/opencode/index.ts @@ -0,0 +1,42 @@ +/** + * OpenCode agent. Owns everything opencode-specific: it wires its own runner + + * parser into the public `opencodeAgent` factory (via the generic + * `createCliAgent` engine) and exports the registry definition the harness uses + * to parse opencode transcripts. Runs in both modes, like Claude Code / Codex. + */ + +import type { AgentHarness } from '../../index.js'; +import { createCliAgent } from '../engine.js'; +import type { AgentDefinition } from '../types.js'; +import { + DEFAULT_OPENCODE_MODEL, + createOpencodeRunner, + type OpenCodeModel, +} from './runner.js'; +import { opencodeParser } from './parser.js'; + +/** + * OpenCode as an `AgentHarness`. Multi-provider: the `provider/model` id selects + * the credential (anthropic / openai / google), so the runner is built per-model + * with the matching API-key env var and provider. + */ +export function opencodeAgent( + options: { + /** opencode model id, `provider/model` (e.g. `openai/gpt-5.4`). */ + model?: OpenCodeModel; + /** Override the pinned CLI version. */ + cliVersion?: string; + } = {} +): AgentHarness { + const model = options.model ?? DEFAULT_OPENCODE_MODEL; + return createCliAgent(createOpencodeRunner(model), opencodeParser, { + model, + cliVersion: options.cliVersion, + }); +} + +/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */ +export const opencodeDefinition: AgentDefinition = { + runner: createOpencodeRunner(DEFAULT_OPENCODE_MODEL), + parser: opencodeParser, +}; diff --git a/packages/core/src/agents/opencode/parser.test.ts b/packages/core/src/agents/opencode/parser.test.ts new file mode 100644 index 00000000..c2760ee8 --- /dev/null +++ b/packages/core/src/agents/opencode/parser.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from 'vitest'; +import { opencodeParser } from './parser.js'; +import { adaptTranscript } from '../../parsers/adapt.js'; + +/** A representative `opencode run --format json` stream (shapes from CLI 1.15.7). */ +const SESSION = [ + JSON.stringify({ type: 'step_start', part: { type: 'step-start' } }), + JSON.stringify({ + type: 'reasoning', + timestamp: 1782295624200, + part: { type: 'reasoning', text: 'I should list the files.' }, + }), + JSON.stringify({ + type: 'text', + timestamp: 1782295624232, + part: { type: 'text', text: 'Listing files.' }, + }), + JSON.stringify({ + type: 'tool_use', + timestamp: 1782295624290, + part: { + type: 'tool', + tool: 'bash', + callID: 'tool_1', + state: { + status: 'completed', + input: { command: 'ls -la', description: 'List files' }, + output: 'file1\nfile2', + metadata: { exit: 0 }, + }, + }, + }), + JSON.stringify({ + type: 'tool_use', + timestamp: 1782295624300, + part: { + type: 'tool', + tool: 'write', + callID: 'tool_2', + state: { + status: 'completed', + input: { filePath: '/work/note.txt', content: 'hi' }, + output: 'written', + }, + }, + }), + JSON.stringify({ + type: 'text', + timestamp: 1782295624400, + part: { type: 'text', text: 'Done.' }, + }), + JSON.stringify({ + type: 'step_finish', + part: { + type: 'step-finish', + reason: 'stop', + tokens: { input: 3, output: 6 }, + }, + }), +].join('\n'); + +describe('opencodeParser', () => { + it('maps bash + write to canonical tool calls, paired with results by callID', () => { + const { events, errors } = opencodeParser.parseTranscript(SESSION); + expect(errors).toEqual([]); + + const calls = events.filter((e) => e.type === 'tool_call'); + expect(calls.map((e) => e.tool?.name)).toEqual(['shell', 'file_write']); + expect(calls.map((e) => e.tool?.originalName)).toEqual(['bash', 'write']); + expect(calls.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']); + // Normalized views on the event; raw args untouched. + expect(calls[0].tool?.command).toBe('ls -la'); + expect(calls[1].tool?.path).toBe('/work/note.txt'); + + const results = events.filter((e) => e.type === 'tool_result'); + expect(results.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']); + expect(results.every((e) => e.tool?.success === true)).toBe(true); + }); + + it('surfaces reasoning + the assistant report via the adapter', () => { + const events = opencodeParser.parseTranscript(SESSION).events; + expect( + events.some( + (e) => e.type === 'thinking' && e.content === 'I should list the files.' + ) + ).toBe(true); + + const adapted = adaptTranscript(events); + expect(adapted.agentReport).toBe('Done.'); + expect(adapted.steps).toBe(2); // two assistant text turns + expect(adapted.toolCalls).toEqual([ + { + endpoint: 'bash', + body: { command: 'ls -la', description: 'List files' }, + name: 'shell', + command: 'ls -la', + result: 'file1\nfile2', + error: undefined, + ts: 1782295624290, // epoch ms preserved through toISO -> parseTs + }, + { + endpoint: 'write', + body: { filePath: '/work/note.txt', content: 'hi' }, + name: 'file_write', + path: '/work/note.txt', + result: 'written', + error: undefined, + ts: 1782295624300, + }, + ]); + }); + + it('surfaces skill loads from the skill tool and from SKILL.md reads', () => { + const stream = [ + JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'skill', + callID: 's1', + state: { + status: 'completed', + input: { name: 'supabase' }, + output: '# Supabase', + }, + }, + }), + JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'read', + callID: 's2', + state: { + status: 'completed', + input: { + filePath: + '.claude/skills/supabase-postgres-best-practices/SKILL.md', + }, + output: '# Postgres', + }, + }, + }), + ].join('\n'); + const adapted = adaptTranscript( + opencodeParser.parseTranscript(stream).events + ); + expect(adapted.toolCalls.map((call) => call.loadedSkill)).toEqual([ + 'supabase', + 'supabase-postgres-best-practices', + ]); + }); + + it('marks a non-zero shell exit as failed (error surfaced via adapter)', () => { + const stream = JSON.stringify({ + type: 'tool_use', + part: { + type: 'tool', + tool: 'bash', + callID: 'c1', + state: { + status: 'completed', + input: { command: 'false' }, + output: 'nope', + metadata: { exit: 1 }, + }, + }, + }); + const events = opencodeParser.parseTranscript(stream).events; + expect(events.find((e) => e.type === 'tool_result')?.tool?.success).toBe( + false + ); + const adapted = adaptTranscript(events); + expect(adapted.toolCalls[0].error).toBe('nope'); + expect(adapted.toolCalls[0].result).toBeUndefined(); + }); + + it('emits an error event and never throws on malformed lines', () => { + const { events, errors } = opencodeParser.parseTranscript( + 'not json\n' + + JSON.stringify({ type: 'error', error: { message: 'boom' } }) + ); + expect(events).toEqual([ + { + timestamp: undefined, + type: 'error', + content: 'boom', + raw: { type: 'error', error: { message: 'boom' } }, + }, + ]); + expect(errors.length).toBe(1); + }); +}); diff --git a/packages/core/src/agents/opencode/parser.ts b/packages/core/src/agents/opencode/parser.ts new file mode 100644 index 00000000..628d8120 --- /dev/null +++ b/packages/core/src/agents/opencode/parser.ts @@ -0,0 +1,230 @@ +/** + * OpenCode transcript parser — for `opencode run --format json` (CLI ≥ 1.15). + * + * The stream is newline-delimited event records, each `{ type, timestamp, + * sessionID, part }`: + * {"type":"step_start","part":{"type":"step-start"}} + * {"type":"text","part":{"type":"text","text":"…"}} + * {"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"…", + * "state":{"status":"completed","input":{…},"output":"…", + * "metadata":{"exit":0}}}} + * {"type":"reasoning","part":{"type":"reasoning","text":"…"}} + * {"type":"error","error":{"message":"…"}} + * {"type":"step_finish","part":{"type":"step-finish","reason":"stop","tokens":{…}}} + * + * A `tool_use` record is self-contained (input + output + status), so it yields + * a paired tool_call + tool_result correlated by `part.callID`. Step records + * carry token/finish info and produce no transcript event (the runner reads the + * terminal `step_finish` reason for the stop reason). + * + * Adapted from `@supabase/agent-evals` (packages/agent-eval/src/parsers). + */ + +import { isRecord, parseJsonlRecords } from '../../json.js'; +import type { + ParsedTranscript, + TranscriptEvent, +} from '../../transcript/types.js'; +import type { AgentTranscriptParser } from '../../parsers/types.js'; +import { + normalizeToolName, + type AgentToolMap, +} from '../../parsers/shared/normalize.js'; +import { + extractArgs, + extractLoadedSkillFromText, + type ArgFieldMap, + type ExtractedArgs, +} from '../../parsers/shared/extract.js'; + +/** + * opencode's tool names → canonical names. opencode uses lowercase built-in tool + * names. Owned here, not in shared. MCP tools arrive under their server name and + * fall through to `tool_use`. + */ +const OPENCODE_TOOLS: AgentToolMap = { + caseInsensitive: true, + tools: { + read: 'file_read', + write: 'file_write', + edit: 'file_edit', + multiedit: 'file_edit', + patch: 'file_edit', + apply_patch: 'file_edit', + bash: 'shell', + shell: 'shell', + webfetch: 'web_fetch', + websearch: 'web_search', + codesearch: 'grep', + glob: 'glob', + grep: 'grep', + list: 'list_dir', + ls: 'list_dir', + task: 'agent_task', + todowrite: 'agent_task', + skill: 'tool_use', + }, +}; + +/** + * opencode tool args → normalized fields. `bash` carries the command in + * `command`; file tools the path in `filePath` (or `path`); `webfetch` the URL + * in `url`. The shared extractor reads whichever keys this map names. + */ +const OPENCODE_ARG_FIELDS: ArgFieldMap = { + path: ['filePath', 'file_path', 'path'], + command: ['command'], + url: ['url'], +}; + +/** Epoch-ms (or pass-through ISO) → ISO string. */ +function toISO(value: unknown): string | undefined { + if (typeof value === 'number') return new Date(value).toISOString(); + if (typeof value === 'string') return value; + return undefined; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +/** Whether a completed tool call succeeded: shell keys off its exit code. */ +function toolSuccess( + canonical: string, + status: string | undefined, + metadata: Record | undefined +): boolean | undefined { + if (status === undefined) return undefined; + if (status !== 'completed') return false; + if (canonical === 'shell') { + const exit = metadata?.exit; + return typeof exit === 'number' ? exit === 0 : true; + } + return true; +} + +function partToEvents( + type: string, + part: Record, + timestamp: string | undefined, + raw: unknown +): TranscriptEvent[] { + switch (type) { + case 'text': { + const text = str(part.text); + return text + ? [ + { + timestamp, + type: 'message', + role: 'assistant', + content: text, + raw, + }, + ] + : []; + } + case 'reasoning': { + const text = str(part.text); + return text ? [{ timestamp, type: 'thinking', content: text, raw }] : []; + } + case 'tool_use': { + const originalName = str(part.tool) ?? 'unknown'; + const id = str(part.callID); + const state = isRecord(part.state) ? part.state : {}; + const args = isRecord(state.input) ? state.input : {}; + const status = str(state.status); + const metadata = isRecord(state.metadata) ? state.metadata : undefined; + const name = normalizeToolName(originalName, OPENCODE_TOOLS); + const normalized: ExtractedArgs = extractArgs(args, OPENCODE_ARG_FIELDS); + + const tool: NonNullable = { + name, + originalName, + id, + args, + }; + if (normalized.path) tool.path = normalized.path; + if (normalized.command) tool.command = normalized.command; + if (normalized.url) tool.url = normalized.url; + tool.loadedSkill = loadedSkillFromOpencodeCall(tool); + + const events: TranscriptEvent[] = [ + { timestamp, type: 'tool_call', tool, raw }, + ]; + // The result is in the same record; emit it only once the call completed. + if (status && status !== 'running' && status !== 'pending') { + events.push({ + timestamp, + type: 'tool_result', + tool: { + name, + originalName, + id, + result: + state.output ?? (isRecord(state.error) ? state.error : undefined), + success: toolSuccess(name, status, metadata), + }, + raw: state, + }); + } + return events; + } + default: + return []; + } +} + +/** + * Identifies opencode skill loads. opencode's native `skill` tool carries the + * skill name in its args; skills read manually surface as `skills// + * SKILL.md` in a file path or shell command. + */ +function loadedSkillFromOpencodeCall( + tool: NonNullable +): string | undefined { + if (tool.originalName.toLowerCase() === 'skill') { + const name = tool.args?.name ?? tool.args?.skill; + if (typeof name === 'string') return name; + } + if (tool.path) return extractLoadedSkillFromText(tool.path); + if (tool.command) return extractLoadedSkillFromText(tool.command); + return undefined; +} + +function recordToEvents(data: Record): TranscriptEvent[] { + const type = str(data.type); + if (!type) return []; + const timestamp = toISO(data.timestamp); + + if (type === 'error') { + const error = isRecord(data.error) ? data.error : undefined; + const message = + str(error?.message) ?? + str(data.message) ?? + JSON.stringify(data.error ?? data); + return [{ timestamp, type: 'error', content: message, raw: data }]; + } + // step_start / step_finish carry no transcript content (tokens + finish reason + // only; the runner reads the terminal step_finish reason for the stop reason). + if (type === 'step_start' || type === 'step_finish') return []; + + const part = isRecord(data.part) ? data.part : undefined; + if (!part) return []; + return partToEvents(type, part, timestamp, data); +} + +export const opencodeParser: AgentTranscriptParser = { + parseTranscript(raw: string): ParsedTranscript { + const { records, errors } = parseJsonlRecords(raw); + const events: TranscriptEvent[] = []; + for (const record of records) { + try { + events.push(...recordToEvents(record)); + } catch (e) { + errors.push(e instanceof Error ? e.message : String(e)); + } + } + return { events, errors }; + }, +}; diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts new file mode 100644 index 00000000..c99f163b --- /dev/null +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from 'vitest'; +import { + buildOpencodeConfig, + createOpencodeRunner, + providerApiKeyEnv, +} from './runner.js'; + +/** A run's terminal records: a mid-run `step_finish` (tool-calls) then the final one. */ +const SESSION = [ + JSON.stringify({ + type: 'step_finish', + part: { type: 'step-finish', reason: 'tool-calls' }, + }), + JSON.stringify({ type: 'text', part: { type: 'text', text: 'Done.' } }), + JSON.stringify({ + type: 'step_finish', + part: { type: 'step-finish', reason: 'stop' }, + }), +].join('\n'); + +describe('opencode runner', () => { + it("resolves the API-key env var from the model's provider prefix", () => { + expect(providerApiKeyEnv('anthropic/claude-sonnet-5')).toBe( + 'ANTHROPIC_API_KEY' + ); + expect(providerApiKeyEnv('openai/gpt-5.4')).toBe('OPENAI_API_KEY'); + // opencode's google provider reads GOOGLE_GENERATIVE_AI_API_KEY, not GEMINI_API_KEY. + expect(providerApiKeyEnv('google/gemini-flash-latest')).toBe( + 'GOOGLE_GENERATIVE_AI_API_KEY' + ); + }); + + it('throws a clear error for an unsupported provider', () => { + expect(() => providerApiKeyEnv('openrouter/some-model')).toThrowError( + /Unsupported opencode provider "openrouter".*Supported: anthropic, openai, google/ + ); + }); + + it('carries the provider on the runner for experiment display metadata', () => { + expect(createOpencodeRunner('openai/gpt-5.4').modelProvider).toBe('openai'); + expect( + createOpencodeRunner('google/gemini-flash-latest').modelProvider + ).toBe('google'); + }); + + it('deriveStopReason reads the terminal step_finish reason', () => { + const runner = createOpencodeRunner('anthropic/claude-sonnet-5'); + const ok = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + expect(runner.deriveStopReason!(SESSION, ok)).toBe('stop'); + // A non-stop terminal reason is surfaced verbatim. + const length = JSON.stringify({ + type: 'step_finish', + part: { reason: 'length' }, + }); + expect(runner.deriveStopReason!(length, ok)).toBe('length'); + // An error event wins regardless of exit code. + const errored = JSON.stringify({ + type: 'error', + error: { message: 'model overloaded' }, + }); + expect(runner.deriveStopReason!(errored, ok)).toBe('error'); + }); + + it("builds opencode's MCP config shape from harness server configs", () => { + const config = JSON.parse( + buildOpencodeConfig({ + supabase: { command: 'npx', args: ['-y', 'srv'], env: { TOKEN: 't' } }, + docs: { command: 'docs-server' }, + }) + ); + expect(config.mcp).toEqual({ + supabase: { + type: 'local', + command: ['npx', '-y', 'srv'], + enabled: true, + environment: { TOKEN: 't' }, + }, + // No env → no `environment` key. + docs: { type: 'local', command: ['docs-server'], enabled: true }, + }); + }); +}); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts new file mode 100644 index 00000000..4cf1b8c5 --- /dev/null +++ b/packages/core/src/agents/opencode/runner.ts @@ -0,0 +1,211 @@ +/** + * OpenCode runner. Headless via `opencode run --format json` (the CLI + * streams newline-delimited event records to stdout; see ./parser.ts). + * + * Two things are opencode-specific: + * - It is **multi-provider**: model ids are `provider/model` (e.g. + * `anthropic/claude-sonnet-5`, `openai/gpt-5.4-mini`, `google/gemini-3.5-flash`) + * and the credential it reads depends on the provider — so the runner is + * built per-model, with `apiKeyEnvVar` and `modelProvider` resolved from the + * model id (see `createOpencodeRunner`). + * - `opencode run` blocks waiting on stdin even when the message is passed as + * an argument, so we redirect stdin from /dev/null. + * + * Like Claude Code / Codex it runs in both modes: tools mode just drops the + * Supabase CLI + local stack, and Supabase access goes through MCP (written to + * an OPENCODE_CONFIG file outside the scored workspace). + */ + +import type { Model as AnthropicModel } from '@anthropic-ai/sdk/resources/messages'; +import type { ChatModel as OpenAIModel } from 'openai/resources/shared'; +import type { GoogleGenerativeAIProvider } from '@ai-sdk/google'; +import type { McpServerConfig } from '../../index.js'; +import type { ModelProvider } from '../../eval-metadata.js'; +import { isRecord, parseJsonlRecords } from '../../json.js'; +import type { AgentRunner } from '../types.js'; +import { + SCRATCH, + npmGlobalBin, + npmInstallGlobal, + processStopReason, + shellQuote, + writeSandboxFile, +} from '../shared.js'; + +/** Gemini model ids, extracted from the exported (callable) provider type. */ +type GeminiModel = Parameters[0]; + +/** + * opencode model id: `provider/model`, where the model name is the original + * vendor's id (opencode passes it straight through to that provider's SDK). The + * three supported providers are typed from their vendor packages; any other + * string is still accepted. + */ +export type OpenCodeModel = + | `anthropic/${AnthropicModel}` + | `openai/${OpenAIModel}` + | `google/${GeminiModel}` + | (string & {}); + +/** Model used when the caller doesn't pick one. */ +export const DEFAULT_OPENCODE_MODEL: OpenCodeModel = + 'anthropic/claude-sonnet-5'; + +/** + * Provider prefix (`provider/model`) → the env var holding its key. opencode and + * the harness both use this name; Google's is `GOOGLE_GENERATIVE_AI_API_KEY` + * (opencode's google provider reads exactly that — not `GEMINI_API_KEY`). + */ +const PROVIDER_API_KEY_ENV: Record = { + anthropic: 'ANTHROPIC_API_KEY', + openai: 'OPENAI_API_KEY', + google: 'GOOGLE_GENERATIVE_AI_API_KEY', +}; + +/** The provider prefix of a `provider/model` id; throws if unsupported. */ +export function providerForModel(model: string): ModelProvider { + const provider = model.split('/')[0]; + if (!(provider in PROVIDER_API_KEY_ENV)) { + throw new Error( + `Unsupported opencode provider "${provider}" in model "${model}". ` + + `Supported: ${Object.keys(PROVIDER_API_KEY_ENV).join(', ')}.` + ); + } + return provider as ModelProvider; +} + +/** The API-key env var for a given `provider/model` id; throws if unsupported. */ +export function providerApiKeyEnv(model: string): string { + return PROVIDER_API_KEY_ENV[providerForModel(model)]; +} + +/** + * Shell path to the MCP config, staged in scratch (outside the workspace). Used + * both as the write target and as the `OPENCODE_CONFIG` env value — the shell + * expands `$HOME` in either position. + */ +const OPENCODE_CONFIG_PATH = '"$HOME/.eval/opencode.json"'; + +/** + * Build an opencode runner bound to one model's provider. opencode is + * multi-provider, but a single run targets one model, so the runner resolves + * `apiKeyEnvVar` and `modelProvider` from the model id (the generic layer's + * `requireApiKey` reads `apiKeyEnvVar`, and `exec` injects that same key). + */ +export function createOpencodeRunner( + model: OpenCodeModel +): AgentRunner { + const modelProvider = providerForModel(model); + return { + id: 'opencode', + displayName: 'OpenCode', + apiKeyEnvVar: providerApiKeyEnv(model), + modelProvider, + cliPackage: 'opencode-ai', + // Pinned: opencode's --format json event schema evolves; bump deliberately + // and re-check the parser. See ./parser.ts. + defaultCliVersion: '1.15.7', + defaultModel: DEFAULT_OPENCODE_MODEL, + + async install(sandbox, version) { + await npmInstallGlobal( + sandbox, + `${this.cliPackage}@${version}`, + this.displayName + ); + }, + + async exec({ + sandbox, + model, + apiKey, + systemPromptPath, + userPromptPath, + mcpServers, + timeoutSec, + }) { + const opencode = npmGlobalBin('opencode'); + + // opencode has no system-prompt flag, so prepend the system prompt to the + // task; both are staged files, joined via command substitution into the + // single message argument. + const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; + + let configPrefix = ''; + if (Object.keys(mcpServers).length > 0) { + await sandbox.exec(`mkdir -p ${SCRATCH}`); + await writeSandboxFile( + sandbox, + OPENCODE_CONFIG_PATH, + buildOpencodeConfig(mcpServers) + ); + configPrefix = `OPENCODE_CONFIG=${OPENCODE_CONFIG_PATH} `; + } + + const flags = [ + 'run', + message, + `--model ${shellQuote(model)}`, + // Newline-delimited JSON event records on stdout. + '--format json', + // The sandbox is the isolation boundary, so let opencode act freely. + '--dangerously-skip-permissions', + ].join(' '); + + // `< /dev/null`: opencode run blocks on stdin otherwise, even with the + // message passed as an argument. + const command = await sandbox.exec( + `${configPrefix}${opencode} ${flags} < /dev/null`, + { + timeoutMs: timeoutSec * 1000, + env: { [this.apiKeyEnvVar]: apiKey }, + } + ); + return { command, raw: command.stdout }; + }, + + deriveStopReason(raw, command) { + if (!raw) return processStopReason(command); + const { records } = parseJsonlRecords(raw); + // An error event means the run failed regardless of exit code. + if (records.some((r) => r.type === 'error')) return 'error'; + // The terminal `step_finish` carries the model's finish reason. + for (let i = records.length - 1; i >= 0; i -= 1) { + if (records[i].type !== 'step_finish') continue; + const part = records[i].part; + const reason = + isRecord(part) && typeof part.reason === 'string' + ? part.reason + : undefined; + if (reason === 'stop') return 'stop'; + if (reason && reason !== 'tool-calls') return reason; // e.g. length — surface verbatim + break; + } + return processStopReason(command); + }, + }; +} + +/** + * opencode's `OPENCODE_CONFIG` MCP schema: `{ mcp: { name: { type: "local", + * command: [...], environment } } }`. The harness's `{command,args,env}` maps + * onto a single `command` array plus `environment`. + */ +export function buildOpencodeConfig( + servers: Record +): string { + const mcp: Record = {}; + for (const [name, server] of Object.entries(servers)) { + mcp[name] = { + type: 'local', + command: [server.command, ...(server.args ?? [])], + enabled: true, + ...(server.env ? { environment: server.env } : {}), + }; + } + return JSON.stringify( + { $schema: 'https://opencode.ai/config.json', mcp }, + null, + 2 + ); +} diff --git a/packages/core/src/agents/registry.ts b/packages/core/src/agents/registry.ts index 6a91a0d5..58db31b8 100644 --- a/packages/core/src/agents/registry.ts +++ b/packages/core/src/agents/registry.ts @@ -13,8 +13,13 @@ import type { AgentHarnessId } from '../eval-metadata.js'; import type { AgentTranscriptParser } from '../parsers/types.js'; import { claudeCodeDefinition } from './claude-code/index.js'; import { codexDefinition } from './codex/index.js'; +import { opencodeDefinition } from './opencode/index.js'; -const AGENTS: AgentDefinition[] = [claudeCodeDefinition, codexDefinition]; +const AGENTS: AgentDefinition[] = [ + claudeCodeDefinition, + codexDefinition, + opencodeDefinition, +]; const byId = new Map(AGENTS.map((agent) => [agent.runner.id, agent])); diff --git a/packages/core/src/agents/shared.test.ts b/packages/core/src/agents/shared.test.ts new file mode 100644 index 00000000..06ab423b --- /dev/null +++ b/packages/core/src/agents/shared.test.ts @@ -0,0 +1,28 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { requireEnv } from './shared.js'; + +const VAR = 'OPENCODE_TEST_ENV_VAR'; + +describe('requireEnv', () => { + afterEach(() => { + delete process.env[VAR]; + }); + + it('returns the value when set', () => { + process.env[VAR] = 'secret'; + expect(requireEnv(VAR)).toBe('secret'); + }); + + it('throws a clear, variable-naming error when unset, including the hint', () => { + expect(() => requireEnv(VAR, 'Set it to run X.')).toThrowError( + `Environment variable ${VAR} is not set. Set it to run X.` + ); + }); + + it('distinguishes set-but-empty from unset', () => { + process.env[VAR] = ' '; + expect(() => requireEnv(VAR)).toThrowError( + `Environment variable ${VAR} is set but empty.` + ); + }); +}); diff --git a/packages/core/src/agents/shared.ts b/packages/core/src/agents/shared.ts index cf1058cb..63d918d9 100644 --- a/packages/core/src/agents/shared.ts +++ b/packages/core/src/agents/shared.ts @@ -1,12 +1,35 @@ /** - * Helpers shared across CLI runners: sandbox scratch paths, file staging, - * global npm install, loopback rewriting, and the default process-exit-based - * stop reason. + * Helpers shared across CLI runners: env-var validation, sandbox scratch paths, + * file staging, global npm install, loopback rewriting, and the default + * process-exit-based stop reason. */ import type { CommandResult, McpServerConfig } from '../index.js'; import type { AgentSandbox } from './types.js'; +/** + * Read a required environment variable, throwing a clear error that names the + * variable (and distinguishes unset from blank). Node-native — reads + * `process.env` directly, no dependency. Shared so every harness validates its + * key the same way and surfaces the same precise message. + */ +export function requireEnv(name: string, hint?: string): string { + // `in` distinguishes "never set" from "set but empty" for a clearer message. + const isSet = name in process.env; + const value = process.env[name]; + if (!isSet || value === undefined) { + throw new Error( + `Environment variable ${name} is not set.${hint ? ` ${hint}` : ''}` + ); + } + if (value.trim() === '') { + throw new Error( + `Environment variable ${name} is set but empty.${hint ? ` ${hint}` : ''}` + ); + } + return value; +} + /** Scratch dir + staged files, outside the workspace so they're never scored. */ export const SCRATCH = '"$HOME/.eval"'; export const SYSTEM_PROMPT_PATH = '"$HOME/.eval/system-prompt.txt"'; diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts index aad716ed..b3f3dea0 100644 --- a/packages/core/src/agents/types.ts +++ b/packages/core/src/agents/types.ts @@ -82,6 +82,12 @@ export interface AgentRunner { displayName: string; /** Env var holding the agent's API key (e.g. `ANTHROPIC_API_KEY`). */ apiKeyEnvVar: string; + /** + * Optional: the model's provider, for multi-provider CLIs whose runner is + * built per-model (e.g. opencode's `provider/model` ids). Single-provider + * agents omit it — the engine derives the provider from the agent id. + */ + modelProvider?: ModelProvider; /** npm package providing the CLI. */ cliPackage: string; /** Pinned CLI version — pinned so transcript-format drift can't silently break parsing. */ diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 1a8d541c..7e9fe927 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -52,10 +52,15 @@ export const experimentSuiteSchema = z.enum([ export const EXPERIMENT_SUITES = experimentSuiteSchema.options; export type ExperimentSuite = z.infer; -export const agentHarnessIdSchema = z.enum(['ai-sdk', 'claude-code', 'codex']); +export const agentHarnessIdSchema = z.enum([ + 'ai-sdk', + 'claude-code', + 'codex', + 'opencode', +]); export type AgentHarnessId = z.infer; -export const modelProviderSchema = z.enum(['anthropic', 'openai']); +export const modelProviderSchema = z.enum(['anthropic', 'openai', 'google']); export type ModelProvider = z.infer; export const reasoningEffortSchema = z.enum([ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c3ccfc0e..caa480d2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -99,10 +99,11 @@ export { rehydrateTruncatedDocsResults, } from './docs-results.js'; export type { DocsResultSandbox } from './docs-results.js'; -// CLI agent harnesses (Claude Code, Codex, and the framework for adding more). +// CLI agent harnesses (Claude Code, Codex, OpenCode, and the framework for adding more). export { createCliAgent } from './agents/engine.js'; export { claudeCodeAgent } from './agents/claude-code/index.js'; export { codexAgent } from './agents/codex/index.js'; +export { opencodeAgent } from './agents/opencode/index.js'; export type { AgentMetadata, AgentSandbox, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f7b2f128..2fbfb54f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,6 +9,9 @@ catalogs: '@ai-sdk/anthropic': specifier: ^3.0.71 version: 3.0.82 + '@ai-sdk/google': + specifier: ^3.0.83 + version: 3.0.100 '@ai-sdk/mcp': specifier: ^1.0.39 version: 1.0.46 @@ -245,6 +248,9 @@ importers: packages/core: dependencies: + '@ai-sdk/google': + specifier: 'catalog:' + version: 3.0.100(zod@4.4.3) '@ai-sdk/mcp': specifier: 'catalog:' version: 1.0.46(zod@4.4.3) @@ -368,6 +374,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/google@3.0.100': + resolution: {integrity: sha512-gmsjuwk+1++/qCsIopfkg9d68nb6TfZiLNlkEOEDd86waawsR4B+FQ4j73p/r3NbVz1NWcnU4IqLz5uHa9Fu9g==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/mcp@1.0.46': resolution: {integrity: sha512-owU0wAP87KzsTzr+2JE9sT9lpsCWKg8ZwHhce/KQmD9D/kbhe69sUZ+lsFF5MoGvV/ZzMujrhAvC1MgmufwMeQ==} engines: {node: '>=18'} @@ -386,10 +398,20 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@4.0.40': + resolution: {integrity: sha512-OL5IrpUm9Y8Dwy+w/vvFwPotS6m52O9W0op2oXgXdCROMJIBalBI0oro6OIBYkPxvm5Xg02GSkoQN25RlR0bnw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider@3.0.10': resolution: {integrity: sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==} engines: {node: '>=18'} + '@ai-sdk/provider@3.0.14': + resolution: {integrity: sha512-5X1k57JBJ4H7H1QjX7CnJYAB1I19r/trVZTMcSms7/kLNZ8RaU4Nt2agcwZzv82Hfx6Q7/TOLU7agAKeFfc8cA==} + engines: {node: '>=18'} + '@anthropic-ai/sdk@0.105.0': resolution: {integrity: sha512-sDyu+aM9cE6uZE+HgRjjHRb+qqb87GHZOx+8bE0YlWetdL1YcVLxn8h9ltxGOflyChTe6PMEo50kMQV4cw0hfg==} hasBin: true @@ -4501,6 +4523,12 @@ snapshots: '@vercel/oidc': 3.2.0 zod: 4.4.3 + '@ai-sdk/google@3.0.100(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.14 + '@ai-sdk/provider-utils': 4.0.40(zod@4.4.3) + zod: 4.4.3 + '@ai-sdk/mcp@1.0.46(zod@4.4.3)': dependencies: '@ai-sdk/provider': 3.0.10 @@ -4521,10 +4549,21 @@ snapshots: eventsource-parser: 3.1.0 zod: 4.4.3 + '@ai-sdk/provider-utils@4.0.40(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.14 + '@standard-schema/spec': 1.1.0 + eventsource-parser: 3.1.0 + zod: 4.4.3 + '@ai-sdk/provider@3.0.10': dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@3.0.14': + dependencies: + json-schema: 0.4.0 + '@anthropic-ai/sdk@0.105.0(zod@4.4.3)': dependencies: json-schema-to-ts: 3.1.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7b181ac5..bbfd992f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,7 @@ catalog: '@anthropic-ai/sdk': ^0.105.0 'openai': ^6.44.0 '@ai-sdk/anthropic': ^3.0.71 + '@ai-sdk/google': ^3.0.83 '@ai-sdk/mcp': ^1.0.39 '@ai-sdk/openai': ^3.0.66 '@electric-sql/pglite': 0.4.5 From ace39b5910bd0e0e1adcda776986673919f6767a Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 19:23:28 +0100 Subject: [PATCH 2/2] remove opencode gemini and gpt 5.4 --- experiments/opencode-gemini-flash.ts | 24 ------------------------ experiments/opencode-gpt-5.4-mini.ts | 21 --------------------- 2 files changed, 45 deletions(-) delete mode 100644 experiments/opencode-gemini-flash.ts delete mode 100644 experiments/opencode-gpt-5.4-mini.ts diff --git a/experiments/opencode-gemini-flash.ts b/experiments/opencode-gemini-flash.ts deleted file mode 100644 index f2f68c8d..00000000 --- a/experiments/opencode-gemini-flash.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { - defineExperiment, - opencodeAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -// OpenCode driving Google's latest Gemini Flash (cheapest tier). Runs in both -// modes (see opencode-claude-sonnet-5.ts); the `google/` prefix selects the -// GOOGLE_GENERATIVE_AI_API_KEY credential (Google AI Studio, not Vertex). -// `gemini-flash-latest` tracks the newest Flash — the only Gemini Flash id that -// the AI-Studio key serves end-to-end (pinned 2.5/3.x-flash ids returned no -// output via opencode 1.15.7). -export default defineExperiment({ - agent: opencodeAgent({ - model: 'google/gemini-flash-latest', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/opencode-gpt-5.4-mini.ts b/experiments/opencode-gpt-5.4-mini.ts deleted file mode 100644 index 4937744c..00000000 --- a/experiments/opencode-gpt-5.4-mini.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { - defineExperiment, - opencodeAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -// OpenCode driving OpenAI GPT-5.4 mini. Runs in both modes (see opencode-claude- -// sonnet-5.ts); the `openai/` model prefix selects the OPENAI_API_KEY -// credential. -export default defineExperiment({ - agent: opencodeAgent({ - model: 'openai/gpt-5.4-mini', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -});