feat(eval): add OpenCode agent harness

.env.example

@@ -1,3 +1,6 @@
 # AI SDK Core direct-provider credentials.
 ANTHROPIC_API_KEY=
 OPENAI_API_KEY=
+
+# Opencode Gemini API key
+GOOGLE_GENERATIVE_AI_API_KEY=

apps/web/src/App.tsx

@@ -198,6 +198,7 @@ const AGENT_LABELS = {
   "ai-sdk": "AI SDK",
   "claude-code": "Claude Code",
   codex: "Codex",
+  opencode: "OpenCode",
 } satisfies Record<ExperimentDisplay["agent"], string>
 
 const EXPERIMENT_SUITES = ["benchmark", "no-skills"] as const
@@ -313,11 +314,15 @@ function formatOpenAiModel(modelId: string) {
 }
 
 function formatModel(display: ExperimentDisplay) {
+  // opencode model ids are `provider/model`; strip the prefix for display.
+  const modelId = display.modelId.replace(`${display.modelProvider}/`, "")
   switch (display.modelProvider) {
     case "anthropic":
-      return formatAnthropicModel(display.modelId)
+      return formatAnthropicModel(modelId)
     case "openai":
-      return formatOpenAiModel(display.modelId)
+      return formatOpenAiModel(modelId)
+    case "google":
+      return modelId
   }
 }
 

experiments/opencode-claude-sonnet-5.ts

@@ -0,0 +1,22 @@
+import {
+  defineExperiment,
+  opencodeAgent,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+
+// OpenCode is a CLI agent driving Claude Sonnet 5. Like Claude Code / Codex it
+// runs in both modes: `runtime` supplies the MCP servers for tools-mode evals
+// (written into opencode's config) and `localStack` drives local-stack evals.
+// Which mode an eval uses is a property of the eval, not the agent.
+export default defineExperiment({
+  agent: opencodeAgent({
+    model: 'anthropic/claude-sonnet-5',
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ['supabase', 'supabase-postgres-best-practices'],
+});

packages/core/package.json

@@ -18,14 +18,15 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "catalog:",
-    "openai": "catalog:",
+    "@ai-sdk/google": "catalog:",
     "@ai-sdk/mcp": "catalog:",
     "@ai-sdk/openai": "catalog:",
     "@supabase-evals/platform-lite": "workspace:*",
     "@supabase/supabase-js": "catalog:",
     "ai": "catalog:",
     "executor": "1.4.29",
     "gray-matter": "^4.0.3",
+    "openai": "catalog:",
     "typescript": "catalog:",
     "zod": "catalog:"
   }

packages/core/src/agents/engine.ts

@@ -29,6 +29,7 @@ import {
   SYSTEM_PROMPT_PATH,
   USER_PROMPT_PATH,
   processStopReason,
+  requireEnv,
   rewriteLoopback,
   writeSandboxFile,
 } from './shared.js';
@@ -39,6 +40,10 @@ function modelProviderForAgent(id: AgentRunner['id']): ModelProvider {
       return 'anthropic';
     case 'codex':
       return 'openai';
+    case 'opencode':
+      throw new Error(
+        'opencode is multi-provider; its runner sets `modelProvider` from the model id'
+      );
     case 'ai-sdk':
       throw new Error('ai-sdk agents are not created through createCliAgent');
   }
@@ -60,7 +65,7 @@ export function createCliAgent<M extends string = string>(
     modelId: options.model,
     metadata: {
       agent: runner.id,
-      modelProvider: modelProviderForAgent(runner.id),
+      modelProvider: runner.modelProvider ?? modelProviderForAgent(runner.id),
       modelId: options.model,
       ...(options.reasoningEffort
         ? { reasoningEffort: options.reasoningEffort }
@@ -117,11 +122,8 @@ export function createCliAgent<M extends string = string>(
 }
 
 function requireApiKey(runner: AgentRunner): string {
-  const apiKey = process.env[runner.apiKeyEnvVar];
-  if (!apiKey) {
-    throw new Error(
-      `Missing ${runner.displayName} credentials. Set ${runner.apiKeyEnvVar} before running ${runner.id} evals.`
-    );
-  }
-  return apiKey;
+  return requireEnv(
+    runner.apiKeyEnvVar,
+    `Set it to run ${runner.displayName} (${runner.id}) evals.`
+  );
 }

packages/core/src/agents/opencode/index.ts

@@ -0,0 +1,42 @@
+/**
+ * OpenCode agent. Owns everything opencode-specific: it wires its own runner +
+ * parser into the public `opencodeAgent` factory (via the generic
+ * `createCliAgent` engine) and exports the registry definition the harness uses
+ * to parse opencode transcripts. Runs in both modes, like Claude Code / Codex.
+ */
+
+import type { AgentHarness } from '../../index.js';
+import { createCliAgent } from '../engine.js';
+import type { AgentDefinition } from '../types.js';
+import {
+  DEFAULT_OPENCODE_MODEL,
+  createOpencodeRunner,
+  type OpenCodeModel,
+} from './runner.js';
+import { opencodeParser } from './parser.js';
+
+/**
+ * OpenCode as an `AgentHarness`. Multi-provider: the `provider/model` id selects
+ * the credential (anthropic / openai / google), so the runner is built per-model
+ * with the matching API-key env var and provider.
+ */
+export function opencodeAgent(
+  options: {
+    /** opencode model id, `provider/model` (e.g. `openai/gpt-5.4`). */
+    model?: OpenCodeModel;
+    /** Override the pinned CLI version. */
+    cliVersion?: string;
+  } = {}
+): AgentHarness {
+  const model = options.model ?? DEFAULT_OPENCODE_MODEL;
+  return createCliAgent(createOpencodeRunner(model), opencodeParser, {
+    model,
+    cliVersion: options.cliVersion,
+  });
+}
+
+/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */
+export const opencodeDefinition: AgentDefinition = {
+  runner: createOpencodeRunner(DEFAULT_OPENCODE_MODEL),
+  parser: opencodeParser,
+};

packages/core/src/agents/opencode/parser.test.ts

@@ -0,0 +1,193 @@
+import { describe, expect, it } from 'vitest';
+import { opencodeParser } from './parser.js';
+import { adaptTranscript } from '../../parsers/adapt.js';
+
+/** A representative `opencode run --format json` stream (shapes from CLI 1.15.7). */
+const SESSION = [
+  JSON.stringify({ type: 'step_start', part: { type: 'step-start' } }),
+  JSON.stringify({
+    type: 'reasoning',
+    timestamp: 1782295624200,
+    part: { type: 'reasoning', text: 'I should list the files.' },
+  }),
+  JSON.stringify({
+    type: 'text',
+    timestamp: 1782295624232,
+    part: { type: 'text', text: 'Listing files.' },
+  }),
+  JSON.stringify({
+    type: 'tool_use',
+    timestamp: 1782295624290,
+    part: {
+      type: 'tool',
+      tool: 'bash',
+      callID: 'tool_1',
+      state: {
+        status: 'completed',
+        input: { command: 'ls -la', description: 'List files' },
+        output: 'file1\nfile2',
+        metadata: { exit: 0 },
+      },
+    },
+  }),
+  JSON.stringify({
+    type: 'tool_use',
+    timestamp: 1782295624300,
+    part: {
+      type: 'tool',
+      tool: 'write',
+      callID: 'tool_2',
+      state: {
+        status: 'completed',
+        input: { filePath: '/work/note.txt', content: 'hi' },
+        output: 'written',
+      },
+    },
+  }),
+  JSON.stringify({
+    type: 'text',
+    timestamp: 1782295624400,
+    part: { type: 'text', text: 'Done.' },
+  }),
+  JSON.stringify({
+    type: 'step_finish',
+    part: {
+      type: 'step-finish',
+      reason: 'stop',
+      tokens: { input: 3, output: 6 },
+    },
+  }),
+].join('\n');
+
+describe('opencodeParser', () => {
+  it('maps bash + write to canonical tool calls, paired with results by callID', () => {
+    const { events, errors } = opencodeParser.parseTranscript(SESSION);
+    expect(errors).toEqual([]);
+
+    const calls = events.filter((e) => e.type === 'tool_call');
+    expect(calls.map((e) => e.tool?.name)).toEqual(['shell', 'file_write']);
+    expect(calls.map((e) => e.tool?.originalName)).toEqual(['bash', 'write']);
+    expect(calls.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']);
+    // Normalized views on the event; raw args untouched.
+    expect(calls[0].tool?.command).toBe('ls -la');
+    expect(calls[1].tool?.path).toBe('/work/note.txt');
+
+    const results = events.filter((e) => e.type === 'tool_result');
+    expect(results.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']);
+    expect(results.every((e) => e.tool?.success === true)).toBe(true);
+  });
+
+  it('surfaces reasoning + the assistant report via the adapter', () => {
+    const events = opencodeParser.parseTranscript(SESSION).events;
+    expect(
+      events.some(
+        (e) => e.type === 'thinking' && e.content === 'I should list the files.'
+      )
+    ).toBe(true);
+
+    const adapted = adaptTranscript(events);
+    expect(adapted.agentReport).toBe('Done.');
+    expect(adapted.steps).toBe(2); // two assistant text turns
+    expect(adapted.toolCalls).toEqual([
+      {
+        endpoint: 'bash',
+        body: { command: 'ls -la', description: 'List files' },
+        name: 'shell',
+        command: 'ls -la',
+        result: 'file1\nfile2',
+        error: undefined,
+        ts: 1782295624290, // epoch ms preserved through toISO -> parseTs
+      },
+      {
+        endpoint: 'write',
+        body: { filePath: '/work/note.txt', content: 'hi' },
+        name: 'file_write',
+        path: '/work/note.txt',
+        result: 'written',
+        error: undefined,
+        ts: 1782295624300,
+      },
+    ]);
+  });
+
+  it('surfaces skill loads from the skill tool and from SKILL.md reads', () => {
+    const stream = [
+      JSON.stringify({
+        type: 'tool_use',
+        part: {
+          type: 'tool',
+          tool: 'skill',
+          callID: 's1',
+          state: {
+            status: 'completed',
+            input: { name: 'supabase' },
+            output: '# Supabase',
+          },
+        },
+      }),
+      JSON.stringify({
+        type: 'tool_use',
+        part: {
+          type: 'tool',
+          tool: 'read',
+          callID: 's2',
+          state: {
+            status: 'completed',
+            input: {
+              filePath:
+                '.claude/skills/supabase-postgres-best-practices/SKILL.md',
+            },
+            output: '# Postgres',
+          },
+        },
+      }),
+    ].join('\n');
+    const adapted = adaptTranscript(
+      opencodeParser.parseTranscript(stream).events
+    );
+    expect(adapted.toolCalls.map((call) => call.loadedSkill)).toEqual([
+      'supabase',
+      'supabase-postgres-best-practices',
+    ]);
+  });
+
+  it('marks a non-zero shell exit as failed (error surfaced via adapter)', () => {
+    const stream = JSON.stringify({
+      type: 'tool_use',
+      part: {
+        type: 'tool',
+        tool: 'bash',
+        callID: 'c1',
+        state: {
+          status: 'completed',
+          input: { command: 'false' },
+          output: 'nope',
+          metadata: { exit: 1 },
+        },
+      },
+    });
+    const events = opencodeParser.parseTranscript(stream).events;
+    expect(events.find((e) => e.type === 'tool_result')?.tool?.success).toBe(
+      false
+    );
+    const adapted = adaptTranscript(events);
+    expect(adapted.toolCalls[0].error).toBe('nope');
+    expect(adapted.toolCalls[0].result).toBeUndefined();
+  });
+
+  it('emits an error event and never throws on malformed lines', () => {
+    const { events, errors } = opencodeParser.parseTranscript(
+      'not json\n' +
+        JSON.stringify({ type: 'error', error: { message: 'boom' } })
+    );
+    expect(events).toEqual([
+      {
+        timestamp: undefined,
+        type: 'error',
+        content: 'boom',
+        raw: { type: 'error', error: { message: 'boom' } },
+      },
+    ]);
+    expect(errors.length).toBe(1);
+  });
+});