feat(eval): add Codex agent & add reasoningEffort as experiment param

.github/workflows/eval-refresh.yml

@@ -6,7 +6,7 @@ on:
       experiments:
         description: "Comma-separated experiment names to run"
         required: true
-        default: "openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-haiku-4.5,claude-code-sonnet-4.6"
+        default: "openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-opus-4.8,claude-code-sonnet-4.6,codex-gpt-5.4-mini,codex-gpt-5.5"
       eval:
         description: "Optional single eval id to run"
         required: false
@@ -72,7 +72,7 @@ jobs:
             runs="${{ inputs.runs }}"
             timeout_sec="${{ inputs.timeout_sec }}"
           else
-            experiments="openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-haiku-4.5,claude-code-sonnet-4.6"
+            experiments="openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-opus-4.8,claude-code-sonnet-4.6,codex-gpt-5.4-mini,codex-gpt-5.5"
             eval_id=""
             suite="benchmark"
             runs="1"

experiments/claude-code-haiku-4.5.ts → experiments/claude-code-opus-4.8.ts

@@ -6,12 +6,10 @@ import {
 } from "@supabase-evals/core";
 import { localStackRuntime } from "@supabase-evals/sandbox";
 
-// Claude Code on Haiku. See experiments/claude-code-sonnet-4.6.ts for the
-// CLI-agent notes (runs in both modes: full sandbox for local-stack evals, bare
-// sandbox + MCP for tools-mode evals).
 export default defineExperiment({
   agent: claudeCodeAgent({
-    model: "claude-haiku-4-5",
+    model: "claude-opus-4-8",
+    reasoningEffort: "high",
   }),
   runtime: platformLiteRuntime({
     mcpServers: [supabaseMcpServer()],

experiments/claude-code-sonnet-4.6.ts

@@ -6,15 +6,10 @@ import {
 } from "@supabase-evals/core";
 import { localStackRuntime } from "@supabase-evals/sandbox";
 
-// Claude Code is a CLI agent: it runs its own harness (Read/Write/Bash/Edit +
-// MCP) inside a sandbox, in BOTH eval modes. Local-stack evals (interface: cli
-// or a local/ workspace) get the full sandbox — the Supabase CLI plus a running
-// local stack. Tools-mode evals get a bare sandbox (same image, no stack) where
-// the eval's tools come from the `runtime` MCP servers (reached host-side via
-// host.docker.internal). The running stack + the CLI is the only mode difference.
 export default defineExperiment({
   agent: claudeCodeAgent({
     model: "claude-sonnet-4-6",
+    reasoningEffort: "high",
   }),
   runtime: platformLiteRuntime({
     mcpServers: [supabaseMcpServer()],

experiments/codex-gpt-5.4-mini.ts

@@ -0,0 +1,19 @@
+import {
+  codexAgent,
+  defineExperiment,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from "@supabase-evals/core";
+import { localStackRuntime } from "@supabase-evals/sandbox";
+
+export default defineExperiment({
+  agent: codexAgent({
+    model: "gpt-5.4-mini",
+    reasoningEffort: "medium",
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ["supabase", "supabase-postgres-best-practices"],
+});

experiments/codex-gpt-5.5.ts

@@ -0,0 +1,19 @@
+import {
+  codexAgent,
+  defineExperiment,
+  platformLiteRuntime,
+  supabaseMcpServer,
+} from "@supabase-evals/core";
+import { localStackRuntime } from "@supabase-evals/sandbox";
+
+export default defineExperiment({
+  agent: codexAgent({
+    model: "gpt-5.5",
+    reasoningEffort: "medium",
+  }),
+  runtime: platformLiteRuntime({
+    mcpServers: [supabaseMcpServer()],
+  }),
+  localStack: localStackRuntime(),
+  skills: ["supabase", "supabase-postgres-best-practices"],
+});

packages/core/package.json

@@ -18,6 +18,7 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "catalog:",
+    "openai": "catalog:",
     "@ai-sdk/mcp": "catalog:",
     "@ai-sdk/openai": "catalog:",
     "@supabase-evals/platform-lite": "workspace:*",

packages/core/src/agents/claude-code/index.ts

@@ -0,0 +1,37 @@
+/**
+ * Claude Code agent. Owns everything Claude-Code-specific: it wires its own
+ * runner + parser into the public `claudeCodeAgent` factory (via the generic
+ * `createCliAgent` engine) and exports the registry definition the harness uses
+ * to parse Claude Code transcripts.
+ */
+
+import type { Model as AnthropicModel } from "@anthropic-ai/sdk/resources/messages";
+import type { AgentHarness } from "../../index.js";
+import { createCliAgent } from "../engine.js";
+import type { AgentDefinition, ClaudeCodeEffort } from "../types.js";
+import { claudeCodeRunner } from "./runner.js";
+import { claudeCodeParser } from "./parser.js";
+
+/** Claude Code as an `AgentHarness`. */
+export function claudeCodeAgent(
+  options: {
+    /** Anthropic model id (typed from `@anthropic-ai/sdk`). Defaults to Sonnet. */
+    model?: AnthropicModel;
+    /** Reasoning effort (`--effort`). Omit to use Claude Code's own default. */
+    reasoningEffort?: ClaudeCodeEffort;
+    /** Override the pinned CLI version. */
+    cliVersion?: string;
+  } = {},
+): AgentHarness {
+  return createCliAgent(claudeCodeRunner, claudeCodeParser, {
+    model: options.model ?? claudeCodeRunner.defaultModel,
+    reasoningEffort: options.reasoningEffort,
+    cliVersion: options.cliVersion,
+  });
+}
+
+/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */
+export const claudeCodeDefinition: AgentDefinition = {
+  runner: claudeCodeRunner,
+  parser: claudeCodeParser,
+};

packages/core/src/parsers/parsers.test.ts → packages/core/src/agents/claude-code/parser.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
-import { claudeCodeParser } from "./claude-code.js";
-import { adaptTranscript } from "./adapt.js";
+import { claudeCodeParser } from "./parser.js";
+import { adaptTranscript } from "../../parsers/adapt.js";
 
 /** A representative Claude Code `--print` JSONL session. */
 const SESSION = [

packages/core/src/parsers/claude-code.ts → packages/core/src/agents/claude-code/parser.ts

@@ -14,11 +14,11 @@
 import type {
   ParsedTranscript,
   TranscriptEvent,
-} from "../transcript/types.js";
-import type { AgentTranscriptParser } from "./types.js";
-import { isRecord, parseJsonlRecords } from "../json.js";
-import { normalizeToolName, type AgentToolMap } from "./shared/normalize.js";
-import { extractArgs, type ArgFieldMap } from "./shared/extract.js";
+} from "../../transcript/types.js";
+import type { AgentTranscriptParser } from "../../parsers/types.js";
+import { isRecord, parseJsonlRecords } from "../../json.js";
+import { normalizeToolName, type AgentToolMap } from "../../parsers/shared/normalize.js";
+import { extractArgs, type ArgFieldMap } from "../../parsers/shared/extract.js";
 
 /** Claude Code's tool names → canonical names (case-sensitive). Owned here, not in shared. */
 const CLAUDE_CODE_TOOLS: AgentToolMap = {

packages/core/src/cli-agent.test.ts → packages/core/src/agents/claude-code/runner.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
-import { claudeCodeRunner } from "./runners/claude-code.js";
-import type { CommandResult } from "./index.js";
+import { claudeCodeRunner } from "./runner.js";
+import type { CommandResult } from "../../index.js";
 
 const ok: CommandResult = { ok: true, exitCode: 0, stdout: "", stderr: "" };
 const timedOut: CommandResult = {

packages/core/src/runners/claude-code.ts → packages/core/src/agents/claude-code/runner.ts

@@ -1,20 +1,20 @@
 /**
  * Claude Code runner. Headless via `claude -p --output-format stream-json`
  * (Anthropic's recommended programmatic path — events stream to stdout, no
- * on-disk session-file race). See parsers/claude-code.ts for the transcript.
+ * on-disk session-file race). See ./parser.ts for the transcript.
  */
 
 import type { Model as AnthropicModel } from "@anthropic-ai/sdk/resources/messages";
-import type { McpServerConfig } from "../index.js";
-import { parseJsonlRecords } from "../json.js";
-import type { AgentRunner } from "./types.js";
+import type { McpServerConfig } from "../../index.js";
+import { parseJsonlRecords } from "../../json.js";
+import type { AgentRunner } from "../types.js";
 import {
   npmGlobalBin,
   npmInstallGlobal,
   processStopReason,
   shellQuote,
   writeSandboxFile,
-} from "./shared.js";
+} from "../shared.js";
 
 const MCP_CONFIG_PATH = '"$HOME/.eval/mcp.json"';
 
@@ -24,15 +24,15 @@ export const claudeCodeRunner: AgentRunner<AnthropicModel> = {
   apiKeyEnvVar: "ANTHROPIC_API_KEY",
   cliPackage: "@anthropic-ai/claude-code",
   // Pinned: Claude Code's transcript format evolves; bump deliberately and
-  // re-check the parser. See packages/core/src/parsers/claude-code.ts.
+  // re-check the parser. See ./parser.ts.
   defaultCliVersion: "2.1.101",
   defaultModel: "claude-sonnet-4-6",
 
   async install(sandbox, version) {
     await npmInstallGlobal(sandbox, `${this.cliPackage}@${version}`, this.displayName);
   },
 
-  async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, timeoutSec }) {
+  async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, reasoningEffort, timeoutSec }) {
     const claude = npmGlobalBin("claude");
     const serverNames = Object.keys(mcpServers);
 
@@ -49,6 +49,8 @@ export const claudeCodeRunner: AgentRunner<AnthropicModel> = {
       "--output-format stream-json",
       "--verbose",
       `--model ${shellQuote(model)}`,
+      // Reasoning effort for the session; omitted leaves Claude Code's default.
+      ...(reasoningEffort ? [`--effort ${shellQuote(reasoningEffort)}`] : []),
       // Append (not replace), from a file (no ARG_MAX/shell-expansion surface),
       // so Claude Code keeps its default coding-agent prompt + tool guidance.
       `--append-system-prompt-file ${systemPromptPath}`,

packages/core/src/agents/codex/index.ts

@@ -0,0 +1,36 @@
+/**
+ * OpenAI Codex agent. Owns everything Codex-specific: it wires its own runner +
+ * parser into the public `codexAgent` factory (via the generic `createCliAgent`
+ * engine) and exports the registry definition the harness uses to parse Codex
+ * transcripts. Runs in both modes, like Claude Code.
+ */
+
+import type { AgentHarness } from "../../index.js";
+import { createCliAgent } from "../engine.js";
+import type { AgentDefinition, CodexReasoningEffort } from "../types.js";
+import { codexRunner, type CodexModel } from "./runner.js";
+import { codexParser } from "./parser.js";
+
+/** OpenAI Codex as an `AgentHarness`. */
+export function codexAgent(
+  options: {
+    /** OpenAI model id (typed from `openai`; any string accepted). */
+    model?: CodexModel;
+    /** Reasoning effort (`model_reasoning_effort`). Omit to use Codex's default. */
+    reasoningEffort?: CodexReasoningEffort;
+    /** Override the pinned CLI version. */
+    cliVersion?: string;
+  } = {},
+): AgentHarness {
+  return createCliAgent(codexRunner, codexParser, {
+    model: options.model ?? codexRunner.defaultModel,
+    reasoningEffort: options.reasoningEffort,
+    cliVersion: options.cliVersion,
+  });
+}
+
+/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */
+export const codexDefinition: AgentDefinition = {
+  runner: codexRunner,
+  parser: codexParser,
+};