From ab4ac516859af6d93a208d17ebd710fc80465db3 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 19 Jun 2026 00:42:20 +0100 Subject: [PATCH 1/5] feat(eval): add Codex agent (AI-849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `codex` (OpenAI Codex) as an eval agent on the runner/parser framework from #47: a runner + a parser + a registry entry + a factory + experiments. Orchestration is unchanged; Codex's transcript is parsed into the same surface scorers already use. - runners/codex.ts: install + `codex login --with-api-key` (key via stdin, never argv), `codex exec --json` (--skip-git-repo-check --dangerously-bypass-approvals-and-sandbox -m , prompt = system+user on stdin), MCP via a generated ~/.codex/config.toml. Model type from openai's ChatModel widened with (string & {}), exported as CodexModel. - runners/codex.ts deriveStopReason: `codex exec` exits 0 even on a failed turn, so the stop reason comes from the terminal turn.completed/turn.failed event, not the exit code — a clean stop and an agent failure stay distinct. - parsers/codex.ts: the thread/turn/item event schema; paired tool_call/ tool_result by item id; owns its CODEX_TOOLS map; normalized command/path via the shared extractArgs (raw args left untouched). Tool success is tri-state — a missing status is unknown, not auto-success. - Registered in parsers/registry.ts; codexAgent factory; experiments for gpt-5.4 and gpt-5.5. Adds the openai SDK as a compile-time dep. - eval-refresh workflow: run codex-gpt-5.4 + codex-gpt-5.5 alongside the claude-code experiments. Like Claude Code, Codex runs in BOTH modes: the sandbox carries its shell/file tools either way, and tools mode just drops the Supabase CLI + local stack so Supabase access goes through MCP. Which mode an eval uses is a property of the eval (interface/local dir), not the agent. Model note: Codex enables a `tool_search` tool that gpt-5.4-nano rejects with `400 invalid_request_error: "Tool 'tool_search' is not supported with gpt-5.4-nano."`, failing the turn immediately — use gpt-5.4-mini or larger. Verified e2e (CLI 0.138, gpt-5.4): local-stack build-tests-001 -> 3/3, and tools-mode investigate-db-001 -> 3/3 using MCP (execute_sql / list_tables / list_projects), both stoppedReason "stop". gpt-5.4-nano reproduces the tool_search 400 -> stoppedReason "error". Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/eval-refresh.yml | 4 +- experiments/codex-gpt-5.4.ts | 22 +++ experiments/codex-gpt-5.5.ts | 22 +++ packages/core/package.json | 1 + packages/core/src/cli-agent.ts | 24 +++- packages/core/src/index.ts | 8 +- packages/core/src/parsers/codex.test.ts | 172 ++++++++++++++++++++++ packages/core/src/parsers/codex.ts | 183 ++++++++++++++++++++++++ packages/core/src/parsers/registry.ts | 2 + packages/core/src/runners/codex.ts | 140 ++++++++++++++++++ pnpm-lock.yaml | 22 +++ pnpm-workspace.yaml | 1 + 12 files changed, 593 insertions(+), 8 deletions(-) create mode 100644 experiments/codex-gpt-5.4.ts create mode 100644 experiments/codex-gpt-5.5.ts create mode 100644 packages/core/src/parsers/codex.test.ts create mode 100644 packages/core/src/parsers/codex.ts create mode 100644 packages/core/src/runners/codex.ts diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index cc9c067a..6ca7b47d 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -6,7 +6,7 @@ on: experiments: description: "Comma-separated experiment names to run" required: true - default: "openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-haiku-4.5,claude-code-sonnet-4.6" + default: "openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-haiku-4.5,claude-code-sonnet-4.6,codex-gpt-5.4,codex-gpt-5.5" eval: description: "Optional single eval id to run" required: false @@ -72,7 +72,7 @@ jobs: runs="${{ inputs.runs }}" timeout_sec="${{ inputs.timeout_sec }}" else - experiments="openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-haiku-4.5,claude-code-sonnet-4.6" + experiments="openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-haiku-4.5,claude-code-sonnet-4.6,codex-gpt-5.4,codex-gpt-5.5" eval_id="" suite="benchmark" runs="1" diff --git a/experiments/codex-gpt-5.4.ts b/experiments/codex-gpt-5.4.ts new file mode 100644 index 00000000..ebaca610 --- /dev/null +++ b/experiments/codex-gpt-5.4.ts @@ -0,0 +1,22 @@ +import { + codexAgent, + defineExperiment, + platformLiteRuntime, + supabaseMcpServer, +} from "@supabase-evals/core"; +import { localStackRuntime } from "@supabase-evals/sandbox"; + +// Codex runs in both modes, like Claude Code: `runtime` drives tools-mode evals +// (the runner writes its MCP servers into ~/.codex/config.toml against platform- +// lite) and `localStack` drives local-stack evals. Which mode an eval uses is a +// property of the eval (interface/local dir), not the agent. +export default defineExperiment({ + agent: codexAgent({ + model: "gpt-5.4", + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ["supabase", "supabase-postgres-best-practices"], +}); diff --git a/experiments/codex-gpt-5.5.ts b/experiments/codex-gpt-5.5.ts new file mode 100644 index 00000000..47f50c31 --- /dev/null +++ b/experiments/codex-gpt-5.5.ts @@ -0,0 +1,22 @@ +import { + codexAgent, + defineExperiment, + platformLiteRuntime, + supabaseMcpServer, +} from "@supabase-evals/core"; +import { localStackRuntime } from "@supabase-evals/sandbox"; + +// Codex runs in both modes, like Claude Code: `runtime` drives tools-mode evals +// (the runner writes its MCP servers into ~/.codex/config.toml against platform- +// lite) and `localStack` drives local-stack evals. Which mode an eval uses is a +// property of the eval (interface/local dir), not the agent. +export default defineExperiment({ + agent: codexAgent({ + model: "gpt-5.5", + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ["supabase", "supabase-postgres-best-practices"], +}); diff --git a/packages/core/package.json b/packages/core/package.json index 4389a2e2..4cce8026 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -18,6 +18,7 @@ }, "dependencies": { "@anthropic-ai/sdk": "catalog:", + "openai": "catalog:", "@ai-sdk/mcp": "catalog:", "@ai-sdk/openai": "catalog:", "@supabase-evals/platform-lite": "workspace:*", diff --git a/packages/core/src/cli-agent.ts b/packages/core/src/cli-agent.ts index 7605dd0f..aac646f1 100644 --- a/packages/core/src/cli-agent.ts +++ b/packages/core/src/cli-agent.ts @@ -2,10 +2,9 @@ * CLI-agent harnesses. * * `aiSdkAgent` drives the model loop in-process: we own the tools and record - * the transcript as it happens. A CLI agent (Claude Code, and later Codex / - * Gemini CLI / …) is the opposite — its own harness with its own tools, loop, - * and MCP client. So we run it inside the eval sandbox and parse the transcript - * it produces. + * the transcript as it happens. A CLI agent (Claude Code, Codex, …) is the + * opposite — its own harness with its own tools, loop, and MCP client. So we + * run it inside the eval sandbox and parse the transcript it produces. * * Three concerns are split so each lives in one place: * - runner (`./runners/.ts`): install + exec + permission flags + MCP @@ -24,7 +23,9 @@ import type { AgentHarness, AgentRunResult } from "./index.js"; import { adaptTranscript } from "./parsers/adapt.js"; import type { AgentTranscriptParser } from "./parsers/types.js"; import { claudeCodeParser } from "./parsers/claude-code.js"; +import { codexParser } from "./parsers/codex.js"; import { claudeCodeRunner } from "./runners/claude-code.js"; +import { codexRunner, type CodexModel } from "./runners/codex.js"; import type { AgentRunner } from "./runners/types.js"; import { SCRATCH, @@ -124,3 +125,18 @@ export function claudeCodeAgent( cliVersion: options.cliVersion, }); } + +/** OpenAI Codex as an `AgentHarness`. Runs in both modes, like Claude Code. */ +export function codexAgent( + options: { + /** OpenAI model id (typed from `openai`; any string accepted). */ + model?: CodexModel; + /** Override the pinned CLI version. */ + cliVersion?: string; + } = {}, +): AgentHarness { + return createCliAgent(codexRunner, codexParser, { + model: options.model ?? codexRunner.defaultModel, + cliVersion: options.cliVersion, + }); +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index a14099e3..c64f4ef3 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -72,8 +72,12 @@ export { rawEvalResultSchema, } from "./eval-metadata.js"; export { parseEvalMarkdown } from "./eval-markdown.js"; -// CLI agent harnesses (Claude Code, and the runner/parser framework for adding more). -export { createCliAgent, claudeCodeAgent } from "./cli-agent.js"; +// CLI agent harnesses (Claude Code, Codex, and the framework for adding more). +export { + createCliAgent, + claudeCodeAgent, + codexAgent, +} from "./cli-agent.js"; export type { AgentSandbox, AgentRunner, diff --git a/packages/core/src/parsers/codex.test.ts b/packages/core/src/parsers/codex.test.ts new file mode 100644 index 00000000..bc28159d --- /dev/null +++ b/packages/core/src/parsers/codex.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from "vitest"; +import { codexParser } from "./codex.js"; +import { codexRunner } from "../runners/codex.js"; +import { adaptTranscript } from "./adapt.js"; + +/** A representative `codex exec --json` stream (shapes captured from CLI 0.138). */ +const SESSION = [ + JSON.stringify({ type: "thread.started", thread_id: "t1" }), + JSON.stringify({ type: "turn.started" }), + JSON.stringify({ + type: "item.completed", + item: { id: "item_0", type: "agent_message", text: "I'll run a command and write a file." }, + }), + JSON.stringify({ + type: "item.completed", + item: { + id: "item_1", + type: "command_execution", + command: "/bin/zsh -lc 'echo hi'", + aggregated_output: "hi\n", + exit_code: 0, + status: "completed", + }, + }), + JSON.stringify({ + type: "item.completed", + item: { + id: "item_2", + type: "file_change", + changes: [{ path: "/work/note.txt", kind: "add" }], + status: "completed", + }, + }), + JSON.stringify({ + type: "item.completed", + item: { id: "item_3", type: "agent_message", text: "Done." }, + }), + JSON.stringify({ type: "turn.completed", usage: { input_tokens: 10, output_tokens: 3 } }), +].join("\n"); + +describe("codexParser", () => { + it("maps command_execution + file_change to canonical tool calls, paired with results", () => { + const { events, errors } = codexParser.parseTranscript(SESSION); + expect(errors).toEqual([]); + + const calls = events.filter((e) => e.type === "tool_call"); + expect(calls.map((e) => e.tool?.name)).toEqual(["shell", "file_write"]); + expect(calls.map((e) => e.tool?.originalName)).toEqual(["command_execution", "file_change"]); + // Normalized views live on the event's tool; raw args are left untouched. + expect(calls[0].tool?.command).toBe("/bin/zsh -lc 'echo hi'"); + expect(calls[0].tool?.args).toEqual({ command: "/bin/zsh -lc 'echo hi'" }); + expect(calls[1].tool?.path).toBe("/work/note.txt"); + expect(calls[1].tool?.args).toEqual({ changes: [{ path: "/work/note.txt", kind: "add" }] }); + + const results = events.filter((e) => e.type === "tool_result"); + expect(results.map((e) => e.tool?.id)).toEqual(["item_1", "item_2"]); + expect(results.every((e) => e.tool?.success === true)).toBe(true); + }); + + it("ignores thread/turn envelopes and surfaces a clean transcript via the adapter", () => { + const adapted = adaptTranscript(codexParser.parseTranscript(SESSION).events); + expect(adapted.agentReport).toBe("Done."); + expect(adapted.steps).toBe(2); // two agent_message turns + expect(adapted.toolCalls).toEqual([ + { + endpoint: "command_execution", + body: { command: "/bin/zsh -lc 'echo hi'" }, + command: "/bin/zsh -lc 'echo hi'", + result: "hi\n", + error: undefined, + ts: 0, + }, + { + endpoint: "file_change", + body: { changes: [{ path: "/work/note.txt", kind: "add" }] }, + path: "/work/note.txt", + result: "completed", + error: undefined, + ts: 0, + }, + ]); + }); + + it("maps reasoning to a thinking event", () => { + const { events } = codexParser.parseTranscript( + JSON.stringify({ + type: "item.completed", + item: { id: "r0", type: "reasoning", text: "Thinking about it." }, + }), + ); + expect(events).toEqual([{ type: "thinking", content: "Thinking about it." }]); + }); + + it("marks a non-zero exit code as a failed shell call (error surfaced via adapter)", () => { + const stream = [ + JSON.stringify({ + type: "item.completed", + item: { + id: "c1", + type: "command_execution", + command: "false", + aggregated_output: "nope", + exit_code: 1, + status: "completed", + }, + }), + ].join("\n"); + + const { events } = codexParser.parseTranscript(stream); + const result = events.find((e) => e.type === "tool_result"); + expect(result?.tool?.success).toBe(false); + // success:false routes the output into `error`, not `result`, in the adapter. + const adapted = adaptTranscript(events); + expect(adapted.toolCalls[0].error).toBe("nope"); + expect(adapted.toolCalls[0].result).toBeUndefined(); + }); + + it("treats a tool item with no recognizable status as unknown, not success", () => { + const stream = JSON.stringify({ + type: "item.completed", + item: { id: "m1", type: "mcp_tool_call", tool: "search_docs", result: "ok" }, + }); + const result = codexParser + .parseTranscript(stream) + .events.find((e) => e.type === "tool_result"); + expect(result?.tool?.success).toBeUndefined(); + expect(result?.tool?.originalName).toBe("search_docs"); + }); + + it("emits an error event for a failed turn", () => { + const stream = [ + JSON.stringify({ type: "turn.started" }), + JSON.stringify({ type: "turn.failed", error: { message: "model overloaded" } }), + ].join("\n"); + const { events } = codexParser.parseTranscript(stream); + expect(events).toEqual([{ type: "error", content: "model overloaded" }]); + }); + + it("never throws on malformed lines and reports them as errors", () => { + const { events, errors } = codexParser.parseTranscript( + "not json\n" + JSON.stringify({ type: "turn.started" }), + ); + expect(events).toEqual([]); + expect(errors.length).toBe(1); + }); +}); + +describe("codexRunner.deriveStopReason", () => { + const ok: Parameters>[1] = { + ok: true, + exitCode: 0, + stdout: "", + stderr: "", + }; + + it("returns 'stop' on a completed turn even though the process also exits 0", () => { + const raw = JSON.stringify({ type: "turn.completed", usage: {} }); + expect(codexRunner.deriveStopReason!(raw, ok)).toBe("stop"); + }); + + it("returns 'error' on a failed turn despite a 0 exit code", () => { + const raw = JSON.stringify({ type: "turn.failed", error: { message: "boom" } }); + // The exit code is 0 (Codex doesn't fail the process), so without this hook + // the run would be mis-reported as a clean stop. + expect(codexRunner.deriveStopReason!(raw, ok)).toBe("error"); + }); + + it("falls back to the process heuristic when there's no terminal turn event", () => { + const timedOut = { ok: false, exitCode: 124, stdout: "", stderr: "timed out" }; + expect(codexRunner.deriveStopReason!("", timedOut)).toBe("timeout"); + }); +}); diff --git a/packages/core/src/parsers/codex.ts b/packages/core/src/parsers/codex.ts new file mode 100644 index 00000000..bb7c134e --- /dev/null +++ b/packages/core/src/parsers/codex.ts @@ -0,0 +1,183 @@ +/** + * Codex transcript parser — for `codex exec --json` (CLI ≥ ~0.130). + * + * The stream is newline-delimited thread/turn/item events: + * {"type":"thread.started","thread_id":"…"} + * {"type":"turn.started"} + * {"type":"item.started","item":{…}} // ignored — item.completed has everything + * {"type":"item.completed","item":{"id","type",…}} + * {"type":"turn.completed","usage":{…}} + * + * Observed `item.type`s: `agent_message` {text}, `reasoning` {text}, + * `command_execution` {command, aggregated_output, exit_code, status}, + * `file_change` {changes:[{path,kind}], status}. MCP / web-search items are + * handled best-effort. Each tool item yields a paired tool_call + tool_result + * (correlated by the item id) so the adapter can attach the output. + * + * NB: this is the `--json` event schema, NOT the `~/.codex/sessions` rollout + * format (event_msg/response_item) that older parsers targeted. + */ + +import { isRecord, parseJsonlRecords } from "../json.js"; +import type { ParsedTranscript, TranscriptEvent } from "../transcript/types.js"; +import type { AgentTranscriptParser } from "./types.js"; +import { normalizeToolName, type AgentToolMap } from "./shared/normalize.js"; +import { extractArgs, type ArgFieldMap, type ExtractedArgs } from "./shared/extract.js"; + +/** + * Codex's tool names → canonical names. Codex names built-in tools by item type + * (`command_execution`/`file_change`), not by a tool name. Owned here, not in shared. + */ +const CODEX_TOOLS: AgentToolMap = { + tools: { + command_execution: "shell", + exec_command: "shell", + local_shell_call: "shell", + file_change: "file_write", + apply_patch: "file_write", + web_search: "web_search", + mcp_tool_call: "tool_use", + }, +}; + +/** + * Codex's tool args → normalized fields. `command_execution` carries the shell + * command in `command`; `file_change`'s path is nested under `changes[].path`, + * so it's extracted directly (see `firstChangedPath`) rather than via this map. + */ +const CODEX_ARG_FIELDS: ArgFieldMap = { + command: ["command"], +}; + +function str(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +/** + * Tri-state success from a Codex `status` field: completed → true, failed → + * false, anything else (absent / in-progress / unrecognized) → undefined + * (unknown). Mirrors how `command_execution` treats a missing exit code, so a + * tool item with no status isn't silently scored as a success. + */ +function statusSuccess(status: unknown): boolean | undefined { + if (status === "completed") return true; + if (status === "failed") return false; + return undefined; +} + +/** Extract the first changed path from a `file_change` item. */ +function firstChangedPath(item: Record): string | undefined { + if (!Array.isArray(item.changes)) return undefined; + for (const change of item.changes) { + if (isRecord(change) && typeof change.path === "string") return change.path; + } + return undefined; +} + +/** + * Emit a paired tool_call + tool_result for one completed tool item. `args` stays + * raw; `normalized` carries the agent-agnostic path/command/url views (set on the + * tool_call so scorers read them without knowing Codex's item shapes). + */ +function toolCallPair( + id: string, + originalName: string, + args: Record, + result: unknown, + success: boolean | undefined, + normalized: ExtractedArgs = {}, +): TranscriptEvent[] { + const name = normalizeToolName(originalName, CODEX_TOOLS); + const tool: NonNullable = { name, originalName, id, args }; + if (normalized.path) tool.path = normalized.path; + if (normalized.command) tool.command = normalized.command; + if (normalized.url) tool.url = normalized.url; + return [ + { type: "tool_call", tool }, + { type: "tool_result", tool: { name, originalName, id, result, success } }, + ]; +} + +function itemToEvents(item: Record): TranscriptEvent[] { + const id = str(item.id) ?? ""; + const itemType = str(item.type); + + switch (itemType) { + case "agent_message": { + const text = str(item.text); + return text ? [{ type: "message", role: "assistant", content: text }] : []; + } + case "reasoning": { + const text = str(item.text); + return text ? [{ type: "thinking", content: text }] : []; + } + case "command_execution": { + const command = str(item.command); + const args = command ? { command } : {}; + const exitCode = typeof item.exit_code === "number" ? item.exit_code : undefined; + return toolCallPair( + id, + "command_execution", + args, + item.aggregated_output, + exitCode === undefined ? undefined : exitCode === 0, + extractArgs(args, CODEX_ARG_FIELDS), + ); + } + case "file_change": { + // Codex may touch several files in one item; `path` normalizes the first + // (matching the single normalized `path` field), raw `changes` keeps all. + const path = firstChangedPath(item); + return toolCallPair( + id, + "file_change", + { changes: item.changes }, + item.status, + statusSuccess(item.status), + { path }, + ); + } + case "mcp_tool_call": { + // Shape not pinned across versions — be defensive about field names and + // treat a missing status as unknown (not success). + const tool = str(item.tool) ?? str(item.name) ?? str(item.server) ?? "mcp_tool_call"; + return toolCallPair(id, tool, item, item.result ?? item.output, statusSuccess(item.status)); + } + case "web_search": { + return toolCallPair(id, "web_search", { query: item.query }, undefined, statusSuccess(item.status)); + } + default: + return []; + } +} + +function recordToEvents(data: Record): TranscriptEvent[] { + switch (data.type) { + case "item.completed": + return isRecord(data.item) ? itemToEvents(data.item) : []; + case "turn.failed": + case "error": { + const message = + (isRecord(data.error) && str(data.error.message)) || str(data.message); + return [{ type: "error", content: message ?? JSON.stringify(data) }]; + } + // thread.started / turn.started / item.started / turn.completed: no event. + default: + return []; + } +} + +export const codexParser: AgentTranscriptParser = { + parseTranscript(raw: string): ParsedTranscript { + const { records, errors } = parseJsonlRecords(raw); + const events: TranscriptEvent[] = []; + for (const record of records) { + try { + events.push(...recordToEvents(record)); + } catch (e) { + errors.push(e instanceof Error ? e.message : String(e)); + } + } + return { events, errors }; + }, +}; diff --git a/packages/core/src/parsers/registry.ts b/packages/core/src/parsers/registry.ts index 6ac305db..e489e434 100644 --- a/packages/core/src/parsers/registry.ts +++ b/packages/core/src/parsers/registry.ts @@ -7,9 +7,11 @@ import type { AgentTranscriptParser } from "./types.js"; import { claudeCodeParser } from "./claude-code.js"; +import { codexParser } from "./codex.js"; const PARSERS: Record = { "claude-code": claudeCodeParser, + codex: codexParser, }; /** Agent ids with a registered transcript parser. */ diff --git a/packages/core/src/runners/codex.ts b/packages/core/src/runners/codex.ts new file mode 100644 index 00000000..c8060914 --- /dev/null +++ b/packages/core/src/runners/codex.ts @@ -0,0 +1,140 @@ +/** + * Codex runner. Headless via `codex exec --json` (newline-delimited thread/turn/ + * item events on stdout; see parsers/codex.ts). Like Claude Code, it runs in + * both modes: the sandbox carries its shell/file tools in either case, and tools + * mode just drops the Supabase CLI + local stack so Supabase access goes through + * MCP (`~/.codex/config.toml`). Runs under `--dangerously-bypass-approvals-and- + * sandbox` — the eval sandbox is the isolation boundary. + */ + +import type { ChatModel } from "openai/resources/shared"; +import type { McpServerConfig } from "../index.js"; +import { parseJsonlRecords } from "../json.js"; +import type { AgentRunner } from "./types.js"; +import { + npmGlobalBin, + npmInstallGlobal, + processStopReason, + shellQuote, + writeSandboxFile, +} from "./shared.js"; + +// ChatModel is a closed union; widen so newer/codex-specific ids still type. +export type CodexModel = ChatModel | (string & {}); + +const CODEX_CONFIG_PATH = '"$HOME/.codex/config.toml"'; + +export const codexRunner: AgentRunner = { + id: "codex", + displayName: "OpenAI Codex", + apiKeyEnvVar: "OPENAI_API_KEY", + cliPackage: "@openai/codex", + // Pinned: Codex's --json event schema evolves; bump deliberately and re-check + // the parser. See packages/core/src/parsers/codex.ts. + defaultCliVersion: "0.138.0", + defaultModel: "gpt-5.4", + + async install(sandbox, version, apiKey) { + await npmInstallGlobal(sandbox, `${this.cliPackage}@${version}`, this.displayName); + // Persist API-key auth to ~/.codex/auth.json (read the key from stdin so it + // never lands in argv or the process table). + const codex = npmGlobalBin("codex"); + const login = await sandbox.exec(`printenv OPENAI_API_KEY | ${codex} login --with-api-key`, { + env: { OPENAI_API_KEY: apiKey }, + }); + if (!login.ok) { + throw new Error(`Codex login failed: ${login.stderr || login.stdout}`); + } + }, + + async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, timeoutSec }) { + const codex = npmGlobalBin("codex"); + if (Object.keys(mcpServers).length > 0) { + await sandbox.exec(`mkdir -p "$HOME/.codex"`); + await writeSandboxFile(sandbox, CODEX_CONFIG_PATH, buildCodexConfig(mcpServers)); + } + + const flags = [ + "exec", + "--json", + // The workspace may not be a git repo; don't refuse to run. + "--skip-git-repo-check", + // The sandbox is the isolation boundary — let Codex run commands freely. + "--dangerously-bypass-approvals-and-sandbox", + `-m ${shellQuote(model)}`, + // Read the prompt from stdin. + "-", + ].join(" "); + + // Codex has no system-prompt flag; prepend the system prompt to the task, + // both staged as files, fed on stdin. + const command = await sandbox.exec( + `{ cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath}; } | ${codex} ${flags}`, + { timeoutMs: timeoutSec * 1000, env: { OPENAI_API_KEY: apiKey } }, + ); + return { command, raw: command.stdout }; + }, + + deriveStopReason(raw, command) { + // `codex exec` exits 0 even when a turn fails, so the process result alone + // can't tell a clean stop from an agent-level failure. Trust the terminal + // stream event: `turn.completed` = clean stop, `turn.failed`/`error` = + // failure. Only when there's no terminal event (crash / kill / timeout) do + // we fall back to the process-exit heuristic. + switch (terminalOutcome(raw)) { + case "completed": + return "stop"; + case "failed": + return "error"; + default: + return processStopReason(command); + } + }, +}; + +/** The last turn-level outcome in a `codex exec --json` stream, if any. */ +function terminalOutcome(raw: string | undefined): "completed" | "failed" | undefined { + if (!raw) return undefined; + const { records } = parseJsonlRecords(raw); + for (let i = records.length - 1; i >= 0; i -= 1) { + const type = records[i].type; + if (type === "turn.completed") return "completed"; + if (type === "turn.failed" || type === "error") return "failed"; + } + return undefined; +} + +/** + * Codex's `~/.codex/config.toml` MCP schema: + * [mcp_servers.] + * command = "npx" + * args = ["…"] + * env = { KEY = "val" } + */ +function buildCodexConfig(servers: Record): string { + const blocks: string[] = []; + for (const [name, server] of Object.entries(servers)) { + const lines = [`[mcp_servers.${tomlKey(name)}]`, `command = ${tomlString(server.command)}`]; + if (server.args?.length) { + lines.push(`args = [${server.args.map(tomlString).join(", ")}]`); + } + if (server.env && Object.keys(server.env).length > 0) { + const entries = Object.entries(server.env) + .map(([k, v]) => `${tomlKey(k)} = ${tomlString(v)}`) + .join(", "); + lines.push(`env = { ${entries} }`); + } + blocks.push(lines.join("\n")); + } + return blocks.join("\n\n") + "\n"; +} + +/** TOML basic string — JSON string escaping is a valid subset. */ +function tomlString(value: string): string { + return JSON.stringify(value); +} + +/** A bare TOML key if safe, else a quoted key. */ +function tomlKey(key: string): string { + return /^[A-Za-z0-9_-]+$/.test(key) ? key : JSON.stringify(key); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 41ff9a4e..2c3d9512 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,6 +36,9 @@ catalogs: ai: specifier: ^6.0.174 version: 6.0.199 + openai: + specifier: ^6.44.0 + version: 6.44.0 typescript: specifier: ^5.9.3 version: 5.9.3 @@ -260,6 +263,9 @@ importers: gray-matter: specifier: ^4.0.3 version: 4.0.3 + openai: + specifier: 'catalog:' + version: 6.44.0(ws@8.21.0)(zod@4.4.3) typescript: specifier: 'catalog:' version: 5.9.3 @@ -3549,6 +3555,17 @@ packages: resolution: {integrity: sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==} engines: {node: '>=20'} + openai@6.44.0: + resolution: {integrity: sha512-09/gH+8jH0RgUwsgWHAaxsKGRT5zVZ95IaJUnqAWj6XejIBmnFRwq2WUIF37VtDEsmGrtPmvCs5+yBSeZGWvkA==} + peerDependencies: + ws: ^8.18.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + openapi-fetch@0.13.8: resolution: {integrity: sha512-yJ4QKRyNxE44baQ9mY5+r/kAzZ8yXMemtNAOFwOzRXJscdjSxxzWSNlyBAr+o5JjkUw9Lc3W7OIoca0cY3PYnQ==} @@ -7552,6 +7569,11 @@ snapshots: powershell-utils: 0.1.0 wsl-utils: 0.3.1 + openai@6.44.0(ws@8.21.0)(zod@4.4.3): + optionalDependencies: + ws: 8.21.0 + zod: 4.4.3 + openapi-fetch@0.13.8: dependencies: openapi-typescript-helpers: 0.0.15 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 41a09273..e37476d2 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -4,6 +4,7 @@ packages: catalog: '@anthropic-ai/sdk': ^0.105.0 + 'openai': ^6.44.0 '@ai-sdk/anthropic': ^3.0.71 '@ai-sdk/mcp': ^1.0.39 '@ai-sdk/openai': ^3.0.66 From 4d9e2fe551d4a8a98409f23930dfeb0661545539 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:39:17 +0000 Subject: [PATCH 2/5] chore: refresh eval results --- apps/web/src/data/eval-results.json | 1779 ++++++++++++++++++++------- 1 file changed, 1330 insertions(+), 449 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index a1356b95..81a55e81 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -13,7 +13,7 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase project initialised (supabase/config.toml exists)", @@ -26,7 +26,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 5 rows" }, { "name": "row level security is enabled on todos", @@ -34,8 +34,7 @@ }, { "name": "a SELECT policy targets the authenticated role", - "passed": false, - "notes": "policies found: [{\"policyname\":\"Authenticated users can read todos\",\"cmd\":\"SELECT\",\"roles\":[\"public\"]}]" + "passed": true }, { "name": "REST API returns no todos to anonymous requests", @@ -45,7 +44,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "5 rows" } ], "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", @@ -91,45 +90,6 @@ "attempts": 1, "sourcePath": "claude-code-haiku-4.5/build-cli-002-declarative-schema.json" }, - { - "experiment": "claude-code-haiku-4.5", - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": false, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": false, - "notes": "HTTP 404: Function not found" - } - ], - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-haiku-4.5/build-cli-003-pg-cron-queue-workflow.json" - }, { "experiment": "claude-code-haiku-4.5", "eval": "build-functions-004-service-role-bypass", @@ -154,22 +114,22 @@ { "name": "user A reads own note", "passed": false, - "notes": "edge function import not supported: https://deno.land/x/jwt@v1.0.2/mod.ts" + "notes": "edge function import not supported: https://deno.land/x/jwt@v0.1.1/mod.ts" }, { "name": "reads only with the caller's JWT", "passed": false, - "notes": "edge function import not supported: https://deno.land/x/jwt@v1.0.2/mod.ts" + "notes": "edge function import not supported: https://deno.land/x/jwt@v0.1.1/mod.ts" }, { "name": "user A cannot force-read user B note", "passed": false, - "notes": "edge function import not supported: https://deno.land/x/jwt@v1.0.2/mod.ts" + "notes": "edge function import not supported: https://deno.land/x/jwt@v0.1.1/mod.ts" }, { "name": "user B cannot force-read user A note", "passed": false, - "notes": "edge function import not supported: https://deno.land/x/jwt@v1.0.2/mod.ts" + "notes": "edge function import not supported: https://deno.land/x/jwt@v0.1.1/mod.ts" } ], "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", @@ -190,45 +150,12 @@ "sdk" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019ef9e2-fe79-7074-a8c8-7900ef2a4515/receipt-alpha.pdf, 019ef9e2-fe79-7074-a8c8-7900ef2a4515/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Creates a private user-files bucket, enables RLS, defines authenticated SELECT and INSERT policies scoped to the bucket and auth.uid()-based folder ownership, and uses createSignedUrl with an expiration for temporary sharing." + "passed": false, + "notes": "no row in storage.buckets with id or name 'user-files'" } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -253,18 +180,18 @@ "checks": [ { "name": "pgTAP test file(s) written under supabase/tests/", - "passed": false, - "notes": "no .sql files found under supabase/tests/" + "passed": true, + "notes": "2 file(s): supabase/tests/tenant_isolation.sql, supabase/tests/tenant_isolation_with_jwt.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": false, - "notes": "no test summary found; exit 0; output: Connecting to local database...\n3.36: Pulling from supabase/pg_prove\ndcccee43ad5d: Pulling fs layer\n06d62d0de6d7: Pulling fs layer\na22cb17b3b93: Pulling fs layer\n4f4fb700ef54: Pulling fs layer\n4f4fb700ef54: Waiting\na22cb17b3b93: Download complete\ndcccee43ad5d: Verifying Checksum\ndcccee43ad5d: Download complete\n4f4fb700ef54: Verifying Checksum\n4f4fb700ef54: Download complete\n06d62d0de6d7: Verifying Checksum\n06d62d0de6d7: Download complete\ndcccee43ad5d: Pull complete\n06d62d0de6d7: Pull complete\na2" + "notes": "no test summary found; exit 1; output: Connecting to local database...\n3.36: Pulling from supabase/pg_prove\ndcccee43ad5d: Pulling fs layer\n06d62d0de6d7: Pulling fs layer\na22cb17b3b93: Pulling fs layer\n4f4fb700ef54: Pulling fs layer\n4f4fb700ef54: Waiting\na22cb17b3b93: Verifying Checksum\na22cb17b3b93: Download complete\ndcccee43ad5d: Verifying Checksum\ndcccee43ad5d: Download complete\n4f4fb700ef54: Verifying Checksum\n4f4fb700ef54: Download complete\n06d62d0de6d7: Verifying Checksum\n06d62d0de6d7: Download complete\ndcccee43ad5d: Pull comple" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with the tenant isolation flaw, explains that authenticated members can read posts from organizations they are not members of, and grounds the conclusion in test results showing notes pass while posts fail. It does not blame notes or dismiss the test signal." + "judgeNotes": "The agent correctly identifies the `posts` table SELECT policy as broken tenant isolation, specifically that authenticated members of any organization can read posts from other organizations due to a missing `m.org_id = posts.org_id` check. It grounds this in its test results showing cross-org post reads. Although it also incorrectly claims notes failures and adds extra issues, it does not blame notes instead of posts and does draw the required conclusion about posts." } ], "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", @@ -290,7 +217,7 @@ { "name": "scorer evaluated vector search", "passed": false, - "notes": "column \"owner_id\" of relation \"documents\" does not exist" + "notes": "operator does not exist: uuid = integer" } ], "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", @@ -318,12 +245,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: Supabase scrape uses basic_auth.password with an environment-substituted secret instead of basic_auth.password_file, and docker-compose.yml does not mount/provide that password_file via a volume or Compose secret." + "judgeNotes": "Fails: Supabase scrape uses basic_auth.password with an environment variable instead of basic_auth.password_file, and docker-compose.yml does not mount/provide that password_file via a volume or Compose secret." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README includes env file setup and Prometheus target verification, but it instructs users to use the Supabase service_role key rather than creating/using a Secret API key matching SUPABASE_SECRET_API_KEY. It also only says to start the stack, not clearly restart/reload an existing Compose stack." + "judgeNotes": "README is too vague: it does not give concrete steps for creating the Secret API key, does not specify the exact required secret/env file contents or variable names, and only says to start the stack rather than restart/reload an existing Compose stack. Verification via Prometheus targets is present, but setup instructions are incomplete." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -357,7 +284,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -369,44 +296,6 @@ "attempts": 1, "sourcePath": "claude-code-haiku-4.5/deploy-functions-001-edge-function-secrets.json" }, - { - "experiment": "claude-code-haiku-4.5", - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-haiku-4.5/deploy-self-hosting-001-docker-compose.json" - }, { "experiment": "claude-code-haiku-4.5", "eval": "investigate-auth-001-deleted-user-access", @@ -431,13 +320,12 @@ }, { "name": "delete-account revokes the user's sessions", - "passed": false, - "notes": "sessions left: 1" + "passed": true, + "notes": "sessions left: 0" }, { "name": "deleted user's refresh token is rejected", - "passed": false, - "notes": "refresh token still produces a session" + "passed": true }, { "name": "deleted user cannot sign back in", @@ -451,7 +339,7 @@ { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The answer correctly identifies that the flow only soft-deletes the profile and explains publishable vs secret keys reasonably, including that secret keys are server-only and bypass RLS. It also explains a JWT expiry window. However, it does not actually fix the delete-account flow with real auth user/session/refresh-token revocation; it only changes RLS policies and lists revocation as a future/consideration. The rubric requires deleting the auth user or equivalently removing identity/sessions and revoking sessions/refresh tokens. It also incorrectly frames the issue partly as RLS not checking deletion rather than the required core fix, and suggests revokeOtherSessions rather than a complete account/session revocation flow." + "judgeNotes": "The answer identifies soft deletion and refresh/session revocation issues, and explains publishable vs secret keys correctly. However, the implemented fix does not delete the auth user or fully/equivalently remove identities and refresh tokens; it only deletes rows from auth.sessions and relies on RLS soft-delete checks. It also does not clearly recommend auth.getUser()/short JWT expiry for server-side checks vs local getClaims(), and incorrectly suggests the remaining risk is only client-side caching/backend RLS blocks rather than explaining stateless JWTs remain valid until expiry for endpoints relying on local validation." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", @@ -471,11 +359,11 @@ "sdk" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "orders table added to supabase_realtime publication", - "passed": true + "passed": false }, { "name": "courier_locations still in supabase_realtime publication", @@ -496,8 +384,8 @@ }, { "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly identified that the channel could subscribe but INSERT events were absent because `orders` was missing from `supabase_realtime`, applied `ALTER PUBLICATION supabase_realtime ADD TABLE public.orders`, verified both `courier_locations` and `orders` remained published, and did not weaken RLS or policies." + "passed": false, + "judgeNotes": "The assistant incorrectly diagnosed the root cause as a missing RLS SELECT policy and proposed adding/weakening policies. It did not identify or fix the actual issue: the orders table missing from the supabase_realtime publication via ALTER PUBLICATION supabase_realtime ADD TABLE orders." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -521,17 +409,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as affected and described the recurring 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z." + "judgeNotes": "The assistant named image-transform as the affected function and described recurring 503 gateway errors across the morning of 2026-04-28, roughly covering the 07:00Z-12:00Z pattern. It did not incorrectly focus on billing-webhook." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes recurring image-transform 503s to the gateway/load balancer/platform layer, grounded in the observed mismatch between gateway 503 log entries and successful function/runtime logs, and distinguishes the avatar-upload 500 as separate." + "judgeNotes": "Attributes image-transform 503s to gateway/load balancer layer, grounded in gateway logs showing 503s while Edge Function logs show successful 200 executions with normal durations. Does not blame function application code." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant provided specific actionable next steps, including checking API gateway health/routing, reviewing deployment logs, inspecting resource metrics for the April 28 07:00-12:00 UTC window, and checking rate limiting." + "judgeNotes": "The assistant recommended multiple concrete next steps, including checking Edge Function concurrency limits, gateway timeout settings, PostgreSQL connection pool usage, infrastructure changes, and function metrics/logs." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -553,7 +441,7 @@ "sdk" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "RLS still enabled on bookmarks", @@ -581,8 +469,8 @@ }, { "name": "diagnosed RLS and added owner-scoped policies", - "passed": false, - "judgeNotes": "The assistant correctly diagnosed RLS deny-all and created owner-scoped SELECT/INSERT policies, but did not scope them to the authenticated role. Omitting TO authenticated makes policies apply to PUBLIC, which the rubric disallows." + "passed": true, + "judgeNotes": "Diagnosed deny-all RLS with no policies, kept RLS enabled, and created owner-scoped SELECT and INSERT policies using auth.uid() = user_id / WITH CHECK. Extra UPDATE/DELETE owner policies do not violate the rubric." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -619,7 +507,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at_desc (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -715,7 +603,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -728,12 +616,12 @@ { "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "0 rows" + "notes": "error 42501: permission denied for table todos" }, { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", @@ -781,45 +669,1189 @@ }, { "experiment": "claude-code-sonnet-4.6", - "eval": "build-cli-003-pg-cron-queue-workflow", + "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ - "database", "edge-functions", - "cron", - "queues" + "auth", + "database" ], "topic": [ - "sql", + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true + }, + { + "name": "user A reads own note", + "passed": true + }, + { + "name": "reads only with the caller's JWT", + "passed": true + }, + { + "name": "user A cannot force-read user B note", + "passed": true + }, + { + "name": "user B cannot force-read user A note", + "passed": true + } + ], + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", "sdk" ], "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019ef70b-b907-73e9-baaf-120cbf095a41/receipt-alpha.pdf, 019ef70b-b907-73e9-baaf-120cbf095a41/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "The answer creates a private user-files bucket, defines authenticated owner-scoped SELECT and INSERT policies on storage.objects using the first path segment equals auth.uid(), keeps RLS enabled implicitly without disabling it, and provides supabase-js createSignedUrl code with an expiry. It does not make the bucket public, use permissive policies, anon/public roles, getPublicUrl, or service role client-side." + } + ], + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/tenant_isolation.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "9 passed, 6 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "Correctly identifies posts (not notes) as having the tenant isolation flaw: the SELECT policy lacks an org_id membership check, allowing authenticated members to read posts from other organizations. Grounds the conclusion in the pgTAP failures for posts isolation tests." + } + ], + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "scorer evaluated vector search", + "passed": false, + "notes": "operator does not exist: uuid = integer" + } + ], + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": false, + "judgeNotes": "Fails because the Supabase scrape uses inline basic_auth.password instead of password_file, and docker-compose.yml does not mount or define the password file via a volume or Compose secret. The app scrape is preserved and endpoint/path/scheme are otherwise correct." + }, + { + "name": "documented live deployment and verification steps", + "passed": false, + "judgeNotes": "README includes creating a Secret API key, restarting the Compose stack, and verification via curl/Prometheus targets/Grafana. However, the required secret-file setup is only optional and not part of the live configuration; prometheus.yml uses an inline password placeholder and docker-compose.yml does not mount the secret file, so the required matching secret file placement/setup is missing/mismatched." + } + ], + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "The answer identifies the soft-delete-only root cause, implements deletion of auth.sessions and auth.users, explains refresh/access token behavior and the JWT validity window, adds/mentions server-side session validation via auth.sessions, and correctly distinguishes publishable/frontend keys from secret/server-only keys that bypass RLS. It does overclaim that RLS session checks make the window zero for protected tables, but it still correctly notes stateless JWTs can remain valid until expiry and warns about non-RLS endpoints." + } + ], + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The assistant correctly identified that the channel can be SUBSCRIBED while orders INSERT events are not delivered because public.orders was missing from the supabase_realtime publication. It fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified the publication contents, and did not disable RLS, weaken policies, recreate the publication, or disturb courier_locations." + } + ], + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "The assistant correctly identified image-transform as the affected function and described the recurring HTTP 503 gateway failures throughout the morning of 2026-04-28, covering all 8 failures from 07:00Z to 12:00Z. It did not misattribute the main issue to billing-webhook." + }, + { + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": false, + "judgeNotes": "The assistant identifies the 503s as gateway-level but attributes them to Edge Function cold starts and recommends keeping/redeploying-style function-level mitigations. The rubric requires attribution to the gateway/platform layer in front of the function, not the function/runtime. Cold starts blame the function runtime/platform interaction rather than clearly the gateway/platform layer, so this fails." + }, + { + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended concrete next steps: setting up a Cron keep-warm job, adding retry logic, pulling specific avatar-upload edge function logs around a time window, and separately investigating billing-webhook 503s." + } + ], + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/investigate-reliability-003-edge-function-5xx-correlation.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all/zero Data API rows, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with USING and WITH CHECK." + } + ], + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "claude-code-sonnet-4.6", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-4.6/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": false + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/build-cli-002-declarative-schema.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true + }, + { + "name": "user A reads own note", + "passed": true + }, + { + "name": "reads only with the caller's JWT", + "passed": true + }, + { + "name": "user A cannot force-read user B note", + "passed": true + }, + { + "name": "user B cannot force-read user A note", + "passed": true + } + ], + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019ef70b-2717-7204-8571-aec531c29178/receipt-alpha.pdf, 019ef70b-2717-7204-8571-aec531c29178/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Configures a private user-files bucket, owner-scoped authenticated SELECT and INSERT policies on storage.objects with RLS left enabled, and uses createSignedUrl with an expiry for temporary sharing. No public bucket/getPublicUrl/service-role client-side issues." + } + ], + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/tenant_isolation_rls_test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "7 passed, 0 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "Correctly identifies `posts` as the tenant isolation flaw, states members of one org could read another org’s posts, distinguishes `notes` as already correct, and grounds the conclusion in reproduced/tested results." + } + ], + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "scorer evaluated vector search", + "passed": false, + "notes": "relation \"document_sections\" does not exist" + } + ], + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": false, + "judgeNotes": "prometheus.yml only preserves the app scrape and does not add a Supabase Metrics API scrape. There is no HTTPS /customer/v1/privileged/metrics target, no basic_auth password_file, and docker-compose.yml does not mount a password_file via volume or Compose secret." + }, + { + "name": "documented live deployment and verification steps", + "passed": false, + "judgeNotes": "README includes concrete verification and restart/recreate steps, but it uses an env file for SUPABASE_SECRET_API_KEY rather than requiring/placing a matching secret file, and does not clearly explain creating the Secret API key. Fails the required secret setup." + } + ], + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "scorer evaluated deleted user access", + "passed": false, + "notes": "duplicate key value violates unique constraint \"profiles_pkey\"" + } + ], + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The assistant correctly identified the root cause as public.orders missing from the supabase_realtime publication despite SUBSCRIBED, fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved courier_locations in the publication, and did not weaken RLS/policies or recreate/drop the publication." + } + ], + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "The assistant clearly identified `image-transform` as the affected function and listed the recurring HTTP 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z. It did mention older billing-webhook 503s but only as supporting context, not as the main issue." + }, + { + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": false, + "judgeNotes": "The answer does attribute 503s to Edge Function/gateway-side before the handler and cites valid observations (runtime logs only show successful executions; 503s before handler; distinction from avatar-upload 500). However it ultimately frames the root cause as Edge Function startup/boot/import issues and recommends inspecting boot logs, reproducing cold start, and redeploying/fixing function packaging, which blames the function/runtime layer rather than the gateway/platform layer in front of the function." + }, + { + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended multiple concrete next steps: inspecting boot logs for specific function versions, reproducing cold starts with named packages, decoupling upload/transform, redeploying functions, and monitoring the 503 pattern." + } + ], + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/investigate-reliability-003-edge-function-5xx-correlation.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS deny-all due to no policies and added authenticated owner-scoped SELECT and INSERT policies with RLS kept enabled." + } + ], + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "codex-gpt-5.4", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "codex-gpt-5.5", + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 3 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "3 rows" + } + ], + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.5/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "codex-gpt-5.5", + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": false + }, + { + "name": "schema file updated to include description column", + "passed": true }, { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" + "name": "a new migration was generated for the change", + "passed": true }, { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" + "name": "description column exists in the live database", + "passed": true } ], - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "codex-gpt-5.5/build-cli-002-declarative-schema.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ @@ -833,7 +1865,7 @@ "sdk" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "rejects missing auth", @@ -845,7 +1877,7 @@ }, { "name": "reads only with the caller's JWT", - "passed": false + "passed": true }, { "name": "user A cannot force-read user B note", @@ -859,10 +1891,10 @@ "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/build-functions-004-service-role-bypass.json" + "sourcePath": "codex-gpt-5.5/build-functions-004-service-role-bypass.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ @@ -891,7 +1923,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ef9e3-87fb-76d6-85cb-e1a72ec67fca/receipt-alpha.pdf, 019ef9e3-87fb-76d6-85cb-e1a72ec67fca/receipt-beta.pdf" + "notes": "saw: 019ef70b-8014-721e-86ec-fc65ddd960e3/receipt-alpha.pdf, 019ef70b-8014-721e-86ec-fc65ddd960e3/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -912,16 +1944,16 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Configured a private user-files bucket, authenticated owner-scoped SELECT and INSERT storage.objects policies using storage.foldername(name)[1] = auth.uid()::text, kept RLS intact, and provided supabase-js createSignedUrl code with an expiry for temporary sharing links." + "judgeNotes": "Meets rubric: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with RLS kept enabled, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/build-storage-001-private-bucket-access.json" + "sourcePath": "codex-gpt-5.5/build-storage-001-private-bucket-access.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "build-tests-001-rls-tenant-isolation", "stage": "build", "product": [ @@ -938,26 +1970,26 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.sql" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "7 passed, 6 failed" + "notes": "8 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy: authenticated users with any membership can read posts from organizations they do not belong to because the policy lacks `m.org_id = posts.org_id`. It grounds this in pgTAP failures for cross-tenant post reads and explicitly contrasts `notes` as correctly isolated. It also notes an additional `memberships` issue, but does not blame `notes` or dismiss the tests." + "judgeNotes": "The agent correctly identifies `posts` as the table with the broken tenant isolation policy, explains that authenticated members could read posts from organizations they do not belong to because membership was not tied to `posts.org_id`, and grounds this in pgTAP results showing `posts` failed cross-tenant assertions while `notes` passed." } ], "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "codex-gpt-5.5/build-tests-001-rls-tenant-isolation.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "build-vectors-001-rag-with-permissions", "stage": "build", "product": [ @@ -980,10 +2012,10 @@ "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/build-vectors-001-rag-with-permissions.json" + "sourcePath": "codex-gpt-5.5/build-vectors-001-rag-with-permissions.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "deploy-database-001-prometheus-metrics", "stage": "deploy", "product": [ @@ -1002,21 +2034,21 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets requirements: preserves app job, adds HTTPS Supabase metrics scrape at /customer/v1/privileged/metrics for .supabase.co:443, uses basic_auth with password_file, and docker-compose mounts the password file read-only." + "judgeNotes": "Meets all rubric requirements: HTTPS Supabase Metrics API scrape, correct path, Basic Auth with password_file, project target substitution, app job preserved, and Compose secret wiring mounts the password file at /run/secrets/supabase_metrics_secret_key." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file path, reload/start steps, and concrete verification via curl and Prometheus targets." + "judgeNotes": "README includes Secret API key creation/use, matching secret file/mount path, Compose deployment/start steps, and concrete verification via Prometheus target UP and Grafana dashboard." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/deploy-database-001-prometheus-metrics.json" + "sourcePath": "codex-gpt-5.5/deploy-database-001-prometheus-metrics.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "deploy-functions-001-edge-function-secrets", "stage": "deploy", "product": [ @@ -1041,7 +2073,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -1051,48 +2083,10 @@ "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "claude-code-sonnet-4.6", - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "codex-gpt-5.5/deploy-functions-001-edge-function-secrets.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "investigate-auth-001-deleted-user-access", "stage": "investigate", "product": [ @@ -1103,11 +2097,12 @@ "sdk" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "victim session active before delete-account", - "passed": true + "passed": false, + "notes": "new row violates row-level security policy for table \"notes\"" }, { "name": "delete_account flow ran for the victim", @@ -1124,25 +2119,27 @@ }, { "name": "deleted user cannot sign back in", - "passed": true + "passed": false, + "notes": "deleted account can still sign in" }, { "name": "other users keep their sessions and access", - "passed": true + "passed": false, + "notes": "new row violates row-level security policy for table \"notes\"" }, { "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements auth user/session removal, explains refresh revocation plus remaining stateless JWT expiry window, recommends stronger server-side/session checks for zero tolerance, and correctly distinguishes frontend publishable/anon keys from server-only secret/service_role keys that bypass RLS." + "passed": false, + "judgeNotes": "Diagnosed the soft-delete/session issue and fixed revocation via sessions/refresh tokens plus banning, and clarified publishable vs secret keys. However, it did not correctly explain the remaining stateless JWT access-token window with the required guidance to use auth.getUser() or short JWT expiry rather than only local JWT validation/getClaims()." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/investigate-auth-001-deleted-user-access.json" + "sourcePath": "codex-gpt-5.5/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "investigate-realtime-001-subscribed-no-events", "stage": "investigate", "product": [ @@ -1179,16 +2176,16 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identified the missing orders table in the supabase_realtime publication as the root cause, explained SUBSCRIBED vs no INSERT events, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified both tables remain in the publication, and did not weaken RLS/policies or disrupt courier_locations." + "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite the channel subscribing, added only public.orders via ALTER PUBLICATION, verified courier_locations remained, and did not weaken RLS/policies or blame client/RLS/networking." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/investigate-realtime-001-subscribed-no-events.json" + "sourcePath": "codex-gpt-5.5/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "investigate-reliability-003-edge-function-5xx-correlation", "stage": "investigate", "product": [ @@ -1203,26 +2200,26 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant correctly identified `image-transform` as the affected function and described the recurring HTTP 503 gateway failures throughout the morning of 2026-04-28, covering the pattern across 07:00Z-12:00Z. It also distinguished the older `billing-webhook` 503s as a separate incident." + "judgeNotes": "The assistant correctly identified image-transform as the affected function and described the recurring HTTP 503 pattern throughout the morning of 2026-04-28, listing all 8 gateway failures from 07:00Z to 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "The assistant attributes the recurring image-transform 503s to the gateway/platform layer, explicitly saying they only appear in API gateway logs and have no corresponding internal execution failures / the function never ran. It also distinguishes these gateway 503s from avatar-upload's function-level 500. Although it adds a specific cold-start explanation, the required layer attribution is present and grounded in valid log observations." + "judgeNotes": "Attributes recurring image-transform 503s to the API/gateway/invocation layer before the function handler, not function code. Grounds this in valid observations: gateway logs show 503s while nearby Edge Function executions return 200 with normal latency and no matching runtime errors; also distinguishes these from a separate avatar-upload 500 that reached the function runtime." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: adding retry/keep-warm cron, reviewing function code and correlated payload details, testing billing-webhook health, checking environment variables, and reviewing deployment/config changes." + "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket with project/region and timestamps, adding retries, structured request logging, and separately investigating the avatar-upload 500." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/investigate-reliability-003-edge-function-5xx-correlation.json" + "sourcePath": "codex-gpt-5.5/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", "product": [ @@ -1264,16 +2261,16 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies and created authenticated SELECT and INSERT owner-scoped policies using user_id = auth.uid(), without disabling RLS." + "judgeNotes": "Diagnosed RLS enabled with no policies as the deny-all cause, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/resolve-dataapi-001-empty-results.json" + "sourcePath": "codex-gpt-5.5/resolve-dataapi-001-empty-results.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "resolve-performance-001-slow-query-cpu-spike", "stage": "resolve", "product": [ @@ -1301,7 +2298,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -1311,10 +2308,10 @@ "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/resolve-performance-001-slow-query-cpu-spike.json" + "sourcePath": "codex-gpt-5.5/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "claude-code-sonnet-4.6", + "experiment": "codex-gpt-5.5", "eval": "resolve-security-002-rls-cross-tenant-leak", "stage": "resolve", "product": [ @@ -1368,7 +2365,7 @@ "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-4.6/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.5/resolve-security-002-rls-cross-tenant-leak.json" }, { "experiment": "openai-gpt-5.4-mini", @@ -1461,45 +2458,6 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/build-cli-002-declarative-schema.json" }, - { - "experiment": "openai-gpt-5.4-mini", - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" - } - ], - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "openai-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" - }, { "experiment": "openai-gpt-5.4-mini", "eval": "build-functions-004-service-role-bypass", @@ -1573,7 +2531,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ef9e1-8e38-76a9-bb5d-ebd7fd404701/receipt-alpha.pdf, 019ef9e1-8e38-76a9-bb5d-ebd7fd404701/receipt-beta.pdf" + "notes": "saw: 019ef70a-0b1f-7479-aafb-8b6d1ba923a3/receipt-alpha.pdf, 019ef70a-0b1f-7479-aafb-8b6d1ba923a3/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -1594,7 +2552,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Configured a private user-files bucket, owner-scoped authenticated SELECT and INSERT policies (plus update/delete) using the user ID path prefix, did not disable RLS, and provided supabase-js createSignedUrl code with an expiry." + "judgeNotes": "The answer creates a private user-files bucket, enables RLS on storage.objects, defines authenticated owner-scoped SELECT and INSERT policies using the first path segment equals auth.uid(), and includes supabase-js createSignedUrl with an expiry for temporary sharing. It does not make the bucket public, use permissive policies, or expose service role usage." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -1615,22 +2573,22 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.sql" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { "name": "pgTAP isolation tests ran and pass", - "passed": false, - "notes": "no test summary found; exit 1; output: Connecting to local database...\n3.36: Pulling from supabase/pg_prove\ndcccee43ad5d: Pulling fs layer\n06d62d0de6d7: Pulling fs layer\na22cb17b3b93: Pulling fs layer\n4f4fb700ef54: Pulling fs layer\n4f4fb700ef54: Waiting\na22cb17b3b93: Verifying Checksum\na22cb17b3b93: Download complete\ndcccee43ad5d: Verifying Checksum\ndcccee43ad5d: Download complete\n4f4fb700ef54: Verifying Checksum\n4f4fb700ef54: Download complete\n06d62d0de6d7: Verifying Checksum\n06d62d0de6d7: Download complete\ndcccee43ad5d: Pull comple" + "passed": true, + "notes": "4 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identified `posts` as the table with the tenant isolation flaw, explaining that its RLS checked only `user_id` membership and not matching `org_id`, allowing cross-org reads. It did not blame `notes` or dismiss the test signal." + "judgeNotes": "The agent correctly identified `posts` as the table with the tenant isolation flaw, specifically that its RLS policy checked only user membership and not `org_id`, allowing cross-org post reads. It wrote and ran pgTAP tests and treated the results as the signal, then fixed the posts policy." } ], "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", @@ -1684,12 +2642,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: Supabase scrape uses basic_auth.password with an environment-injected Secret API key instead of basic_auth.password_file, and docker-compose.yml does not mount/provide a password file via volume or Compose secret." + "judgeNotes": "Supabase scrape preserves the app job and uses HTTPS with the correct metrics path and project target, but it uses basic_auth.password with env substitution instead of basic_auth.password_file. docker-compose.yml also does not mount or provide a password_file via volume or Compose secret, so secret wiring is missing." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README includes Secret API key creation and Prometheus target verification, but it does not instruct placing a matching secret file and does not specifically restart/reload the Compose stack. It also uses environment variables rather than the required secret-file setup." + "passed": true, + "judgeNotes": "README includes Secret API key creation, putting it in observability/.env used by Compose, restarting/recreating Prometheus, and verifying the supabase job via Prometheus /targets." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -1735,44 +2693,6 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/deploy-functions-001-edge-function-secrets.json" }, - { - "experiment": "openai-gpt-5.4-mini", - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "openai-gpt-5.4-mini/deploy-self-hosting-001-docker-compose.json" - }, { "experiment": "openai-gpt-5.4-mini", "eval": "investigate-auth-001-deleted-user-access", @@ -1789,7 +2709,8 @@ "checks": [ { "name": "victim session active before delete-account", - "passed": true + "passed": false, + "notes": "permission denied for table users" }, { "name": "delete_account flow ran for the victim", @@ -1806,17 +2727,17 @@ }, { "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "passed": true }, { "name": "other users keep their sessions and access", - "passed": true + "passed": false, + "notes": "permission denied for table users" }, { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The answer correctly identifies the original soft-delete issue and gives a mostly correct frontend/backend key rundown, but it does not actually fix the flow by deleting the auth user or removing their identity; it only marks the profile deleted and deletes sessions, leaving the user able to sign in again. It also fails to explain the stateless JWT access-token expiry window and the need for server-side auth.getUser() or short JWT expiry instead of relying only on local JWT validation/getClaims()." + "judgeNotes": "Fails because the answer does not correctly explain that access tokens are stateless JWTs that remain valid until expiry after session/user revocation, nor does it mention using auth.getUser() or short JWT expiry instead of local JWT validation/getClaims(). It instead says there is effectively no window for protected data. Other required parts—identifying soft delete, revoking/deleting auth records, and publishable vs secret key clarification—are mostly present." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", @@ -1862,7 +2783,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Diagnosed missing orders table from supabase_realtime publication and fixed by adding public.orders, while preserving existing courier_locations feed and not altering RLS/policies." + "judgeNotes": "The assistant correctly identified the missing orders table in the supabase_realtime publication as the cause, fixed it with ALTER PUBLICATION ... ADD TABLE public.orders, and did not weaken RLS/policies or disturb courier_locations." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -1886,17 +2807,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described recurring API gateway 503s throughout the morning at :00/:30, with subsequent 200s and no function execution logs for the 503s." + "judgeNotes": "The assistant identified image-transform as the affected function and listed the recurring 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "The response correctly attributes the 503s to the gateway/platform layer and grounds this in missing function execution logs plus nearby successes. However, it recommends redeploying `image-transform` as a next step, which the rubric explicitly lists as a fail condition." + "judgeNotes": "The assistant did not clearly attribute the recurring image-transform 503s to the gateway/Edge Functions platform layer in front of the function. It instead framed them as likely boot/startup/runtime/dependency-layer failures, suggested redeploying/fixing dependencies, and did not ground the conclusion in valid observations such as gateway-only 503s with no corresponding invocation/runtime rows, unchanged deployment_id, or distinction from function-level 500s." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: checking for a platform incident, redeploying the Edge Function, adding retry/backoff, and inspecting downstream dependencies." + "judgeNotes": "The assistant recommended concrete next steps: inspect specific Edge Function logs for boot errors/exceptions, redeploy affected functions, pin/fix dependency versions if import issues appear, and open a Supabase support ticket with timestamps if platform boot errors persist." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -1947,7 +2868,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "Diagnosed RLS deny-all due to no policies and added authenticated SELECT and INSERT owner-scoped policies using auth.uid(), without disabling RLS." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -1984,7 +2905,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at_desc (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -2081,27 +3002,27 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-98a38a0f\nTry rerunning the command with --debug to troubleshoot the error.\n" + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "row level security is enabled on todos", "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-98a38a0f\nTry rerunning the command with --debug to troubleshoot the error.\n" + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "a SELECT policy targets the authenticated role", "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-98a38a0f\nTry rerunning the command with --debug to troubleshoot the error.\n" + "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "REST API returns no todos to anonymous requests", "passed": false, - "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-98a38a0f\nTry rerunning the command with --debug to troubleshoot the error.\n" + "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" }, { "name": "REST API returns the todos to authenticated requests", "passed": false, - "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-98a38a0f\nTry rerunning the command with --debug to troubleshoot the error.\n" + "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" } ], "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", @@ -2130,7 +3051,8 @@ }, { "name": "schema file updated to include description column", - "passed": true + "passed": false, + "notes": "description not found in any schema file" }, { "name": "a new migration was generated for the change", @@ -2146,45 +3068,6 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/build-cli-002-declarative-schema.json" }, - { - "experiment": "openai-gpt-5.4-nano", - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": false, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 10 -> 11" - }, - { - "name": "process-tasks function drains the queue", - "passed": false, - "notes": "function returned 200 but message 12 is still queued, so it was read but never removed" - } - ], - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "openai-gpt-5.4-nano/build-cli-003-pg-cron-queue-workflow.json" - }, { "experiment": "openai-gpt-5.4-nano", "eval": "build-functions-004-service-role-bypass", @@ -2258,7 +3141,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ef9e1-a72f-70df-891d-8de1c5e9b7a3/receipt-alpha.pdf, 019ef9e1-a72f-70df-891d-8de1c5e9b7a3/receipt-beta.pdf" + "notes": "saw: 019ef70a-16c4-70fc-bd5e-29aa1a708d2b/receipt-alpha.pdf, 019ef70a-16c4-70fc-bd5e-29aa1a708d2b/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -2279,7 +3162,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Created a private user-files bucket, enabled RLS, added owner-scoped SELECT and INSERT policies using the user-id path prefix, and provided supabase-js createSignedUrl code with an expiry for temporary sharing links." + "judgeNotes": "Configured a private user-files bucket, kept RLS enabled, added authenticated owner-scoped SELECT and INSERT policies using the user-id path prefix, and provided supabase-js createSignedUrl code with an expiry for temporary sharing." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -2287,6 +3170,42 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/build-storage-001-private-bucket-access.json" }, + { + "experiment": "openai-gpt-5.4-nano", + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": false, + "notes": "no .sql files found under supabase/tests/" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": false, + "notes": "no test summary found; exit 0; output: Connecting to local database...\n3.36: Pulling from supabase/pg_prove\ndcccee43ad5d: Pulling fs layer\n06d62d0de6d7: Pulling fs layer\na22cb17b3b93: Pulling fs layer\n4f4fb700ef54: Pulling fs layer\n4f4fb700ef54: Waiting\na22cb17b3b93: Verifying Checksum\na22cb17b3b93: Download complete\ndcccee43ad5d: Verifying Checksum\ndcccee43ad5d: Download complete\n06d62d0de6d7: Verifying Checksum\n06d62d0de6d7: Download complete\n4f4fb700ef54: Verifying Checksum\n4f4fb700ef54: Download complete\ndcccee43ad5d: Pull comple" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies posts as the table with broken tenant isolation, explains that authenticated members can read posts from organizations they are not members of, and grounds the conclusion in the test results showing notes pass while posts fail. It does not blame notes or dismiss the tests." + } + ], + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "openai-gpt-5.4-nano/build-tests-001-rls-tenant-isolation.json" + }, { "experiment": "openai-gpt-5.4-nano", "eval": "build-vectors-001-rag-with-permissions", @@ -2333,12 +3252,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Supabase scrape is not deployable: it uses placeholder host/port, http scheme, /metrics path, no Basic Auth password_file, and docker-compose.yml does not mount or provide the password_file. The existing app scrape is preserved." + "judgeNotes": "Fails because prometheus.yml uses basic_auth.password instead of password_file, and docker-compose.yml does not mount/provide that password file via a volume or Compose secret. The app scrape is preserved and the HTTPS metrics path/target shape are present, but the required secret wiring is missing." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README.md does not explain how to make the Supabase integration live. It lacks steps to create a Secret API key, place the matching secret file, restart/reload the Compose stack, and verify via Prometheus targets/PromQL/Grafana." + "judgeNotes": "README includes creating a Secret API key, endpoint, restart, and a curl verification step, but it does not require placing the key in a matching secret file. Instead it instructs editing prometheus.yml with the secret inline, which fails the required secret-file setup." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -2372,7 +3291,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get('WEATHER_API_KEY')." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -2384,46 +3303,6 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/deploy-functions-001-edge-function-secrets.json" }, - { - "experiment": "openai-gpt-5.4-nano", - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": false, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": false, - "notes": "still default or empty: POSTGRES_PASSWORD, JWT_SECRET, DASHBOARD_PASSWORD, VAULT_ENC_KEY, PG_META_CRYPTO_KEY" - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": false, - "notes": "JWT_SECRET missing" - } - ], - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "openai-gpt-5.4-nano/deploy-self-hosting-001-docker-compose.json" - }, { "experiment": "openai-gpt-5.4-nano", "eval": "investigate-auth-001-deleted-user-access", @@ -2440,7 +3319,8 @@ "checks": [ { "name": "victim session active before delete-account", - "passed": true + "passed": false, + "notes": "permission denied for table users" }, { "name": "delete_account flow ran for the victim", @@ -2463,12 +3343,13 @@ }, { "name": "other users keep their sessions and access", - "passed": true + "passed": false, + "notes": "permission denied for table users" }, { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The answer correctly identifies the soft-delete/RLS issue and gives a mostly correct publishable-vs-secret key rundown, but it does not fix the delete flow by deleting/removing the Auth user or revoking sessions/refresh tokens. It instead relies on RLS changes and client sign-out. It also incorrectly says there is effectively no access window, rather than explaining that stateless JWT access tokens can remain valid until expiry and that sensitive server checks should use auth.getUser() or short JWT expiry instead of only local JWT validation." + "judgeNotes": "The answer correctly identifies the original soft-delete/RLS issue and correctly explains publishable vs secret keys. However, it does not implement real Auth deletion/session and refresh-token revocation; it only updates auth.users.deleted_at. It also fails to explain that stateless JWT access tokens can remain valid until expiry and that server-side checks should use auth.getUser() or short JWT expiry rather than only local JWT validation/getClaims(). It incorrectly implies the next DB request will be denied immediately as the full token-window answer." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", @@ -2509,12 +3390,12 @@ { "name": "staff can still read orders through RLS", "passed": true, - "notes": "authenticated sees 3 of 3 orders" + "notes": "authenticated sees 2 of 2 orders" }, { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite the channel reaching SUBSCRIBED, fixed it with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserved the existing courier_locations feed, policies, and RLS." + "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite the channel subscribing successfully, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not change RLS, policies, client code, or disrupt courier_locations." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -2538,17 +3419,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described intermittent/recurring 503s throughout the morning of 2026-04-28 with multiple times spanning the reported window. It slightly included a ~06:30 example outside the rubric window and did not explicitly say all 8 gateway failures, but it clearly captured the required pattern." + "judgeNotes": "The assistant correctly identified image-transform as the affected function and described recurring/intermittent HTTP 503 gateway/API responses during the 2026-04-28 morning period, while distinguishing them from function-runtime successes and unrelated errors." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "The assistant explicitly attributes the 503s to the Edge Function image-transform code/runtime path and recommends investigating/hardening the function/module. It does not attribute them to the gateway/platform layer, despite mentioning intermittent gateway-like 503s. It also treats the avatar-upload 500 as related broader pipeline evidence rather than distinguishing it from gateway 503s." + "judgeNotes": "The answer correctly attributes the recurring 503s to the gateway/API layer and grounds that in valid observations, especially gateway 503s with only successful image-transform function runtime logs. However, it also recommends temporarily redeploying image-transform, which the rubric explicitly lists as a fail condition when presented as remediation/next step." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: investigate the image-transform Edge Function/module internals, add retries/backoff and detailed error logging, isolate the dependency, and verify whether 503s are still occurring." + "judgeNotes": "The assistant recommended concrete next steps, including checking deployment/traffic spikes, redeploying the function, collecting request IDs from failing 503 responses, and correlating them with logs." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -2599,7 +3480,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing Data API empty results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -2627,7 +3508,7 @@ }, { "name": "ran EXPLAIN on the expensive query", - "passed": true + "passed": false }, { "name": "created index covering user_id and created_at", @@ -2636,7 +3517,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", From fd35822d23e682e8e2522c8228defac109cfbca1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 09:55:09 +0000 Subject: [PATCH 3/5] chore: refresh eval results --- apps/web/src/data/eval-results.json | 355 ++++++++++++++++------------ 1 file changed, 201 insertions(+), 154 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 81a55e81..280c6973 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -26,7 +26,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 5 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -44,7 +44,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "5 rows" + "notes": "3 rows" } ], "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", @@ -114,22 +114,22 @@ { "name": "user A reads own note", "passed": false, - "notes": "edge function import not supported: https://deno.land/x/jwt@v0.1.1/mod.ts" + "notes": "edge function import not supported: https://deno.land/x/djwt@v3.0.1/mod.ts" }, { "name": "reads only with the caller's JWT", "passed": false, - "notes": "edge function import not supported: https://deno.land/x/jwt@v0.1.1/mod.ts" + "notes": "edge function import not supported: https://deno.land/x/djwt@v3.0.1/mod.ts" }, { "name": "user A cannot force-read user B note", "passed": false, - "notes": "edge function import not supported: https://deno.land/x/jwt@v0.1.1/mod.ts" + "notes": "edge function import not supported: https://deno.land/x/djwt@v3.0.1/mod.ts" }, { "name": "user B cannot force-read user A note", "passed": false, - "notes": "edge function import not supported: https://deno.land/x/jwt@v0.1.1/mod.ts" + "notes": "edge function import not supported: https://deno.land/x/djwt@v3.0.1/mod.ts" } ], "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", @@ -150,12 +150,45 @@ "sdk" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "bucket user-files exists", - "passed": false, - "notes": "no row in storage.buckets with id or name 'user-files'" + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019ef906-8253-766b-9c80-c064d32613ea/receipt-alpha.pdf, 019ef906-8253-766b-9c80-c064d32613ea/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Creates a private user-files bucket, keeps/enables RLS on storage.objects, adds authenticated INSERT and SELECT policies scoped to the bucket and file owner/user folder, and uses createSignedUrl with an expiry for temporary sharing." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -180,18 +213,18 @@ "checks": [ { "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "2 file(s): supabase/tests/tenant_isolation.sql, supabase/tests/tenant_isolation_with_jwt.sql" + "passed": false, + "notes": "no .sql files found under supabase/tests/" }, { "name": "pgTAP isolation tests ran and pass", "passed": false, - "notes": "no test summary found; exit 1; output: Connecting to local database...\n3.36: Pulling from supabase/pg_prove\ndcccee43ad5d: Pulling fs layer\n06d62d0de6d7: Pulling fs layer\na22cb17b3b93: Pulling fs layer\n4f4fb700ef54: Pulling fs layer\n4f4fb700ef54: Waiting\na22cb17b3b93: Verifying Checksum\na22cb17b3b93: Download complete\ndcccee43ad5d: Verifying Checksum\ndcccee43ad5d: Download complete\n4f4fb700ef54: Verifying Checksum\n4f4fb700ef54: Download complete\n06d62d0de6d7: Verifying Checksum\n06d62d0de6d7: Download complete\ndcccee43ad5d: Pull comple" + "notes": "no test summary found; exit 0; output: Connecting to local database...\n3.36: Pulling from supabase/pg_prove\ndcccee43ad5d: Pulling fs layer\n06d62d0de6d7: Pulling fs layer\na22cb17b3b93: Pulling fs layer\n4f4fb700ef54: Pulling fs layer\n4f4fb700ef54: Waiting\na22cb17b3b93: Download complete\n4f4fb700ef54: Verifying Checksum\n4f4fb700ef54: Download complete\ndcccee43ad5d: Verifying Checksum\ndcccee43ad5d: Download complete\n06d62d0de6d7: Verifying Checksum\n06d62d0de6d7: Download complete\ndcccee43ad5d: Pull complete\n06d62d0de6d7: Pull complete\na2" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies the `posts` table SELECT policy as broken tenant isolation, specifically that authenticated members of any organization can read posts from other organizations due to a missing `m.org_id = posts.org_id` check. It grounds this in its test results showing cross-org post reads. Although it also incorrectly claims notes failures and adds extra issues, it does not blame notes instead of posts and does draw the required conclusion about posts." + "judgeNotes": "The agent correctly identifies `posts` as the vulnerable table, explains that authenticated members can read posts from organizations they are not members of due to a missing `org_id` check, and grounds the conclusion in test results showing cross-org visibility. It also correctly treats `notes` as secure rather than blaming it." } ], "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", @@ -245,12 +278,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: Supabase scrape uses basic_auth.password with an environment variable instead of basic_auth.password_file, and docker-compose.yml does not mount/provide that password_file via a volume or Compose secret." + "judgeNotes": "prometheus.yml only preserves the app scrape; it does not add a Supabase Metrics API scrape target with HTTPS /customer/v1/privileged/metrics, Basic Auth password_file, or project-ref.supabase.co/red target. docker-compose.yml also does not mount or define the required password_file/secret." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README is too vague: it does not give concrete steps for creating the Secret API key, does not specify the exact required secret/env file contents or variable names, and only says to start the stack rather than restart/reload an existing Compose stack. Verification via Prometheus targets is present, but setup instructions are incomplete." + "judgeNotes": "README is too vague and does not include required steps to create a Secret API key, place the matching secret file, restart/reload the Compose stack, or concretely verify via Prometheus targets/PromQL/Grafana. It also references setup elsewhere rather than documenting the required live/verification workflow." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -284,7 +317,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -308,7 +341,7 @@ "sdk" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "victim session active before delete-account", @@ -329,8 +362,7 @@ }, { "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "passed": true }, { "name": "other users keep their sessions and access", @@ -338,8 +370,8 @@ }, { "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The answer identifies soft deletion and refresh/session revocation issues, and explains publishable vs secret keys correctly. However, the implemented fix does not delete the auth user or fully/equivalently remove identities and refresh tokens; it only deletes rows from auth.sessions and relies on RLS soft-delete checks. It also does not clearly recommend auth.getUser()/short JWT expiry for server-side checks vs local getClaims(), and incorrectly suggests the remaining risk is only client-side caching/backend RLS blocks rather than explaining stateless JWTs remain valid until expiry for endpoints relying on local validation." + "passed": true, + "judgeNotes": "Meets the main rubric: identifies soft-delete without auth/session revocation, changes flow to delete auth user/revoke refresh sessions, explains stateless JWT access-token window and short expiry, and correctly says publishable keys are frontend-safe while secret/service keys are server-only and bypass RLS. Minor omission: it does not explicitly mention auth.getUser() vs getClaims(), but it gives equivalent mitigation context via short JWT expiry/session validation." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", @@ -359,11 +391,11 @@ "sdk" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "orders table added to supabase_realtime publication", - "passed": false + "passed": true }, { "name": "courier_locations still in supabase_realtime publication", @@ -384,8 +416,8 @@ }, { "name": "diagnosed missing publication membership", - "passed": false, - "judgeNotes": "The assistant incorrectly diagnosed the root cause as a missing RLS SELECT policy and proposed adding/weakening policies. It did not identify or fix the actual issue: the orders table missing from the supabase_realtime publication via ALTER PUBLICATION supabase_realtime ADD TABLE orders." + "passed": true, + "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication, added only public.orders via ALTER PUBLICATION, preserved courier_locations, and did not weaken RLS/policies or blame client/RLS/networking." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -404,22 +436,22 @@ "observability" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant named image-transform as the affected function and described recurring 503 gateway errors across the morning of 2026-04-28, roughly covering the 07:00Z-12:00Z pattern. It did not incorrectly focus on billing-webhook." + "judgeNotes": "Identified image-transform as the affected function and described the recurring HTTP 503 pattern throughout the morning of 2026-04-28, including all 8 gateway failures from 07:00Z to 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes image-transform 503s to gateway/load balancer layer, grounded in gateway logs showing 503s while Edge Function logs show successful 200 executions with normal durations. Does not blame function application code." + "passed": false, + "judgeNotes": "The assistant attributes the 503s to the image-transform/avatar-upload function dependencies, resource exhaustion, cold starts, or unhandled code errors, and recommends package/code fixes. It does not ground the cause in gateway/platform-layer evidence or distinguish gateway 503s from function-level errors." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended multiple concrete next steps, including checking Edge Function concurrency limits, gateway timeout settings, PostgreSQL connection pool usage, infrastructure changes, and function metrics/logs." + "judgeNotes": "The assistant provided concrete next steps, including checking package versions/known issues, testing under load, adding retries, error logging, timeouts, and error boundaries." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -470,7 +502,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed deny-all RLS with no policies, kept RLS enabled, and created owner-scoped SELECT and INSERT policies using auth.uid() = user_id / WITH CHECK. Extra UPDATE/DELETE owner policies do not violate the rubric." + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results. Created owner-scoped SELECT and INSERT policies for users via auth.uid() = user_id / WITH CHECK, and kept RLS enabled. Extra UPDATE/DELETE policies do not violate the rubric." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -603,7 +635,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 4 rows" }, { "name": "row level security is enabled on todos", @@ -621,7 +653,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "4 rows" } ], "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", @@ -650,8 +682,7 @@ }, { "name": "schema file updated to include description column", - "passed": false, - "notes": "description not found in any schema file" + "passed": true }, { "name": "a new migration was generated for the change", @@ -740,7 +771,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ef70b-b907-73e9-baaf-120cbf095a41/receipt-alpha.pdf, 019ef70b-b907-73e9-baaf-120cbf095a41/receipt-beta.pdf" + "notes": "saw: 019ef907-40fa-71f4-8c84-4464e51ce335/receipt-alpha.pdf, 019ef907-40fa-71f4-8c84-4464e51ce335/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -761,7 +792,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "The answer creates a private user-files bucket, defines authenticated owner-scoped SELECT and INSERT policies on storage.objects using the first path segment equals auth.uid(), keeps RLS enabled implicitly without disabling it, and provides supabase-js createSignedUrl code with an expiry. It does not make the bucket public, use permissive policies, anon/public roles, getPublicUrl, or service role client-side." + "judgeNotes": "Meets requirements: creates private user-files bucket, owner-scoped SELECT and INSERT storage.objects policies for authenticated role with WITH CHECK on INSERT, does not disable RLS, and uses createSignedUrl with an expiry for temporary sharing." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -792,12 +823,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "9 passed, 6 failed" + "notes": "9 passed, 8 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies posts (not notes) as having the tenant isolation flaw: the SELECT policy lacks an org_id membership check, allowing authenticated members to read posts from other organizations. Grounds the conclusion in the pgTAP failures for posts isolation tests." + "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, specifically that any authenticated member of any org can read posts from all orgs due to the missing `m.org_id = posts.org_id` condition. It grounds this in pgTAP test failures and distinguishes `notes` as correctly isolated." } ], "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", @@ -842,7 +873,7 @@ "observability" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "preserved existing app scrape job", @@ -850,13 +881,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Fails because the Supabase scrape uses inline basic_auth.password instead of password_file, and docker-compose.yml does not mount or define the password file via a volume or Compose secret. The app scrape is preserved and endpoint/path/scheme are otherwise correct." + "passed": true, + "judgeNotes": "Meets requirements: app scrape preserved; Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, project-ref supabase.co target placeholder, and docker-compose wires the password file via a Compose secret mounted at the matching path." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README includes creating a Secret API key, restarting the Compose stack, and verification via curl/Prometheus targets/Grafana. However, the required secret-file setup is only optional and not part of the live configuration; prometheus.yml uses an inline password placeholder and docker-compose.yml does not mount the secret file, so the required matching secret file placement/setup is missing/mismatched." + "passed": true, + "judgeNotes": "README includes Secret API key creation, matching Docker Compose secret file placement, project ref configuration, stack restart, and concrete verification via curl plus Prometheus targets." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -914,7 +945,7 @@ "sdk" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "victim session active before delete-account", @@ -943,8 +974,8 @@ }, { "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements deletion of auth.sessions and auth.users, explains refresh/access token behavior and the JWT validity window, adds/mentions server-side session validation via auth.sessions, and correctly distinguishes publishable/frontend keys from secret/server-only keys that bypass RLS. It does overclaim that RLS session checks make the window zero for protected tables, but it still correctly notes stateless JWTs can remain valid until expiry and warns about non-RLS endpoints." + "passed": false, + "judgeNotes": "Covers the soft-delete root cause, deletes auth.users to revoke sessions/refresh tokens, and correctly distinguishes publishable vs secret keys. However, it does not clarify that server-side authorization must use auth.getUser() or short JWT expiry rather than relying only on local JWT validation/getClaims(), which is a required rubric point for the remaining access-token window." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", @@ -990,7 +1021,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified that the channel can be SUBSCRIBED while orders INSERT events are not delivered because public.orders was missing from the supabase_realtime publication. It fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified the publication contents, and did not disable RLS, weaken policies, recreate the publication, or disturb courier_locations." + "judgeNotes": "The assistant correctly identified the missing orders table in the supabase_realtime publication as the root cause, added public.orders to the existing publication, verified both tables are present, and did not blame or modify RLS/policies/client code." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -1014,17 +1045,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant correctly identified image-transform as the affected function and described the recurring HTTP 503 gateway failures throughout the morning of 2026-04-28, covering all 8 failures from 07:00Z to 12:00Z. It did not misattribute the main issue to billing-webhook." + "judgeNotes": "Assistant identified image-transform as the primary affected function and described recurring gateway HTTP 503s throughout the morning of 2026-04-28, including the spread from 07:00Z through 12:00Z. It did not incorrectly focus on billing-webhook as the main issue." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "The assistant identifies the 503s as gateway-level but attributes them to Edge Function cold starts and recommends keeping/redeploying-style function-level mitigations. The rubric requires attribution to the gateway/platform layer in front of the function, not the function/runtime. Cold starts blame the function runtime/platform interaction rather than clearly the gateway/platform layer, so this fails." + "judgeNotes": "The assistant correctly notes that image-transform 503s appear only in gateway/API logs with no function execution logs, but then attributes the cause to the function worker/code crashing, unhandled exceptions, and a possible npm package leak, and recommends fixing the function code. This violates the rubric's requirement to attribute the recurring 503s to the gateway/platform layer rather than the function application code." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: setting up a Cron keep-warm job, adding retry logic, pulling specific avatar-upload edge function logs around a time window, and separately investigating billing-webhook 503s." + "judgeNotes": "The assistant recommended concrete next steps: add error handling to specific Edge Functions, investigate the image-transform npm package for resource leaks, add retry logic, and check a correlated billing-webhook issue. These are actionable and go beyond vague log-checking." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -1075,7 +1106,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all/zero Data API rows, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with USING and WITH CHECK." + "judgeNotes": "Diagnosed RLS deny-all due to enabled RLS with no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -1344,7 +1375,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ef70b-2717-7204-8571-aec531c29178/receipt-alpha.pdf, 019ef70b-2717-7204-8571-aec531c29178/receipt-beta.pdf" + "notes": "saw: 019ef907-5d0a-717d-9f9f-41e594fc5b09/receipt-alpha.pdf, 019ef907-5d0a-717d-9f9f-41e594fc5b09/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -1365,7 +1396,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Configures a private user-files bucket, owner-scoped authenticated SELECT and INSERT policies on storage.objects with RLS left enabled, and uses createSignedUrl with an expiry for temporary sharing. No public bucket/getPublicUrl/service-role client-side issues." + "judgeNotes": "The answer creates a private user-files bucket (public=false), keeps storage.objects RLS enabled, adds authenticated SELECT and INSERT policies scoped to bucket_id and the first folder segment matching auth.uid() via WITH CHECK for insert, and provides supabase-js createSignedUrl with a 15-minute expiry. It does not use public buckets, getPublicUrl, permissive policies, anon access, disabled RLS, or client-side service role keys." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -1391,17 +1422,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_rls_test.sql" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "7 passed, 0 failed" + "notes": "4 passed, 3 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as the tenant isolation flaw, states members of one org could read another org’s posts, distinguishes `notes` as already correct, and grounds the conclusion in reproduced/tested results." + "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation policy where authenticated members of any org can read posts from other orgs, and grounds this in manual and pgTAP test results (`posts` returns 2 rows; cross-org posts visible; failed assertion). It does not blame `notes` and treats the tests as authoritative. Extra finding about `memberships` does not undermine the required conclusion." } ], "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", @@ -1455,12 +1486,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "prometheus.yml only preserves the app scrape and does not add a Supabase Metrics API scrape. There is no HTTPS /customer/v1/privileged/metrics target, no basic_auth password_file, and docker-compose.yml does not mount a password_file via volume or Compose secret." + "judgeNotes": "Supabase scrape uses HTTPS, correct metrics_path, basic_auth with password_file, preserves the app job, and mounts the secrets directory. However, no actual Supabase project target (.supabase.co or .supabase.red) is provided in the submitted config; it relies on an external supabase-targets.yml whose contents are not shown, so the required project target is missing." }, { "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README includes concrete verification and restart/recreate steps, but it uses an env file for SUPABASE_SECRET_API_KEY rather than requiring/placing a matching secret file, and does not clearly explain creating the Secret API key. Fails the required secret setup." + "passed": true, + "judgeNotes": "README explains how to make the Supabase scrape live: obtain/use a Supabase Secret API key, place it in the mounted secret file, configure targets, restart the Compose stack, and verify via Prometheus targets API. Endpoint and auth match the Prometheus configuration and no secret is hardcoded." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -1494,7 +1525,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get('WEATHER_API_KEY')." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -1518,12 +1549,37 @@ "sdk" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { - "name": "scorer evaluated deleted user access", - "passed": false, - "notes": "duplicate key value violates unique constraint \"profiles_pkey\"" + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "The answer identifies the soft-delete-only bug, changes the flow to delete auth.users and thereby sessions/refresh tokens, adds session-aware RLS, explains JWT expiry/stale token caveats including need for session/user checks beyond local signature validation, and correctly distinguishes publishable frontend keys from secret/server-only RLS-bypassing keys." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", @@ -1569,7 +1625,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as public.orders missing from the supabase_realtime publication despite SUBSCRIBED, fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved courier_locations in the publication, and did not weaken RLS/policies or recreate/drop the publication." + "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite subscriptions reaching SUBSCRIBED, added public.orders to the existing publication with ALTER PUBLICATION, preserved courier_locations and RLS/policies, and did not blame or weaken RLS/client/networking." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -1593,17 +1649,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant clearly identified `image-transform` as the affected function and listed the recurring HTTP 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z. It did mention older billing-webhook 503s but only as supporting context, not as the main issue." + "judgeNotes": "The assistant correctly identified `image-transform` as the affected function and explicitly described the recurring HTTP 503 pattern across the morning of 2026-04-28, listing all 8 gateway failures from 07:00Z through 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "The answer does attribute 503s to Edge Function/gateway-side before the handler and cites valid observations (runtime logs only show successful executions; 503s before handler; distinction from avatar-upload 500). However it ultimately frames the root cause as Edge Function startup/boot/import issues and recommends inspecting boot logs, reproducing cold start, and redeploying/fixing function packaging, which blames the function/runtime layer rather than the gateway/platform layer in front of the function." + "judgeNotes": "The response correctly notes gateway 503s with no matching runtime failures and says failures likely occur before user code. However, it also recommends redeploying image-transform/avatar-upload and pinning/fixing function imports as remediation, which the rubric explicitly marks as failing." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended multiple concrete next steps: inspecting boot logs for specific function versions, reproducing cold starts with named packages, decoupling upload/transform, redeploying functions, and monitoring the 503 pattern." + "judgeNotes": "The assistant recommended multiple concrete next steps: redeploying specific functions, classifying 503s as boot_error/internal_failure, opening a Supabase support ticket with timestamps, pinning npm versions, adding logging/error handling, and implementing retry/backoff." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -1654,7 +1710,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies and added authenticated owner-scoped SELECT and INSERT policies with RLS kept enabled." + "judgeNotes": "Diagnosed RLS enabled with no policies, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid() with WITH CHECK for insert. Did not disable RLS or create permissive/public policies." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -1787,7 +1843,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 2 rows" }, { "name": "row level security is enabled on todos", @@ -1805,7 +1861,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "2 rows" } ], "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", @@ -1923,7 +1979,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ef70b-8014-721e-86ec-fc65ddd960e3/receipt-alpha.pdf, 019ef70b-8014-721e-86ec-fc65ddd960e3/receipt-beta.pdf" + "notes": "saw: 019ef906-ed4f-762c-adad-6e53589bf815/receipt-alpha.pdf, 019ef906-ed4f-762c-adad-6e53589bf815/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -1944,7 +2000,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets rubric: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with RLS kept enabled, and supabase-js createSignedUrl with expiry for temporary sharing." + "judgeNotes": "Meets all criteria: private user-files bucket, RLS kept enabled, authenticated SELECT and INSERT policies scoped to bucket and auth.uid() path prefix, no permissive/public policies, and supabase-js uses createSignedUrl with an expiry for temporary sharing." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -1970,17 +2026,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "8 passed, 0 failed" + "notes": "10 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with the broken tenant isolation policy, explains that authenticated members could read posts from organizations they do not belong to because membership was not tied to `posts.org_id`, and grounds this in pgTAP results showing `posts` failed cross-tenant assertions while `notes` passed." + "judgeNotes": "The agent correctly identified `posts` as the tenant isolation flaw, contrasted it with `notes`, described the issue as users with any membership being able to read other orgs' posts, and used pgTAP tenant-isolation tests to validate the fix." } ], "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", @@ -2025,7 +2081,7 @@ "observability" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -2034,12 +2090,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets all rubric requirements: HTTPS Supabase Metrics API scrape, correct path, Basic Auth with password_file, project target substitution, app job preserved, and Compose secret wiring mounts the password file at /run/secrets/supabase_metrics_secret_key." + "judgeNotes": "Prometheus preserves the app scrape and adds an HTTPS Supabase scrape at /customer/v1/privileged/metrics using basic_auth with password_file. The target is rendered to .supabase.co:443, and docker-compose mounts the secrets directory containing the password file. No bearer auth or hardcoded key is used." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes Secret API key creation/use, matching secret file/mount path, Compose deployment/start steps, and concrete verification via Prometheus target UP and Grafana dashboard." + "passed": false, + "judgeNotes": "README includes correct endpoint/auth, secret file path, Compose recreate step, and concrete Prometheus/curl verification. However it does not provide steps to create/generate the Supabase Secret API key in Supabase; it only tells the user to write a placeholder `sb_secret_...` into the file." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -2073,7 +2129,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -2101,8 +2157,7 @@ "checks": [ { "name": "victim session active before delete-account", - "passed": false, - "notes": "new row violates row-level security policy for table \"notes\"" + "passed": true }, { "name": "delete_account flow ran for the victim", @@ -2119,18 +2174,16 @@ }, { "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "passed": true }, { "name": "other users keep their sessions and access", - "passed": false, - "notes": "new row violates row-level security policy for table \"notes\"" + "passed": true }, { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "Diagnosed the soft-delete/session issue and fixed revocation via sessions/refresh tokens plus banning, and clarified publishable vs secret keys. However, it did not correctly explain the remaining stateless JWT access-token window with the required guidance to use auth.getUser() or short JWT expiry rather than only local JWT validation/getClaims()." + "judgeNotes": "The answer correctly identifies the soft-delete-only bug, deletes the Auth user/sessions, discusses JWTs remaining valid until expiry, and correctly explains publishable vs secret keys. However, it does not clarify the required server-side validation distinction: using auth.getUser() or short JWT expiry instead of relying only on local JWT validation such as getClaims()." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", @@ -2176,7 +2229,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite the channel subscribing, added only public.orders via ALTER PUBLICATION, verified courier_locations remained, and did not weaken RLS/policies or blame client/RLS/networking." + "judgeNotes": "Identified orders missing from supabase_realtime publication despite successful subscription, added only public.orders to existing publication, preserved courier_locations and RLS/policies." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -2195,22 +2248,22 @@ "observability" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant correctly identified image-transform as the affected function and described the recurring HTTP 503 pattern throughout the morning of 2026-04-28, listing all 8 gateway failures from 07:00Z to 12:00Z." + "judgeNotes": "Identified image-transform as affected and described the recurring 503 pattern across 2026-04-28 morning, covering all 8 gateway failures from 07:00Z to 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes recurring image-transform 503s to the API/gateway/invocation layer before the function handler, not function code. Grounds this in valid observations: gateway logs show 503s while nearby Edge Function executions return 200 with normal latency and no matching runtime errors; also distinguishes these from a separate avatar-upload 500 that reached the function runtime." + "passed": false, + "judgeNotes": "The response correctly attributes the 503s to the gateway/platform before invocation and grounds this in missing Edge Function invocation logs. However, it also recommends redeploying `image-transform` and pinning/changing the dependency as a remediation, which the rubric explicitly lists as a fail condition." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket with project/region and timestamps, adding retries, structured request logging, and separately investigating the avatar-upload 500." + "judgeNotes": "The assistant recommended specific actionable next steps, including treating it as an Edge Function runtime issue, adding retries/logging, redeploying with pinned dependency, and opening a Supabase support ticket with gateway log IDs and timestamps." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -2261,7 +2314,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies as the deny-all cause, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all behavior; kept RLS enabled; created authenticated SELECT policy scoped to user_id = auth.uid() and authenticated INSERT policy with WITH CHECK enforcing user_id = auth.uid()." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -2441,8 +2494,7 @@ }, { "name": "schema file updated to include description column", - "passed": false, - "notes": "description not found in any schema file" + "passed": true }, { "name": "a new migration was generated for the change", @@ -2531,7 +2583,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ef70a-0b1f-7479-aafb-8b6d1ba923a3/receipt-alpha.pdf, 019ef70a-0b1f-7479-aafb-8b6d1ba923a3/receipt-beta.pdf" + "notes": "saw: 019ef906-043e-7308-8998-96190742b8c6/receipt-alpha.pdf, 019ef906-043e-7308-8998-96190742b8c6/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -2552,7 +2604,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "The answer creates a private user-files bucket, enables RLS on storage.objects, defines authenticated owner-scoped SELECT and INSERT policies using the first path segment equals auth.uid(), and includes supabase-js createSignedUrl with an expiry for temporary sharing. It does not make the bucket public, use permissive policies, or expose service role usage." + "judgeNotes": "Meets rubric: private bucket, RLS enabled, authenticated owner-scoped SELECT/INSERT policies with WITH CHECK, and supabase-js createSignedUrl with expiry." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -2578,7 +2630,7 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", @@ -2588,7 +2640,7 @@ { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identified `posts` as the table with the tenant isolation flaw, specifically that its RLS policy checked only user membership and not `org_id`, allowing cross-org post reads. It wrote and ran pgTAP tests and treated the results as the signal, then fixed the posts policy." + "judgeNotes": "The agent correctly identified `posts` as the tenant isolation flaw, specifically that membership in any org allowed reading posts across orgs, and reported pgTAP-based testing results after fixing it." } ], "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", @@ -2642,12 +2694,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Supabase scrape preserves the app job and uses HTTPS with the correct metrics path and project target, but it uses basic_auth.password with env substitution instead of basic_auth.password_file. docker-compose.yml also does not mount or provide a password_file via volume or Compose secret, so secret wiring is missing." + "judgeNotes": "Fails: Supabase scrape uses basic_auth.password instead of password_file, and docker-compose.yml does not mount the password file via a volume or Compose secret. The existing app scrape is preserved and endpoint path/scheme are correct, but the required secret wiring is missing." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes Secret API key creation, putting it in observability/.env used by Compose, restarting/recreating Prometheus, and verifying the supabase job via Prometheus /targets." + "passed": false, + "judgeNotes": "README mentions creating a Supabase Secret API key and basic Prometheus target verification, but it does not require placing the matching secret file, and the Compose stack has no secret/env-file wiring. Restart/reload steps are vague, so the secret setup is mismatched/incomplete." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -2681,7 +2733,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get('WEATHER_API_KEY')." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -2737,7 +2789,7 @@ { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "Fails because the answer does not correctly explain that access tokens are stateless JWTs that remain valid until expiry after session/user revocation, nor does it mention using auth.getUser() or short JWT expiry instead of local JWT validation/getClaims(). It instead says there is effectively no window for protected data. Other required parts—identifying soft delete, revoking/deleting auth records, and publishable vs secret key clarification—are mostly present." + "judgeNotes": "The answer correctly identifies the soft-delete problem, revokes auth access, and explains publishable vs secret keys. However, it does not clearly state that access tokens are stateless JWTs that remain valid until expiry after revocation, nor does it advise server-side checks to use auth.getUser() instead of local JWT validation/getClaims()." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", @@ -2783,7 +2835,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified the missing orders table in the supabase_realtime publication as the cause, fixed it with ALTER PUBLICATION ... ADD TABLE public.orders, and did not weaken RLS/policies or disturb courier_locations." + "judgeNotes": "Diagnosed the missing orders table in the supabase_realtime publication and fixed it with ALTER PUBLICATION ADD TABLE public.orders, while not changing RLS, policies, or other realtime feeds." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -2802,22 +2854,22 @@ "observability" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant identified image-transform as the affected function and listed the recurring 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z." + "judgeNotes": "Assistant identified image-transform as affected and described recurring intermittent 503 gateway responses throughout the morning, with successful 200s between. It did not incorrectly focus on old billing-webhook errors." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "The assistant did not clearly attribute the recurring image-transform 503s to the gateway/Edge Functions platform layer in front of the function. It instead framed them as likely boot/startup/runtime/dependency-layer failures, suggested redeploying/fixing dependencies, and did not ground the conclusion in valid observations such as gateway-only 503s with no corresponding invocation/runtime rows, unchanged deployment_id, or distinction from function-level 500s." + "passed": true, + "judgeNotes": "Attributes recurring image-transform 503s to Supabase gateway/platform layer, not function code, and grounds it in valid observations: gateway 503 pattern, successful nearby 200s, unchanged deployment/version, no boot/runtime failures, and distinguishes avatar-upload 500 as separate app-level error." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: inspect specific Edge Function logs for boot errors/exceptions, redeploy affected functions, pin/fix dependency versions if import issues appear, and open a Supabase support ticket with timestamps if platform boot errors persist." + "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket for the intermittent gateway 503 pattern, adding retry/backoff, improving upload error logging, and separately investigating recurring avatar-upload errors." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -2868,7 +2920,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies and added authenticated SELECT and INSERT owner-scoped policies using auth.uid(), without disabling RLS." + "judgeNotes": "Diagnosed deny-all RLS with no policies and added authenticated SELECT and INSERT policies scoped to user_id = auth.uid(), without disabling RLS." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -2896,7 +2948,7 @@ }, { "name": "ran EXPLAIN on the expensive query", - "passed": true + "passed": false }, { "name": "created index covering user_id and created_at", @@ -2905,7 +2957,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at_desc (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -2988,41 +3040,38 @@ ], "suite": "benchmark", "interface": "cli", - "passed": false, + "passed": true, "checks": [ { "name": "supabase project initialised (supabase/config.toml exists)", - "passed": false + "passed": true }, { "name": "todos table is created by a migration file", - "passed": false, - "notes": "supabase/migrations does not exist — was a Supabase project initialised?" + "passed": true }, { "name": "todos table exists with at least 2 seeded rows", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true, + "notes": "found 2 rows" }, { "name": "row level security is enabled on todos", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true }, { "name": "a SELECT policy targets the authenticated role", - "passed": false, - "notes": "could not read DB_URL from `supabase status -o json` after 5 attempts — the local stack must be running. Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true }, { "name": "REST API returns no todos to anonymous requests", - "passed": false, - "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true, + "notes": "0 rows" }, { "name": "REST API returns the todos to authenticated requests", - "passed": false, - "notes": "could not read API_URL/PUBLISHABLE_KEY from `supabase status -o json` after 5 attempts — the local stack must be running and include the auth service (status only reports API keys while gotrue is up; add `gotrue` to the eval's services). Last status: failed to inspect container health: Error response from daemon: No such container: supabase_db_sandbox-f9c97169\nTry rerunning the command with --debug to troubleshoot the error.\n" + "passed": true, + "notes": "2 rows" } ], "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", @@ -3083,7 +3132,7 @@ "sdk" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "rejects missing auth", @@ -3091,19 +3140,19 @@ }, { "name": "user A reads own note", - "passed": true + "passed": false }, { "name": "reads only with the caller's JWT", - "passed": true + "passed": false }, { "name": "user A cannot force-read user B note", - "passed": true + "passed": false }, { "name": "user B cannot force-read user A note", - "passed": true + "passed": false } ], "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", @@ -3141,7 +3190,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ef70a-16c4-70fc-bd5e-29aa1a708d2b/receipt-alpha.pdf, 019ef70a-16c4-70fc-bd5e-29aa1a708d2b/receipt-beta.pdf" + "notes": "saw: 019ef906-4fa0-703c-bb42-0293de1c3924/receipt-alpha.pdf, 019ef906-4fa0-703c-bb42-0293de1c3924/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -3162,7 +3211,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Configured a private user-files bucket, kept RLS enabled, added authenticated owner-scoped SELECT and INSERT policies using the user-id path prefix, and provided supabase-js createSignedUrl code with an expiry for temporary sharing." + "judgeNotes": "Configured a private/default user-files bucket, owner-scoped SELECT and INSERT policies for authenticated users on storage.objects, kept RLS intact, and provided supabase-js createSignedUrl code with an expiry." } ], "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", @@ -3193,12 +3242,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": false, - "notes": "no test summary found; exit 0; output: Connecting to local database...\n3.36: Pulling from supabase/pg_prove\ndcccee43ad5d: Pulling fs layer\n06d62d0de6d7: Pulling fs layer\na22cb17b3b93: Pulling fs layer\n4f4fb700ef54: Pulling fs layer\n4f4fb700ef54: Waiting\na22cb17b3b93: Verifying Checksum\na22cb17b3b93: Download complete\ndcccee43ad5d: Verifying Checksum\ndcccee43ad5d: Download complete\n06d62d0de6d7: Verifying Checksum\n06d62d0de6d7: Download complete\n4f4fb700ef54: Verifying Checksum\n4f4fb700ef54: Download complete\ndcccee43ad5d: Pull comple" + "notes": "no test summary found; exit 0; output: Connecting to local database...\n3.36: Pulling from supabase/pg_prove\ndcccee43ad5d: Pulling fs layer\n06d62d0de6d7: Pulling fs layer\na22cb17b3b93: Pulling fs layer\n4f4fb700ef54: Pulling fs layer\n4f4fb700ef54: Waiting\na22cb17b3b93: Download complete\ndcccee43ad5d: Verifying Checksum\ndcccee43ad5d: Download complete\n4f4fb700ef54: Verifying Checksum\n4f4fb700ef54: Download complete\n06d62d0de6d7: Verifying Checksum\n06d62d0de6d7: Download complete\ndcccee43ad5d: Pull complete\n06d62d0de6d7: Pull complete\na2" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies posts as the table with broken tenant isolation, explains that authenticated members can read posts from organizations they are not members of, and grounds the conclusion in the test results showing notes pass while posts fail. It does not blame notes or dismiss the tests." + "judgeNotes": "The agent correctly identified `posts` as the table with broken tenant isolation, specifically that authenticated members can read posts from orgs they do not belong to. It grounded the conclusion in the test results showing `count(posts)=2` failures for both users, while `notes` read isolation passed." } ], "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", @@ -3252,12 +3301,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails because prometheus.yml uses basic_auth.password instead of password_file, and docker-compose.yml does not mount/provide that password file via a volume or Compose secret. The app scrape is preserved and the HTTPS metrics path/target shape are present, but the required secret wiring is missing." + "judgeNotes": "Fails: Supabase scrape uses basic_auth.password instead of password_file, docker-compose.yml does not mount/provide the password file, and the target is a placeholder rather than a deployable project ref. Existing app scrape is preserved and endpoint/scheme are otherwise correct." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README includes creating a Secret API key, endpoint, restart, and a curl verification step, but it does not require placing the key in a matching secret file. Instead it instructs editing prometheus.yml with the secret inline, which fails the required secret-file setup." + "judgeNotes": "README.md does not explain how to make the Supabase metrics integration live. It lacks steps to create a Secret API key, place a matching secret file, restart/reload the Compose stack, and verify via Prometheus targets/PromQL/Grafana. The shown Prometheus config also uses a literal placeholder password rather than a wired secret." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -3291,7 +3340,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from runtime environment using Deno.env.get." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -3319,8 +3368,7 @@ "checks": [ { "name": "victim session active before delete-account", - "passed": false, - "notes": "permission denied for table users" + "passed": true }, { "name": "delete_account flow ran for the victim", @@ -3343,13 +3391,12 @@ }, { "name": "other users keep their sessions and access", - "passed": false, - "notes": "permission denied for table users" + "passed": true }, { "name": "diagnosed and explained session revocation", "passed": false, - "judgeNotes": "The answer correctly identifies the original soft-delete/RLS issue and correctly explains publishable vs secret keys. However, it does not implement real Auth deletion/session and refresh-token revocation; it only updates auth.users.deleted_at. It also fails to explain that stateless JWT access tokens can remain valid until expiry and that server-side checks should use auth.getUser() or short JWT expiry rather than only local JWT validation/getClaims(). It incorrectly implies the next DB request will be denied immediately as the full token-window answer." + "judgeNotes": "The answer correctly identifies that delete_account only soft-deletes the profile and discusses publishable vs secret keys/RLS mostly correctly. However it does not fix the required flow by deleting the auth user or revoking sessions/refresh tokens; it only changes RLS policies. It also incorrectly says after commit there is no practical JWT/session window, failing to explain that stateless access JWTs can remain valid until expiry and that server-side checks should use auth.getUser() or short JWT expiry rather than local getClaims()." } ], "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", @@ -3395,7 +3442,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite the channel subscribing successfully, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not change RLS, policies, client code, or disrupt courier_locations." + "judgeNotes": "Diagnosed the silent subscription as orders missing from supabase_realtime, fixed it with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not alter RLS/policies or disrupt courier_locations." } ], "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", @@ -3419,17 +3466,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant correctly identified image-transform as the affected function and described recurring/intermittent HTTP 503 gateway/API responses during the 2026-04-28 morning period, while distinguishing them from function-runtime successes and unrelated errors." + "judgeNotes": "The assistant correctly identified `image-transform` as the affected function and described a recurring intermittent pattern of HTTP 503 responses during the morning of 2026-04-28, with 200s in between. It did not quantify all 8 gateway failures or specify the full 07:00Z-12:00Z spread, but it met the rubric's core requirements." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": false, - "judgeNotes": "The answer correctly attributes the recurring 503s to the gateway/API layer and grounds that in valid observations, especially gateway 503s with only successful image-transform function runtime logs. However, it also recommends temporarily redeploying image-transform, which the rubric explicitly lists as a fail condition when presented as remediation/next step." + "judgeNotes": "The response does not clearly attribute the recurring 503s to the gateway/platform layer in front of the function. It frames the issue as the image-transform backend/function being unavailable, mentions possible function runtime/dependency problems, and recommends redeploying the function. It also lacks the required grounding observations such as gateway-only 503s with no invocation/runtime rows, unchanged deployment_id, or distinction from function-level errors." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including checking deployment/traffic spikes, redeploying the function, collecting request IDs from failing 503 responses, and correlating them with logs." + "judgeNotes": "The assistant recommended concrete next steps, including adding retries, making transformation asynchronous, redeploying the Edge Function, verifying dependencies, and monitoring Edge Function logs for recurring 503s." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -3480,7 +3527,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API behavior, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", @@ -3504,7 +3551,7 @@ "checks": [ { "name": "inspected pg_stat_statements for query performance", - "passed": false + "passed": true }, { "name": "ran EXPLAIN on the expensive query", @@ -3517,7 +3564,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", From d34f45aa897706d0a970b639b741e283d0aa7516 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Thu, 25 Jun 2026 15:35:40 +0100 Subject: [PATCH 4/5] change default models and add reasoning effort --- .github/workflows/eval-refresh.yml | 4 ++-- ...e-haiku-4.5.ts => claude-code-opus-4.8.ts} | 6 ++---- experiments/claude-code-sonnet-4.6.ts | 7 +------ ...codex-gpt-5.4.ts => codex-gpt-5.4-mini.ts} | 7 ++----- experiments/codex-gpt-5.5.ts | 5 +---- packages/core/src/cli-agent.ts | 12 +++++++++++- packages/core/src/index.ts | 3 ++- packages/core/src/runners/claude-code.ts | 4 +++- packages/core/src/runners/codex.ts | 7 ++++++- packages/core/src/runners/types.ts | 19 +++++++++++++++++++ 10 files changed, 49 insertions(+), 25 deletions(-) rename experiments/{claude-code-haiku-4.5.ts => claude-code-opus-4.8.ts} (65%) rename experiments/{codex-gpt-5.4.ts => codex-gpt-5.4-mini.ts} (56%) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 6ca7b47d..a2746f27 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -6,7 +6,7 @@ on: experiments: description: "Comma-separated experiment names to run" required: true - default: "openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-haiku-4.5,claude-code-sonnet-4.6,codex-gpt-5.4,codex-gpt-5.5" + default: "openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-opus-4.8,claude-code-sonnet-4.6,codex-gpt-5.4-mini,codex-gpt-5.5" eval: description: "Optional single eval id to run" required: false @@ -72,7 +72,7 @@ jobs: runs="${{ inputs.runs }}" timeout_sec="${{ inputs.timeout_sec }}" else - experiments="openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-haiku-4.5,claude-code-sonnet-4.6,codex-gpt-5.4,codex-gpt-5.5" + experiments="openai-gpt-5.4-mini,openai-gpt-5.4-nano,claude-code-opus-4.8,claude-code-sonnet-4.6,codex-gpt-5.4-mini,codex-gpt-5.5" eval_id="" suite="benchmark" runs="1" diff --git a/experiments/claude-code-haiku-4.5.ts b/experiments/claude-code-opus-4.8.ts similarity index 65% rename from experiments/claude-code-haiku-4.5.ts rename to experiments/claude-code-opus-4.8.ts index 79b61608..31978854 100644 --- a/experiments/claude-code-haiku-4.5.ts +++ b/experiments/claude-code-opus-4.8.ts @@ -6,12 +6,10 @@ import { } from "@supabase-evals/core"; import { localStackRuntime } from "@supabase-evals/sandbox"; -// Claude Code on Haiku. See experiments/claude-code-sonnet-4.6.ts for the -// CLI-agent notes (runs in both modes: full sandbox for local-stack evals, bare -// sandbox + MCP for tools-mode evals). export default defineExperiment({ agent: claudeCodeAgent({ - model: "claude-haiku-4-5", + model: "claude-opus-4-8", + reasoningEffort: "high", }), runtime: platformLiteRuntime({ mcpServers: [supabaseMcpServer()], diff --git a/experiments/claude-code-sonnet-4.6.ts b/experiments/claude-code-sonnet-4.6.ts index 05d200e5..266fb331 100644 --- a/experiments/claude-code-sonnet-4.6.ts +++ b/experiments/claude-code-sonnet-4.6.ts @@ -6,15 +6,10 @@ import { } from "@supabase-evals/core"; import { localStackRuntime } from "@supabase-evals/sandbox"; -// Claude Code is a CLI agent: it runs its own harness (Read/Write/Bash/Edit + -// MCP) inside a sandbox, in BOTH eval modes. Local-stack evals (interface: cli -// or a local/ workspace) get the full sandbox — the Supabase CLI plus a running -// local stack. Tools-mode evals get a bare sandbox (same image, no stack) where -// the eval's tools come from the `runtime` MCP servers (reached host-side via -// host.docker.internal). The running stack + the CLI is the only mode difference. export default defineExperiment({ agent: claudeCodeAgent({ model: "claude-sonnet-4-6", + reasoningEffort: "high", }), runtime: platformLiteRuntime({ mcpServers: [supabaseMcpServer()], diff --git a/experiments/codex-gpt-5.4.ts b/experiments/codex-gpt-5.4-mini.ts similarity index 56% rename from experiments/codex-gpt-5.4.ts rename to experiments/codex-gpt-5.4-mini.ts index ebaca610..95b4c29e 100644 --- a/experiments/codex-gpt-5.4.ts +++ b/experiments/codex-gpt-5.4-mini.ts @@ -6,13 +6,10 @@ import { } from "@supabase-evals/core"; import { localStackRuntime } from "@supabase-evals/sandbox"; -// Codex runs in both modes, like Claude Code: `runtime` drives tools-mode evals -// (the runner writes its MCP servers into ~/.codex/config.toml against platform- -// lite) and `localStack` drives local-stack evals. Which mode an eval uses is a -// property of the eval (interface/local dir), not the agent. export default defineExperiment({ agent: codexAgent({ - model: "gpt-5.4", + model: "gpt-5.4-mini", + reasoningEffort: "medium", }), runtime: platformLiteRuntime({ mcpServers: [supabaseMcpServer()], diff --git a/experiments/codex-gpt-5.5.ts b/experiments/codex-gpt-5.5.ts index 47f50c31..7bfbf985 100644 --- a/experiments/codex-gpt-5.5.ts +++ b/experiments/codex-gpt-5.5.ts @@ -6,13 +6,10 @@ import { } from "@supabase-evals/core"; import { localStackRuntime } from "@supabase-evals/sandbox"; -// Codex runs in both modes, like Claude Code: `runtime` drives tools-mode evals -// (the runner writes its MCP servers into ~/.codex/config.toml against platform- -// lite) and `localStack` drives local-stack evals. Which mode an eval uses is a -// property of the eval (interface/local dir), not the agent. export default defineExperiment({ agent: codexAgent({ model: "gpt-5.5", + reasoningEffort: "medium", }), runtime: platformLiteRuntime({ mcpServers: [supabaseMcpServer()], diff --git a/packages/core/src/cli-agent.ts b/packages/core/src/cli-agent.ts index aac646f1..492036c9 100644 --- a/packages/core/src/cli-agent.ts +++ b/packages/core/src/cli-agent.ts @@ -26,6 +26,7 @@ import { claudeCodeParser } from "./parsers/claude-code.js"; import { codexParser } from "./parsers/codex.js"; import { claudeCodeRunner } from "./runners/claude-code.js"; import { codexRunner, type CodexModel } from "./runners/codex.js"; +import type { ClaudeCodeEffort, CodexReasoningEffort } from "./runners/types.js"; import type { AgentRunner } from "./runners/types.js"; import { SCRATCH, @@ -41,13 +42,15 @@ export type { AgentRunner, RunnerExecArgs, RunnerExecResult, + ClaudeCodeEffort, + CodexReasoningEffort, } from "./runners/types.js"; /** Compose a runner + parser into an `AgentHarness`. */ export function createCliAgent( runner: AgentRunner, parser: AgentTranscriptParser, - options: { model: M; cliVersion?: string }, + options: { model: M; cliVersion?: string; reasoningEffort?: string }, ): AgentHarness { const version = options.cliVersion ?? runner.defaultCliVersion; return { @@ -82,6 +85,7 @@ export function createCliAgent( // Rewrite loopback hosts so in-container MCP servers can reach host-side // platform-lite; the runner writes them in its own config format. mcpServers: rewriteLoopback(args.mcpServers ?? {}), + reasoningEffort: options.reasoningEffort, timeoutSec: args.timeoutSec, }); @@ -116,12 +120,15 @@ export function claudeCodeAgent( options: { /** Anthropic model id (typed from `@anthropic-ai/sdk`). Defaults to Sonnet. */ model?: AnthropicModel; + /** Reasoning effort (`--effort`). Omit to use Claude Code's own default. */ + reasoningEffort?: ClaudeCodeEffort; /** Override the pinned CLI version. */ cliVersion?: string; } = {}, ): AgentHarness { return createCliAgent(claudeCodeRunner, claudeCodeParser, { model: options.model ?? claudeCodeRunner.defaultModel, + reasoningEffort: options.reasoningEffort, cliVersion: options.cliVersion, }); } @@ -131,12 +138,15 @@ export function codexAgent( options: { /** OpenAI model id (typed from `openai`; any string accepted). */ model?: CodexModel; + /** Reasoning effort (`model_reasoning_effort`). Omit to use Codex's default. */ + reasoningEffort?: CodexReasoningEffort; /** Override the pinned CLI version. */ cliVersion?: string; } = {}, ): AgentHarness { return createCliAgent(codexRunner, codexParser, { model: options.model ?? codexRunner.defaultModel, + reasoningEffort: options.reasoningEffort, cliVersion: options.cliVersion, }); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c64f4ef3..d1b843a2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -74,7 +74,6 @@ export { export { parseEvalMarkdown } from "./eval-markdown.js"; // CLI agent harnesses (Claude Code, Codex, and the framework for adding more). export { - createCliAgent, claudeCodeAgent, codexAgent, } from "./cli-agent.js"; @@ -83,6 +82,8 @@ export type { AgentRunner, RunnerExecArgs, RunnerExecResult, + ClaudeCodeEffort, + CodexReasoningEffort, } from "./cli-agent.js"; // Generic transcript vocabulary + parser layer used by CLI agents. export { createParser, supportedParsers } from "./parsers/registry.js"; diff --git a/packages/core/src/runners/claude-code.ts b/packages/core/src/runners/claude-code.ts index c776d443..3e4c6a26 100644 --- a/packages/core/src/runners/claude-code.ts +++ b/packages/core/src/runners/claude-code.ts @@ -32,7 +32,7 @@ export const claudeCodeRunner: AgentRunner = { await npmInstallGlobal(sandbox, `${this.cliPackage}@${version}`, this.displayName); }, - async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, timeoutSec }) { + async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, reasoningEffort, timeoutSec }) { const claude = npmGlobalBin("claude"); const serverNames = Object.keys(mcpServers); @@ -49,6 +49,8 @@ export const claudeCodeRunner: AgentRunner = { "--output-format stream-json", "--verbose", `--model ${shellQuote(model)}`, + // Reasoning effort for the session; omitted leaves Claude Code's default. + ...(reasoningEffort ? [`--effort ${shellQuote(reasoningEffort)}`] : []), // Append (not replace), from a file (no ARG_MAX/shell-expansion surface), // so Claude Code keeps its default coding-agent prompt + tool guidance. `--append-system-prompt-file ${systemPromptPath}`, diff --git a/packages/core/src/runners/codex.ts b/packages/core/src/runners/codex.ts index c8060914..67016822 100644 --- a/packages/core/src/runners/codex.ts +++ b/packages/core/src/runners/codex.ts @@ -47,7 +47,7 @@ export const codexRunner: AgentRunner = { } }, - async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, timeoutSec }) { + async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, reasoningEffort, timeoutSec }) { const codex = npmGlobalBin("codex"); if (Object.keys(mcpServers).length > 0) { await sandbox.exec(`mkdir -p "$HOME/.codex"`); @@ -62,6 +62,11 @@ export const codexRunner: AgentRunner = { // The sandbox is the isolation boundary — let Codex run commands freely. "--dangerously-bypass-approvals-and-sandbox", `-m ${shellQuote(model)}`, + // Reasoning effort via config override; omitted leaves Codex's default. + // The value is parsed as TOML, so pass it as a quoted TOML string. + ...(reasoningEffort + ? [`-c ${shellQuote(`model_reasoning_effort="${reasoningEffort}"`)}`] + : []), // Read the prompt from stdin. "-", ].join(" "); diff --git a/packages/core/src/runners/types.ts b/packages/core/src/runners/types.ts index ec307267..b7392f1f 100644 --- a/packages/core/src/runners/types.ts +++ b/packages/core/src/runners/types.ts @@ -10,8 +10,22 @@ * concern that diverges per agent lives in exactly one place. */ +import type { OutputConfig } from "@anthropic-ai/sdk/resources/messages"; +import type { ReasoningEffort } from "openai/resources/shared"; import type { CommandResult, McpServerConfig } from "../index.js"; +/** + * Reasoning effort for Claude Code's `--effort` flag. Derived from the Anthropic + * SDK's `OutputConfig.effort` (minus null) so it tracks the provider's set. + */ +export type ClaudeCodeEffort = NonNullable; + +/** + * Reasoning effort for Codex's `model_reasoning_effort` config. Derived from the + * OpenAI SDK's `ReasoningEffort` (minus null) so it tracks the provider's set. + */ +export type CodexReasoningEffort = NonNullable; + /** * The slice of an execution environment a CLI agent needs: a workspace, a way * to run shell commands in it, and a way to read files back out. The local- @@ -46,6 +60,11 @@ export interface RunnerExecArgs { userPromptPath: string; /** MCP servers to expose, already loopback-rewritten. Empty when none. */ mcpServers: Record; + /** + * Reasoning effort the CLI should run at, if the caller pinned one. + * Undefined = leave the CLI's own default. + */ + reasoningEffort?: string; timeoutSec: number; } From b46eb1419502f04d04ec28209ae64cba0b26f67b Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Thu, 25 Jun 2026 16:15:52 +0100 Subject: [PATCH 5/5] refactor(core): co-locate CLI agents and unify the agent registry Split the CLI-agent layer so each agent owns its own factory. The generic orchestration (createCliAgent) moves to agents/engine.ts and no longer imports any concrete runner, parser, or model/effort type. Each agent now lives under agents// (runner + parser + an index.ts that wires them into the public claudeCodeAgent/codexAgent factory). Collapse the separate parsers/registry.ts into agents/registry.ts, derived from the AgentDefinition each agent module exports (id comes from runner.id, so it is no longer written twice). Generic transcript tooling stays in parsers/. Adding an agent is now one folder + one registry line; no shared file changes. Public API is unchanged; createCliAgent and AgentDefinition are newly exported. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/core/src/agents/claude-code/index.ts | 37 ++++++++++ .../claude-code/parser.test.ts} | 4 +- .../claude-code/parser.ts} | 10 +-- .../claude-code/runner.test.ts} | 4 +- .../claude-code/runner.ts} | 12 ++-- packages/core/src/agents/codex/index.ts | 36 ++++++++++ .../codex/parser.test.ts} | 6 +- .../codex.ts => agents/codex/parser.ts} | 10 +-- .../codex.ts => agents/codex/runner.ts} | 12 ++-- .../src/{cli-agent.ts => agents/engine.ts} | 72 +++---------------- packages/core/src/agents/registry.ts | 34 +++++++++ .../core/src/{runners => agents}/shared.ts | 0 .../core/src/{runners => agents}/types.ts | 20 +++++- packages/core/src/index.ts | 14 ++-- packages/core/src/parsers/registry.ts | 31 -------- packages/core/src/parsers/types.ts | 4 +- 16 files changed, 173 insertions(+), 133 deletions(-) create mode 100644 packages/core/src/agents/claude-code/index.ts rename packages/core/src/{parsers/parsers.test.ts => agents/claude-code/parser.test.ts} (98%) rename packages/core/src/{parsers/claude-code.ts => agents/claude-code/parser.ts} (96%) rename packages/core/src/{cli-agent.test.ts => agents/claude-code/runner.test.ts} (92%) rename packages/core/src/{runners/claude-code.ts => agents/claude-code/runner.ts} (91%) create mode 100644 packages/core/src/agents/codex/index.ts rename packages/core/src/{parsers/codex.test.ts => agents/codex/parser.test.ts} (97%) rename packages/core/src/{parsers/codex.ts => agents/codex/parser.ts} (94%) rename packages/core/src/{runners/codex.ts => agents/codex/runner.ts} (94%) rename packages/core/src/{cli-agent.ts => agents/engine.ts} (57%) create mode 100644 packages/core/src/agents/registry.ts rename packages/core/src/{runners => agents}/shared.ts (100%) rename packages/core/src/{runners => agents}/types.ts (83%) delete mode 100644 packages/core/src/parsers/registry.ts diff --git a/packages/core/src/agents/claude-code/index.ts b/packages/core/src/agents/claude-code/index.ts new file mode 100644 index 00000000..67855b51 --- /dev/null +++ b/packages/core/src/agents/claude-code/index.ts @@ -0,0 +1,37 @@ +/** + * Claude Code agent. Owns everything Claude-Code-specific: it wires its own + * runner + parser into the public `claudeCodeAgent` factory (via the generic + * `createCliAgent` engine) and exports the registry definition the harness uses + * to parse Claude Code transcripts. + */ + +import type { Model as AnthropicModel } from "@anthropic-ai/sdk/resources/messages"; +import type { AgentHarness } from "../../index.js"; +import { createCliAgent } from "../engine.js"; +import type { AgentDefinition, ClaudeCodeEffort } from "../types.js"; +import { claudeCodeRunner } from "./runner.js"; +import { claudeCodeParser } from "./parser.js"; + +/** Claude Code as an `AgentHarness`. */ +export function claudeCodeAgent( + options: { + /** Anthropic model id (typed from `@anthropic-ai/sdk`). Defaults to Sonnet. */ + model?: AnthropicModel; + /** Reasoning effort (`--effort`). Omit to use Claude Code's own default. */ + reasoningEffort?: ClaudeCodeEffort; + /** Override the pinned CLI version. */ + cliVersion?: string; + } = {}, +): AgentHarness { + return createCliAgent(claudeCodeRunner, claudeCodeParser, { + model: options.model ?? claudeCodeRunner.defaultModel, + reasoningEffort: options.reasoningEffort, + cliVersion: options.cliVersion, + }); +} + +/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */ +export const claudeCodeDefinition: AgentDefinition = { + runner: claudeCodeRunner, + parser: claudeCodeParser, +}; diff --git a/packages/core/src/parsers/parsers.test.ts b/packages/core/src/agents/claude-code/parser.test.ts similarity index 98% rename from packages/core/src/parsers/parsers.test.ts rename to packages/core/src/agents/claude-code/parser.test.ts index f52095ed..26dfb58c 100644 --- a/packages/core/src/parsers/parsers.test.ts +++ b/packages/core/src/agents/claude-code/parser.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { claudeCodeParser } from "./claude-code.js"; -import { adaptTranscript } from "./adapt.js"; +import { claudeCodeParser } from "./parser.js"; +import { adaptTranscript } from "../../parsers/adapt.js"; /** A representative Claude Code `--print` JSONL session. */ const SESSION = [ diff --git a/packages/core/src/parsers/claude-code.ts b/packages/core/src/agents/claude-code/parser.ts similarity index 96% rename from packages/core/src/parsers/claude-code.ts rename to packages/core/src/agents/claude-code/parser.ts index e1e4f3e5..72032ecb 100644 --- a/packages/core/src/parsers/claude-code.ts +++ b/packages/core/src/agents/claude-code/parser.ts @@ -14,11 +14,11 @@ import type { ParsedTranscript, TranscriptEvent, -} from "../transcript/types.js"; -import type { AgentTranscriptParser } from "./types.js"; -import { isRecord, parseJsonlRecords } from "../json.js"; -import { normalizeToolName, type AgentToolMap } from "./shared/normalize.js"; -import { extractArgs, type ArgFieldMap } from "./shared/extract.js"; +} from "../../transcript/types.js"; +import type { AgentTranscriptParser } from "../../parsers/types.js"; +import { isRecord, parseJsonlRecords } from "../../json.js"; +import { normalizeToolName, type AgentToolMap } from "../../parsers/shared/normalize.js"; +import { extractArgs, type ArgFieldMap } from "../../parsers/shared/extract.js"; /** Claude Code's tool names → canonical names (case-sensitive). Owned here, not in shared. */ const CLAUDE_CODE_TOOLS: AgentToolMap = { diff --git a/packages/core/src/cli-agent.test.ts b/packages/core/src/agents/claude-code/runner.test.ts similarity index 92% rename from packages/core/src/cli-agent.test.ts rename to packages/core/src/agents/claude-code/runner.test.ts index 2ceaed2d..43779cbc 100644 --- a/packages/core/src/cli-agent.test.ts +++ b/packages/core/src/agents/claude-code/runner.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { claudeCodeRunner } from "./runners/claude-code.js"; -import type { CommandResult } from "./index.js"; +import { claudeCodeRunner } from "./runner.js"; +import type { CommandResult } from "../../index.js"; const ok: CommandResult = { ok: true, exitCode: 0, stdout: "", stderr: "" }; const timedOut: CommandResult = { diff --git a/packages/core/src/runners/claude-code.ts b/packages/core/src/agents/claude-code/runner.ts similarity index 91% rename from packages/core/src/runners/claude-code.ts rename to packages/core/src/agents/claude-code/runner.ts index 3e4c6a26..b293c6d7 100644 --- a/packages/core/src/runners/claude-code.ts +++ b/packages/core/src/agents/claude-code/runner.ts @@ -1,20 +1,20 @@ /** * Claude Code runner. Headless via `claude -p --output-format stream-json` * (Anthropic's recommended programmatic path — events stream to stdout, no - * on-disk session-file race). See parsers/claude-code.ts for the transcript. + * on-disk session-file race). See ./parser.ts for the transcript. */ import type { Model as AnthropicModel } from "@anthropic-ai/sdk/resources/messages"; -import type { McpServerConfig } from "../index.js"; -import { parseJsonlRecords } from "../json.js"; -import type { AgentRunner } from "./types.js"; +import type { McpServerConfig } from "../../index.js"; +import { parseJsonlRecords } from "../../json.js"; +import type { AgentRunner } from "../types.js"; import { npmGlobalBin, npmInstallGlobal, processStopReason, shellQuote, writeSandboxFile, -} from "./shared.js"; +} from "../shared.js"; const MCP_CONFIG_PATH = '"$HOME/.eval/mcp.json"'; @@ -24,7 +24,7 @@ export const claudeCodeRunner: AgentRunner = { apiKeyEnvVar: "ANTHROPIC_API_KEY", cliPackage: "@anthropic-ai/claude-code", // Pinned: Claude Code's transcript format evolves; bump deliberately and - // re-check the parser. See packages/core/src/parsers/claude-code.ts. + // re-check the parser. See ./parser.ts. defaultCliVersion: "2.1.101", defaultModel: "claude-sonnet-4-6", diff --git a/packages/core/src/agents/codex/index.ts b/packages/core/src/agents/codex/index.ts new file mode 100644 index 00000000..cdb82a6f --- /dev/null +++ b/packages/core/src/agents/codex/index.ts @@ -0,0 +1,36 @@ +/** + * OpenAI Codex agent. Owns everything Codex-specific: it wires its own runner + + * parser into the public `codexAgent` factory (via the generic `createCliAgent` + * engine) and exports the registry definition the harness uses to parse Codex + * transcripts. Runs in both modes, like Claude Code. + */ + +import type { AgentHarness } from "../../index.js"; +import { createCliAgent } from "../engine.js"; +import type { AgentDefinition, CodexReasoningEffort } from "../types.js"; +import { codexRunner, type CodexModel } from "./runner.js"; +import { codexParser } from "./parser.js"; + +/** OpenAI Codex as an `AgentHarness`. */ +export function codexAgent( + options: { + /** OpenAI model id (typed from `openai`; any string accepted). */ + model?: CodexModel; + /** Reasoning effort (`model_reasoning_effort`). Omit to use Codex's default. */ + reasoningEffort?: CodexReasoningEffort; + /** Override the pinned CLI version. */ + cliVersion?: string; + } = {}, +): AgentHarness { + return createCliAgent(codexRunner, codexParser, { + model: options.model ?? codexRunner.defaultModel, + reasoningEffort: options.reasoningEffort, + cliVersion: options.cliVersion, + }); +} + +/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */ +export const codexDefinition: AgentDefinition = { + runner: codexRunner, + parser: codexParser, +}; diff --git a/packages/core/src/parsers/codex.test.ts b/packages/core/src/agents/codex/parser.test.ts similarity index 97% rename from packages/core/src/parsers/codex.test.ts rename to packages/core/src/agents/codex/parser.test.ts index bc28159d..3b901897 100644 --- a/packages/core/src/parsers/codex.test.ts +++ b/packages/core/src/agents/codex/parser.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; -import { codexParser } from "./codex.js"; -import { codexRunner } from "../runners/codex.js"; -import { adaptTranscript } from "./adapt.js"; +import { codexParser } from "./parser.js"; +import { codexRunner } from "./runner.js"; +import { adaptTranscript } from "../../parsers/adapt.js"; /** A representative `codex exec --json` stream (shapes captured from CLI 0.138). */ const SESSION = [ diff --git a/packages/core/src/parsers/codex.ts b/packages/core/src/agents/codex/parser.ts similarity index 94% rename from packages/core/src/parsers/codex.ts rename to packages/core/src/agents/codex/parser.ts index bb7c134e..ec0e3a97 100644 --- a/packages/core/src/parsers/codex.ts +++ b/packages/core/src/agents/codex/parser.ts @@ -18,11 +18,11 @@ * format (event_msg/response_item) that older parsers targeted. */ -import { isRecord, parseJsonlRecords } from "../json.js"; -import type { ParsedTranscript, TranscriptEvent } from "../transcript/types.js"; -import type { AgentTranscriptParser } from "./types.js"; -import { normalizeToolName, type AgentToolMap } from "./shared/normalize.js"; -import { extractArgs, type ArgFieldMap, type ExtractedArgs } from "./shared/extract.js"; +import { isRecord, parseJsonlRecords } from "../../json.js"; +import type { ParsedTranscript, TranscriptEvent } from "../../transcript/types.js"; +import type { AgentTranscriptParser } from "../../parsers/types.js"; +import { normalizeToolName, type AgentToolMap } from "../../parsers/shared/normalize.js"; +import { extractArgs, type ArgFieldMap, type ExtractedArgs } from "../../parsers/shared/extract.js"; /** * Codex's tool names → canonical names. Codex names built-in tools by item type diff --git a/packages/core/src/runners/codex.ts b/packages/core/src/agents/codex/runner.ts similarity index 94% rename from packages/core/src/runners/codex.ts rename to packages/core/src/agents/codex/runner.ts index 67016822..215193ef 100644 --- a/packages/core/src/runners/codex.ts +++ b/packages/core/src/agents/codex/runner.ts @@ -1,6 +1,6 @@ /** * Codex runner. Headless via `codex exec --json` (newline-delimited thread/turn/ - * item events on stdout; see parsers/codex.ts). Like Claude Code, it runs in + * item events on stdout; see ./parser.ts). Like Claude Code, it runs in * both modes: the sandbox carries its shell/file tools in either case, and tools * mode just drops the Supabase CLI + local stack so Supabase access goes through * MCP (`~/.codex/config.toml`). Runs under `--dangerously-bypass-approvals-and- @@ -8,16 +8,16 @@ */ import type { ChatModel } from "openai/resources/shared"; -import type { McpServerConfig } from "../index.js"; -import { parseJsonlRecords } from "../json.js"; -import type { AgentRunner } from "./types.js"; +import type { McpServerConfig } from "../../index.js"; +import { parseJsonlRecords } from "../../json.js"; +import type { AgentRunner } from "../types.js"; import { npmGlobalBin, npmInstallGlobal, processStopReason, shellQuote, writeSandboxFile, -} from "./shared.js"; +} from "../shared.js"; // ChatModel is a closed union; widen so newer/codex-specific ids still type. export type CodexModel = ChatModel | (string & {}); @@ -30,7 +30,7 @@ export const codexRunner: AgentRunner = { apiKeyEnvVar: "OPENAI_API_KEY", cliPackage: "@openai/codex", // Pinned: Codex's --json event schema evolves; bump deliberately and re-check - // the parser. See packages/core/src/parsers/codex.ts. + // the parser. See ./parser.ts. defaultCliVersion: "0.138.0", defaultModel: "gpt-5.4", diff --git a/packages/core/src/cli-agent.ts b/packages/core/src/agents/engine.ts similarity index 57% rename from packages/core/src/cli-agent.ts rename to packages/core/src/agents/engine.ts index 492036c9..5c8db066 100644 --- a/packages/core/src/cli-agent.ts +++ b/packages/core/src/agents/engine.ts @@ -1,5 +1,5 @@ /** - * CLI-agent harnesses. + * CLI-agent engine — the generic orchestration shared by every CLI agent. * * `aiSdkAgent` drives the model loop in-process: we own the tools and record * the transcript as it happens. A CLI agent (Claude Code, Codex, …) is the @@ -7,27 +7,22 @@ * run it inside the eval sandbox and parse the transcript it produces. * * Three concerns are split so each lives in one place: - * - runner (`./runners/.ts`): install + exec + permission flags + MCP + * - runner (`.//runner.ts`): install + exec + permission flags + MCP * config format/placement — everything CLI-shaped. - * - parser (`./parsers/.ts`): raw transcript → canonical events. + * - parser (`.//parser.ts`): raw transcript → canonical events. * - composition (here): `createCliAgent(runner, parser)` does the generic * orchestration (stage prompts, rewrite MCP hosts, run, parse, adapt) and * produces an `AgentHarness`. * - * Adding an agent = a runner + a parser + a registry entry; the orchestration - * never changes. + * Each agent's `.//index.ts` wires its own runner + parser into a public + * factory by calling `createCliAgent`. This engine knows nothing about any + * specific agent, so adding one never touches this file. */ -import type { Model as AnthropicModel } from "@anthropic-ai/sdk/resources/messages"; -import type { AgentHarness, AgentRunResult } from "./index.js"; -import { adaptTranscript } from "./parsers/adapt.js"; -import type { AgentTranscriptParser } from "./parsers/types.js"; -import { claudeCodeParser } from "./parsers/claude-code.js"; -import { codexParser } from "./parsers/codex.js"; -import { claudeCodeRunner } from "./runners/claude-code.js"; -import { codexRunner, type CodexModel } from "./runners/codex.js"; -import type { ClaudeCodeEffort, CodexReasoningEffort } from "./runners/types.js"; -import type { AgentRunner } from "./runners/types.js"; +import type { AgentHarness, AgentRunResult } from "../index.js"; +import { adaptTranscript } from "../parsers/adapt.js"; +import type { AgentTranscriptParser } from "../parsers/types.js"; +import type { AgentRunner } from "./types.js"; import { SCRATCH, SYSTEM_PROMPT_PATH, @@ -35,16 +30,7 @@ import { processStopReason, rewriteLoopback, writeSandboxFile, -} from "./runners/shared.js"; - -export type { - AgentSandbox, - AgentRunner, - RunnerExecArgs, - RunnerExecResult, - ClaudeCodeEffort, - CodexReasoningEffort, -} from "./runners/types.js"; +} from "./shared.js"; /** Compose a runner + parser into an `AgentHarness`. */ export function createCliAgent( @@ -114,39 +100,3 @@ function requireApiKey(runner: AgentRunner): string { } return apiKey; } - -/** Claude Code as an `AgentHarness`. */ -export function claudeCodeAgent( - options: { - /** Anthropic model id (typed from `@anthropic-ai/sdk`). Defaults to Sonnet. */ - model?: AnthropicModel; - /** Reasoning effort (`--effort`). Omit to use Claude Code's own default. */ - reasoningEffort?: ClaudeCodeEffort; - /** Override the pinned CLI version. */ - cliVersion?: string; - } = {}, -): AgentHarness { - return createCliAgent(claudeCodeRunner, claudeCodeParser, { - model: options.model ?? claudeCodeRunner.defaultModel, - reasoningEffort: options.reasoningEffort, - cliVersion: options.cliVersion, - }); -} - -/** OpenAI Codex as an `AgentHarness`. Runs in both modes, like Claude Code. */ -export function codexAgent( - options: { - /** OpenAI model id (typed from `openai`; any string accepted). */ - model?: CodexModel; - /** Reasoning effort (`model_reasoning_effort`). Omit to use Codex's default. */ - reasoningEffort?: CodexReasoningEffort; - /** Override the pinned CLI version. */ - cliVersion?: string; - } = {}, -): AgentHarness { - return createCliAgent(codexRunner, codexParser, { - model: options.model ?? codexRunner.defaultModel, - reasoningEffort: options.reasoningEffort, - cliVersion: options.cliVersion, - }); -} diff --git a/packages/core/src/agents/registry.ts b/packages/core/src/agents/registry.ts new file mode 100644 index 00000000..8f7b4ad2 --- /dev/null +++ b/packages/core/src/agents/registry.ts @@ -0,0 +1,34 @@ +/** + * Agent registry: the single list of CLI agents the harness knows about. + * + * Each agent module contributes one `AgentDefinition` (runner + parser). The + * registry derives the supported-agent list and the transcript-parser lookup + * from it, so adding an agent is one import + one array entry here (plus the + * agent's own `/` module). The run-time factories (`claudeCodeAgent`, + * `codexAgent`) are exported directly from those modules for use in experiments. + */ + +import type { AgentDefinition } from "./types.js"; +import type { AgentTranscriptParser } from "../parsers/types.js"; +import { claudeCodeDefinition } from "./claude-code/index.js"; +import { codexDefinition } from "./codex/index.js"; + +const AGENTS: AgentDefinition[] = [claudeCodeDefinition, codexDefinition]; + +const byId = new Map(AGENTS.map((agent) => [agent.runner.id, agent])); + +/** Agent ids with a registered transcript parser. */ +export function supportedParsers(): string[] { + return [...byId.keys()]; +} + +/** Look up a parser by agent id, or throw with the supported list. */ +export function createParser(agent: string): AgentTranscriptParser { + const definition = byId.get(agent); + if (!definition) { + throw new Error( + `Unknown agent parser: "${agent}". Supported: ${supportedParsers().join(", ")}`, + ); + } + return definition.parser; +} diff --git a/packages/core/src/runners/shared.ts b/packages/core/src/agents/shared.ts similarity index 100% rename from packages/core/src/runners/shared.ts rename to packages/core/src/agents/shared.ts diff --git a/packages/core/src/runners/types.ts b/packages/core/src/agents/types.ts similarity index 83% rename from packages/core/src/runners/types.ts rename to packages/core/src/agents/types.ts index b7392f1f..f2100642 100644 --- a/packages/core/src/runners/types.ts +++ b/packages/core/src/agents/types.ts @@ -1,10 +1,11 @@ /** - * Runner layer: how to install and drive a CLI coding agent inside a sandbox. + * Shared types for the agent layer. * * A runner owns only the CLI-execution strategy (install, exec, permission * flags, MCP-config format/placement). It does NOT parse transcripts — that's - * the parser layer (`../parsers`). `createCliAgent` (in `../cli-agent.ts`) - * composes a runner + a parser into an `AgentHarness`. + * the parser layer (`../parsers`). `createCliAgent` (in `./engine.ts`) composes + * a runner + a parser into an `AgentHarness`, and each agent's own module + * (`.//index.ts`) wires its runner + parser into a public factory. * * This mirrors `@supabase/agent-evals`'s runner/parser/agent split so each * concern that diverges per agent lives in exactly one place. @@ -13,6 +14,7 @@ import type { OutputConfig } from "@anthropic-ai/sdk/resources/messages"; import type { ReasoningEffort } from "openai/resources/shared"; import type { CommandResult, McpServerConfig } from "../index.js"; +import type { AgentTranscriptParser } from "../parsers/types.js"; /** * Reasoning effort for Claude Code's `--effort` flag. Derived from the Anthropic @@ -98,3 +100,15 @@ export interface AgentRunner { */ deriveStopReason?(raw: string | undefined, command: CommandResult): string; } + +/** + * Everything the harness needs to know about one CLI agent, bundled so each + * agent's module is its single source of truth. The agent id comes from + * `runner.id`. `agents/registry.ts` collects these to drive transcript parsing + * (`createParser`) and to list supported agents; the public run-time factory is + * exported separately by the agent's `index.ts`. + */ +export interface AgentDefinition { + runner: AgentRunner; + parser: AgentTranscriptParser; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d1b843a2..09332db9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -32,7 +32,7 @@ import { type ServerHandle, } from "@supabase-evals/platform-lite"; import type { CheckResult } from "./eval-metadata.js"; -import type { AgentSandbox } from "./cli-agent.js"; +import type { AgentSandbox } from "./agents/types.js"; import { isRecord } from "./json.js"; // Resolved lazily on first use, not at module load: `import.meta.resolve` is a @@ -73,20 +73,20 @@ export { } from "./eval-metadata.js"; export { parseEvalMarkdown } from "./eval-markdown.js"; // CLI agent harnesses (Claude Code, Codex, and the framework for adding more). -export { - claudeCodeAgent, - codexAgent, -} from "./cli-agent.js"; +export { createCliAgent } from "./agents/engine.js"; +export { claudeCodeAgent } from "./agents/claude-code/index.js"; +export { codexAgent } from "./agents/codex/index.js"; export type { AgentSandbox, AgentRunner, RunnerExecArgs, RunnerExecResult, + AgentDefinition, ClaudeCodeEffort, CodexReasoningEffort, -} from "./cli-agent.js"; +} from "./agents/types.js"; // Generic transcript vocabulary + parser layer used by CLI agents. -export { createParser, supportedParsers } from "./parsers/registry.js"; +export { createParser, supportedParsers } from "./agents/registry.js"; export { adaptTranscript } from "./parsers/adapt.js"; export type { AdaptedTranscript } from "./parsers/adapt.js"; export type { AgentTranscriptParser } from "./parsers/types.js"; diff --git a/packages/core/src/parsers/registry.ts b/packages/core/src/parsers/registry.ts deleted file mode 100644 index e489e434..00000000 --- a/packages/core/src/parsers/registry.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Transcript-parser registry: maps an agent id to its parser. - * - * Adding a new agent is one import + one map entry here (plus the parser file - * and a tool-name map in `shared/normalize.ts`). - */ - -import type { AgentTranscriptParser } from "./types.js"; -import { claudeCodeParser } from "./claude-code.js"; -import { codexParser } from "./codex.js"; - -const PARSERS: Record = { - "claude-code": claudeCodeParser, - codex: codexParser, -}; - -/** Agent ids with a registered transcript parser. */ -export function supportedParsers(): string[] { - return Object.keys(PARSERS); -} - -/** Look up a parser by agent id, or throw with the supported list. */ -export function createParser(agent: string): AgentTranscriptParser { - const parser = PARSERS[agent]; - if (!parser) { - throw new Error( - `Unknown agent parser: "${agent}". Supported: ${supportedParsers().join(", ")}`, - ); - } - return parser; -} diff --git a/packages/core/src/parsers/types.ts b/packages/core/src/parsers/types.ts index 84b818cb..843b5d75 100644 --- a/packages/core/src/parsers/types.ts +++ b/packages/core/src/parsers/types.ts @@ -5,8 +5,8 @@ * the agent CLI) into canonical `TranscriptEvent`s. It owns no I/O and no * agent-specific orchestration — that lives in the CLI agent harness. * - * To add a new agent, implement this interface in `parsers/.ts` and - * register it in `parsers/registry.ts`. Nothing else in the harness changes. + * To add a new agent, implement this interface in `agents//parser.ts` + * and register its definition in `agents/registry.ts`. Nothing else changes. */ import type { ParsedTranscript } from "../transcript/types.js";