From 73be43a59958b28100e4a90dfb449dd24da2bf1c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 18:22:16 +0000 Subject: [PATCH] revert(results): restore eval results to state at 4e50650 The public evals site is rendering an incomplete row in the Benchmark table: "Claude Code / Opus 4.8 (high)" shows a Build score with em-dashes (no data) for Deploy, Investigate, and Resolve, yet still reports a 100% TOTAL. The row is backed by only two Build-stage records (build-cli-001-bootstrap-app and build-database-001-migrate-postgres-to-supabase), so the aggregate is misleading to anyone reading the public table. That data landed via #129 (9bd7320), which is the only commit to touch apps/web/src/data/eval-results.json since 4e50650. There is no experiments/claude-code-opus-4.8*.ts definition in the repo, so the row cannot be reproduced by an eval refresh - it exists only as checked-in JSON, and #129 also re-attributed those two Build evals away from the claude-code-opus-5 experiment and flipped the codex gpt-5.6-sol records for the same two evals from medium to low reasoning effort. This restores apps/web/src/data/eval-results.json byte-for-byte to its state at 4e50650. Scope is deliberately limited to that one file: regression suite results live separately in apps/web/src/data/regression-eval-results.json and are untouched. Note that this also drops the exported results #129 added for the three new supabase-js SDK evals (build-auth-001-email-password-flow, build-dataapi-001-relational-report, build-dataapi-002-restock-alert-report) across the other 14 benchmark rows. Their eval definitions remain in evals/, so the next benchmark refresh will repopulate them. Claude-Session: https://claude.ai/code/session_01TXk7jJgnnwPuancUutKkAg --- apps/web/src/data/eval-results.json | 3844 +++++---------------------- 1 file changed, 620 insertions(+), 3224 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 109ad607..5b28a57e 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -1,11 +1,11 @@ [ { - "experiment": "claude-code-opus-4.8", + "experiment": "claude-code-opus-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-4-8", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, "eval": "build-cli-001-bootstrap-app", @@ -95,12 +95,213 @@ "sourcePath": "claude-code-opus-5/build-cli-001-bootstrap-app.json" }, { - "experiment": "claude-code-opus-4.8", + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": true + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"declarative database schemas add column generate migration\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + } + ], + "resultChars": 66910 + } + ] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/build-cli-002-declarative-schema.json" + }, + { + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 10) from the queue" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgmq queues send read pop archive pgmq_public API\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" + } + ], + "resultChars": 39340 + }, + { + "source": "web_fetch", + "query": "How to create/schedule a cron job named X running every minute, and how to schedule a job that calls a Supabase Queue send / edge function. Show exact SQL syntax for cron.schedule and cron.unschedule, and any notes about naming or the pg_cron version. Include the full code examples verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cron/quickstart.md" + } + ], + "resultChars": 1475 + }, + { + "source": "web_fetch", + "query": "List any entries tagged breaking-change, and any entries related to Queues/pgmq, pg_cron/Cron, or Edge Functions runtime/deployment. Include dates and links.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 8295 + }, + { + "source": "web_fetch", + "query": "What exactly changed about the delay parameter behavior in pgmq 1.5.1 versus 1.4.4? Quote the details precisely, including any impact on send()/send_batch() and what values of delay are affected.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects" + } + ], + "resultChars": 659 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "claude-code-opus-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-4-8", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, "eval": "build-database-001-migrate-postgres-to-supabase", @@ -209,712 +410,54 @@ "sourcePath": "claude-code-opus-5/build-database-001-migrate-postgres-to-supabase.json" }, { - "experiment": "claude-code-opus-4.8-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-4-8", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-cli-001-bootstrap-app", + "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ - "database", - "data-api" + "edge-functions", + "auth", + "database" ], "topic": [ - "migrations", - "rls" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", + "name": "rejects missing auth", "passed": true, - "notes": "found 2 rows" + "notes": "status=401" }, { - "name": "row level security is enabled on todos", - "passed": true + "name": "user A reads own note", + "passed": true, + "notes": "status=200" }, { - "name": "a SELECT policy targets the authenticated role", - "passed": true + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "REST API returns no todos to anonymous requests", + "name": "user A cannot force-read user B note", "passed": true, - "notes": "0 rows" + "notes": "status=403" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "user B cannot force-read user A note", "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-opus-4.8-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-4-8", - "reasoningEffort": "high" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user a71b8f5f-f48f-424d-9f63-39951eef350a, signUp returned {\"userId\":\"a71b8f5f-f48f-424d-9f63-39951eef350a\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"a71b8f5f-f48f-424d-9f63-39951eef350a\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"signUp email password user metadata options data javascript\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - } - ], - "resultChars": 22309 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3161 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -iE \"^## |Breaking Change\" | grep -iE \"auth|supabase-js|signup|session|key\" | head -30", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 2427 - } - ] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-auth-001-email-password-flow.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"declarative database schemas add column generate migration\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - } - ], - "resultChars": 66910 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 10) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pgmq queues send read pop archive pgmq_public API\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - } - ], - "resultChars": 39340 - }, - { - "source": "web_fetch", - "query": "How to create/schedule a cron job named X running every minute, and how to schedule a job that calls a Supabase Queue send / edge function. Show exact SQL syntax for cron.schedule and cron.unschedule, and any notes about naming or the pg_cron version. Include the full code examples verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/quickstart.md" - } - ], - "resultChars": 1475 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and any entries related to Queues/pgmq, pg_cron/Cron, or Edge Functions runtime/deployment. Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 8295 - }, - { - "source": "web_fetch", - "query": "What exactly changed about the delay parameter behavior in pgmq 1.5.1 versus 1.4.4? Quote the details precisely, including any impact on send()/send_batch() and what values of delay are affected.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog/39378-potential-breaking-change-in-pgmq-from-1-4-4-to-1-5-1-and-temporary-halt-on-upgrade-for-existing-projects" - } - ], - "resultChars": 659 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and any entries about API keys (publishable/secret keys), supabase-js releases, or PostgREST behavior changes. Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3218 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"server-side supabase-js secret key service role createClient node\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/astrojs", - "title": "Use Supabase with Astro" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - } - ], - "resultChars": 54410 - }, - { - "source": "web_fetch", - "query": "How should the new secret key (sb_secret_...) be used from server-side code with supabase-js? Does it go in the createClient key argument? Any notes about it bypassing RLS, and about not using it in browsers?", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/api-keys.md" - } - ], - "resultChars": 1176 - } - ] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-dataapi-001-relational-report.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"secret key service_role server-side createClient API keys sb_secret\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - } - ], - "resultChars": 95806 - } - ] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "claude-code-opus-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=403" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=403" + "notes": "status=403" } ], "skills": { @@ -2712,60 +2255,50 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-auth-001-email-password-flow", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "auth", - "database" + "database", + "data-api" ], "topic": [ - "sdk", + "migrations", "rls" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user e7a03db9-3c5a-4ddf-a197-00b92683665a, signUp returned {\"userId\":\"e7a03db9-3c5a-4ddf-a197-00b92683665a\"}" + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true }, { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" + "name": "todos table is created by a migration file", + "passed": true }, { - "name": "wrong password is rejected gracefully (no throw, no session)", + "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" + "notes": "found 2 rows" }, { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"e7a03db9-3c5a-4ddf-a197-00b92683665a\"}" + "name": "row level security is enabled on todos", + "passed": true }, { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + "name": "a SELECT policy targets the authenticated role", + "passed": true }, { - "name": "app code does not use the secret / service-role key", + "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "no secret-key references found" + "notes": "0 rows" }, { - "name": "implementation uses @supabase/supabase-js", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "imports found in: app/src/auth.mjs" + "notes": "2 rows" } ], "skills": { @@ -2775,10 +2308,10 @@ "docs": { "calls": [] }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-auth-001-email-password-flow.json" + "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" }, { "experiment": "claude-code-opus-5-no-skills", @@ -2947,105 +2480,37 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-002-restock-alert-report", + "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ - "data-api", "database" ], "topic": [ - "sdk" + "migrations" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true }, { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" + "name": "foreign key constraints survived the restore", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" + "name": "tasks_team_status_idx index survived the restore", + "passed": true }, { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -3055,10 +2520,10 @@ "docs": { "calls": [] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-002-restock-alert-report.json" + "sourcePath": "claude-code-opus-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { "experiment": "claude-code-opus-5-no-skills", @@ -4206,88 +3671,6 @@ "attempts": 1, "sourcePath": "claude-code-opus-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user eb698502-60bc-40af-bc9d-e1d8bd7185b6, signUp returned {\"userId\":\"eb698502-60bc-40af-bc9d-e1d8bd7185b6\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"eb698502-60bc-40af-bc9d-e1d8bd7185b6\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-auth-001-email-password-flow.json" - }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -4452,208 +3835,9 @@ "notes": "queue depth 0 -> 1" }, { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule pgmq queue send message example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - } - ], - "resultChars": 55834 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function supabase-js service role client import example Deno.serve\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" - } - ], - "resultChars": 26350 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions default secrets SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY automatically populated local\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 28165 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", + "name": "process-tasks function drains the queue", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -4666,12 +3850,81 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_cron schedule pgmq queue send message example\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + } + ], + "resultChars": 55834 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function supabase-js service role client import example Deno.serve\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" + } + ], + "resultChars": 26350 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Edge Functions default secrets SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY automatically populated local\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 28165 + } + ] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-dataapi-002-restock-alert-report.json" + "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" }, { "experiment": "claude-code-sonnet-5", @@ -6446,83 +5699,6 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user ae26a6fb-04fb-47ad-aa91-e1289c754f86, signUp returned {\"userId\":\"ae26a6fb-04fb-47ad-aa91-e1289c754f86\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"ae26a6fb-04fb-47ad-aa91-e1289c754f86\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-auth-001-email-password-flow.json" - }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -6538,148 +5714,44 @@ "database", "data-api" ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" + "topic": [ + "migrations", + "rls" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "schedule='* * * * *', active=true" + "notes": "found 2 rows" }, { - "name": "cron command enqueues to the 'tasks' queue", + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "0 rows" }, { - "name": "process-tasks function drains the queue", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" + "notes": "2 rows" } ], "skills": { @@ -6689,10 +5761,10 @@ "docs": { "calls": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-001-bootstrap-app.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6703,44 +5775,34 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-dataapi-001-relational-report", + "eval": "build-cli-002-declarative-schema", "stage": "build", "product": [ - "data-api", "database" ], "topic": [ - "sdk" + "declarative-schema", + "migrations" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + "name": "supabase db diff used to generate the migration", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" + "name": "schema file updated to include description column", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "name": "a new migration was generated for the change", + "passed": true }, { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "name": "description column exists in the live database", + "passed": true } ], "skills": { @@ -6750,10 +5812,10 @@ "docs": { "calls": [] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6764,44 +5826,36 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-dataapi-002-restock-alert-report", + "eval": "build-cli-003-pg-cron-queue-workflow", "stage": "build", "product": [ - "data-api", - "database" + "database", + "edge-functions", + "cron", + "queues" ], "topic": [ + "sql", "sdk" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + "notes": "schedule='* * * * *', active=true" }, { - "name": "tables stay locked down (publishable key reads nothing)", + "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "notes": "queue depth 0 -> 1" }, { - "name": "report queries via the Data API, not raw SQL", + "name": "process-tasks function drains the queue", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -6811,10 +5865,10 @@ "docs": { "calls": [] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-002-restock-alert-report.json" + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -7805,211 +6859,56 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user e669932c-47e9-48f1-bc47-b48321540cb8, signUp returned {\"userId\":\"e669932c-47e9-48f1-bc47-b48321540cb8\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"e669932c-47e9-48f1-bc47-b48321540cb8\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": false, - "notes": "{\"error\":\"No user is signed in.\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"sign up email password auth/v1 token grant_type password profiles select authenticated REST current user session\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - } - ], - "resultChars": 89847 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"auth signup email password user_metadata display_name signInWithPassword getUser supabase-js\", limit: 10) { nodes { __typename ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" - } - ], - "resultChars": 249983 - } - ] + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -8199,244 +7098,14 @@ "notes": "schedule='* * * * *', active=true" }, { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 1 -> 2" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pg_cron schedule cron jobs Supabase local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - } - ], - "resultChars": 94119 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq pop send delete read edge functions local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - } - ], - "resultChars": 18872 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase edge functions local development deploy serve deno\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 47492 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6745 - } - ] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 1 -> 2" }, { - "name": "report queries via the Data API, not raw SQL", + "name": "process-tasks function drains the queue", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "notes": "function removed the seeded message (id 4) from the queue" } ], "skills": { @@ -8445,17 +7114,101 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pg_cron schedule cron jobs Supabase local development\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", + "title": "Send Email Hook" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + } + ], + "resultChars": 94119 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Queues pgmq pop send delete read edge functions local development\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + }, + { + "url": "https://supabase.com/docs/guides/functions/architecture", + "title": "Edge Functions Architecture" + } + ], + "resultChars": 18872 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase edge functions local development deploy serve deno\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-environment", + "title": "Development Environment" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + } + ], + "resultChars": 47492 + } + ] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report.json" + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -10748,83 +9501,6 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 8bd4dfe4-46d9-4631-8e00-f9d3972f1d91, signUp returned {\"userId\":\"8bd4dfe4-46d9-4631-8e00-f9d3972f1d91\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"8bd4dfe4-46d9-4631-8e00-f9d3972f1d91\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-auth-001-email-password-flow.json" - }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -11025,134 +9701,12 @@ "query": "site:supabase.com/docs edge function config.toml verify_jwt false", "pages": [] } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] + ] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-002-restock-alert-report.json" + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", @@ -12413,183 +10967,6 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user d6d4378b-47b7-404d-9832-307530f2207f, signUp returned {\"userId\":\"d6d4378b-47b7-404d-9832-307530f2207f\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"d6d4378b-47b7-404d-9832-307530f2207f\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- auth.mjs ---' && sed -n '1,240p' app/src/auth.mjs && printf '%s\\\\n' '--- package.json ---' && cat app/package.json && printf '%s\\\\n' '--- migration ---' && cat supabase/migrations/0000_profiles_schema.sql && printf '%s\\\\n' '--- config relevant ---' && rg -n 'email|confirm|site_url|api|db' supabase/config.toml | sed -n '1,160p' && printf '%s\\\\n' '--- changelog auth hits ---' && curl -fsSL https://supabase.com/changelog.md | rg -i -n 'breaking-change|auth|signInWithPassword|signUp' | sed -n '1,100p'\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 22069 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword select single auth getUser current session\", limit: 8) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", - "title": "Login with GitHub" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", - "title": "Login with Zoom" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" - } - ], - "resultChars": 121357 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data user_metadata\", limit: 5) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - } - ], - "resultChars": 24012 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript auth signInWithPassword email password\", limit: 3) { nodes { title href content ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - } - ], - "resultChars": 49059 - } - ] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ @@ -13326,277 +11703,72 @@ }, { "url": "https://supabase.com/docs/guides/database/functions#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#advanced-logging", - "title": "Advanced logging" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#function-privileges", - "title": "Function privileges" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#security-definer-vs-invoker", - "title": "Security definer vs invoker" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#database-functions-vs-edge-functions", - "title": "Database Functions vs Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#suggestions", - "title": "Suggestions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#passing-parameters", - "title": "Passing parameters" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#returning-data-sets", - "title": "Returning data sets" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#simple-functions", - "title": "Simple functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#getting-started", - "title": "Getting started" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#quick-demo", - "title": "Quick demo" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#debugging-functions", - "title": "Debugging functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#using-database-functions-to-call-an-external-api", - "title": "Using Database Functions to call an external API" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#call-database-functions-using-javascript", - "title": "Call Database Functions using JavaScript" - }, - { - "url": "https://supabase.com/docs/reference/javascript/rpc" - } - ], - "resultChars": 288356 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript nested relationships select foreign key joins order customer service role backend\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "title": "Resources" + }, { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" + "url": "https://supabase.com/docs/guides/database/functions#advanced-logging", + "title": "Advanced logging" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/database/functions#function-privileges", + "title": "Function privileges" }, { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" + "url": "https://supabase.com/docs/guides/database/functions#security-definer-vs-invoker", + "title": "Security definer vs invoker" }, { - "url": "https://supabase.com/docs/guides/integrations", - "title": "Integrations" + "url": "https://supabase.com/docs/guides/database/functions#database-functions-vs-edge-functions", + "title": "Database Functions vs Edge Functions" }, { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" - } - ], - "resultChars": 37732 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog relevant tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking-change|postgrest|supabase-js|data api' || true; printf '%s\\\\n' '--- app/report.mjs ---'; sed -n '1,240p' app/report.mjs; printf '%s\\\\n' '--- app/package.json ---'; cat app/package.json; printf '%s\\\\n' '--- migration ---'; sed -n '1,300p' supabase/migrations/0000_orders_schema.sql; printf '%s\\\\n' '--- config ---'; sed -n '1,240p' supabase/config.toml; printf '%s\\\\n' '--- status ---'; supabase status\"", - "pages": [ + "url": "https://supabase.com/docs/guides/database/functions#suggestions", + "title": "Suggestions" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1507 - } - ] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select nested foreign tables service role secret key createClient server backend\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/functions#passing-parameters", + "title": "Passing parameters" + }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/database/functions#returning-data-sets", + "title": "Returning data sets" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + "url": "https://supabase.com/docs/guides/database/functions#simple-functions", + "title": "Simple functions" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-regenerateclientsecret" + "url": "https://supabase.com/docs/guides/database/functions#getting-started", + "title": "Getting started" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" - } - ], - "resultChars": 14444 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- app/package.json ---'; cat app/package.json; printf '%s\\\\n' '--- migration ---'; cat supabase/migrations/0000_inventory_schema.sql; printf '%s\\\\n' '--- git status ---'; git status --short; printf '%s\\\\n' '--- supabase status ---'; supabase status -o env; printf '%s\\\\n' '--- relevant changelog lines ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"breaking-change|javascript|api key|secret key|postgrest\\\" | head -80\"", - "pages": [ + "url": "https://supabase.com/docs/guides/database/functions#quick-demo", + "title": "Quick demo" + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/database/functions#debugging-functions", + "title": "Debugging functions" + }, + { + "url": "https://supabase.com/docs/guides/database/functions#using-database-functions-to-call-an-external-api", + "title": "Using Database Functions to call an external API" + }, + { + "url": "https://supabase.com/docs/guides/database/functions#call-database-functions-using-javascript", + "title": "Call Database Functions using JavaScript" + }, + { + "url": "https://supabase.com/docs/reference/javascript/rpc" } ], - "resultChars": 8069 + "resultChars": 288356 } ] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" }, { "experiment": "codex-gpt-5.6", @@ -13605,7 +11777,7 @@ "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" + "reasoningEffort": "medium" }, "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", @@ -15345,191 +13517,36 @@ "skills": { "available": [ "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant team workspace membership policies auth.uid() security definer\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - } - ], - "resultChars": 38609 - } - ] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 4c1f81ac-8d75-4942-a45c-acaba38a767c, signUp returned {\"userId\":\"4c1f81ac-8d75-4942-a45c-acaba38a767c\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"4c1f81ac-8d75-4942-a45c-acaba38a767c\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient auth signUp email password options data signInWithPassword getUser select single browser session persistence\", limit: 6) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side", - "title": "Server-Side Rendering" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - } - ], - "resultChars": 136587 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data signInWithPassword select maybeSingle single auth getUser\", limit: 10) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-identity-linking", - "title": "Identity Linking" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" - }, + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security multi tenant team workspace membership policies auth.uid() security definer\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" } ], - "resultChars": 102688 + "resultChars": 38609 } ] }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-auth-001-email-password-flow.json" + "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -15538,7 +13555,7 @@ "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" + "reasoningEffort": "medium" }, "eval": "build-cli-001-bootstrap-app", "stage": "build", @@ -16310,181 +14327,6 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript service role select related tables pagination range\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-listclients" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - } - ], - "resultChars": 21553 - } - ] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript supabase-js select nested relationships service role Node createClient\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-deleteclient" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-listclients" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - } - ], - "resultChars": 8566 - } - ] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ @@ -18020,133 +15862,13 @@ "available": [], "loaded": [] }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 9d354523-f72e-4ae3-818b-97124b969105, signUp returned {\"userId\":\"9d354523-f72e-4ae3-818b-97124b969105\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"9d354523-f72e-4ae3-818b-97124b969105\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -c 4000", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3973 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"supabase-js auth signUp with email password user metadata display name signInWithPassword getSession\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - } - ], - "resultChars": 123195 - } - ] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-auth-001-email-password-flow.json" + "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { "experiment": "opencode-kimi-k3", @@ -18409,136 +16131,6 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-dataapi-001-relational-report.json" - }, - { - "experiment": "opencode-kimi-k3", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3/build-dataapi-002-restock-alert-report.json" - }, { "experiment": "opencode-kimi-k3", "experimentSuite": "benchmark", @@ -19950,82 +17542,6 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 1d1b3e06-0667-40d6-b4b2-eb0a732941dd, signUp returned {\"userId\":\"1d1b3e06-0667-40d6-b4b2-eb0a732941dd\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"1d1b3e06-0667-40d6-b4b2-eb0a732941dd\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3-no-skills/build-auth-001-email-password-flow.json" - }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", @@ -20223,126 +17739,6 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3-no-skills/build-dataapi-001-relational-report.json" - }, - { - "experiment": "opencode-kimi-k3-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", - "product": [ - "data-api", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" - }, - { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - }, - { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "opencode-kimi-k3-no-skills/build-dataapi-002-restock-alert-report.json" - }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills",