From f2ed5bb4995c7aa9ede70abae40e8e679ced3f20 Mon Sep 17 00:00:00 2001 From: Jordi Enric Date: Tue, 21 Jul 2026 15:10:22 +0200 Subject: [PATCH 1/3] feat(eval): tool selection for query_logs (hosted) vs get_logs (self-hosted) Two MCP-mode evals asserting description-driven log tool selection: - hosted project -> query_logs - self-hosted project -> get_logs From supabase/mcp#333 review feedback (Rodriguespn). Blocked on query_logs shipping in a released @supabase/mcp-server-supabase version and bumping MCP_SERVER_VERSION in this repo; not runnable until then. --- .../EVAL.ts | 15 +++++++++++++++ .../PROMPT.md | 13 +++++++++++++ .../EVAL.ts | 18 ++++++++++++++++++ .../PROMPT.md | 15 +++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 evals/investigate-logs-002-tool-selection-hosted/EVAL.ts create mode 100644 evals/investigate-logs-002-tool-selection-hosted/PROMPT.md create mode 100644 evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts create mode 100644 evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md diff --git a/evals/investigate-logs-002-tool-selection-hosted/EVAL.ts b/evals/investigate-logs-002-tool-selection-hosted/EVAL.ts new file mode 100644 index 00000000..27d98f7d --- /dev/null +++ b/evals/investigate-logs-002-tool-selection-hosted/EVAL.ts @@ -0,0 +1,15 @@ +import type { CheckResult, ToolScorer } from "@supabase-evals/core"; + +const scorer: ToolScorer = async (ctx) => { + const usedQueryLogs = ctx.toolCalls.some((tc) => tc.endpoint === "query_logs"); + const usedGetLogs = ctx.toolCalls.some((tc) => tc.endpoint === "get_logs"); + + const checks: CheckResult[] = [ + { name: "used query_logs on a hosted project", passed: usedQueryLogs }, + { name: "did not fall back to get_logs", passed: !usedGetLogs }, + ]; + + return { passed: checks.every((check) => check.passed), checks }; +}; + +export default scorer; diff --git a/evals/investigate-logs-002-tool-selection-hosted/PROMPT.md b/evals/investigate-logs-002-tool-selection-hosted/PROMPT.md new file mode 100644 index 00000000..35566f33 --- /dev/null +++ b/evals/investigate-logs-002-tool-selection-hosted/PROMPT.md @@ -0,0 +1,13 @@ +--- +stage: investigate +suite: regression +interface: mcp +product: + - edge-functions +topic: + - observability +motivation: supabase/mcp#333 review (Rodriguespn) — verify the agent selects query_logs on hosted projects, driven by the log tool descriptions. +--- + +This project runs on hosted Supabase (production, on supabase.com). Pull the edge +function logs from the last 15 minutes so I can see what's been happening. diff --git a/evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts b/evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts new file mode 100644 index 00000000..d9e458da --- /dev/null +++ b/evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts @@ -0,0 +1,18 @@ +import type { CheckResult, ToolScorer } from "@supabase-evals/core"; + +const scorer: ToolScorer = async (ctx) => { + const usedGetLogs = ctx.toolCalls.some((tc) => tc.endpoint === "get_logs"); + const usedQueryLogs = ctx.toolCalls.some((tc) => tc.endpoint === "query_logs"); + + const checks: CheckResult[] = [ + { name: "used get_logs on a self-hosted project", passed: usedGetLogs }, + { + name: "did not attempt query_logs (unavailable on self-hosted)", + passed: !usedQueryLogs, + }, + ]; + + return { passed: checks.every((check) => check.passed), checks }; +}; + +export default scorer; diff --git a/evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md b/evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md new file mode 100644 index 00000000..c8ec522f --- /dev/null +++ b/evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md @@ -0,0 +1,15 @@ +--- +stage: investigate +suite: regression +interface: mcp +product: + - edge-functions +topic: + - observability + - self-hosting +motivation: supabase/mcp#333 review (Rodriguespn) — verify the agent falls back to get_logs on self-hosted projects, where ClickHouse-backed query_logs is unavailable. +--- + +This is a self-hosted Supabase instance (running via the self-hosting Docker +setup, not on supabase.com). Pull the edge function logs from the last 15 minutes +so I can see what's been happening. From 09e301c8711a1adc42ee75eb80be942aa051b147 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 12:35:31 +0000 Subject: [PATCH 2/3] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index f110b830..59f9c6be 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -315,6 +315,101 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/investigate-functions-001-546-resource-limit.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-002-tool-selection-hosted", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "regression", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "used query_logs on a hosted project", + "passed": false + }, + { + "name": "did not fall back to get_logs", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "This project runs on hosted Supabase (production, on supabase.com). Pull the edge\nfunction logs from the last 15 minutes so I can see what's been happening.", + "promptSourcePath": "evals/investigate-logs-002-tool-selection-hosted/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/investigate-logs-002-tool-selection-hosted.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-003-tool-selection-self-hosted", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability", + "self-hosting" + ], + "suite": "regression", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "used get_logs on a self-hosted project", + "passed": false + }, + { + "name": "did not attempt query_logs (unavailable on self-hosted)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "This is a self-hosted Supabase instance (running via the self-hosting Docker\nsetup, not on supabase.com). Pull the edge function logs from the last 15 minutes\nso I can see what's been happening.", + "promptSourcePath": "evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/investigate-logs-003-tool-selection-self-hosted.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", @@ -806,6 +901,91 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/investigate-functions-001-546-resource-limit.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-002-tool-selection-hosted", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "regression", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "used query_logs on a hosted project", + "passed": false + }, + { + "name": "did not fall back to get_logs", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "This project runs on hosted Supabase (production, on supabase.com). Pull the edge\nfunction logs from the last 15 minutes so I can see what's been happening.", + "promptSourcePath": "evals/investigate-logs-002-tool-selection-hosted/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-logs-002-tool-selection-hosted.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-003-tool-selection-self-hosted", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability", + "self-hosting" + ], + "suite": "regression", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "used get_logs on a self-hosted project", + "passed": false + }, + { + "name": "did not attempt query_logs (unavailable on self-hosted)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "This is a self-hosted Supabase instance (running via the self-hosting Docker\nsetup, not on supabase.com). Pull the edge function logs from the last 15 minutes\nso I can see what's been happening.", + "promptSourcePath": "evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-logs-003-tool-selection-self-hosted.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "regression", From 6b71232b48c29669c38cf8af149ddff314c0fe99 Mon Sep 17 00:00:00 2001 From: Jordi Enric Date: Thu, 6 Aug 2026 18:51:38 +0200 Subject: [PATCH 3/3] feat(eval): drop untestable self-hosted arm, retitle remaining scenario The self-hosted scenario assumed the evals harness could simulate a platform without ClickHouse-backed query_logs support, but supabaseMcpServer() always spawns @supabase/mcp-server-supabase against api-platform.ts, which implements queryLogs unconditionally regardless of --api-url. There is no way to exercise the get_logs-only arm through this harness; that behavior is already covered by supabase/mcp's own unit tests (server.test.ts, debugging-tools.test.ts). Keep and rename the hosted scenario, since query_logs is now the only listed logs tool on the platform-lite fixture (get_logs is hidden), so this verifies query_logs works end-to-end rather than a two-tool choice. --- .../EVAL.ts | 0 .../PROMPT.md | 12 ++++++++++++ .../PROMPT.md | 13 ------------- .../EVAL.ts | 18 ------------------ .../PROMPT.md | 15 --------------- 5 files changed, 12 insertions(+), 46 deletions(-) rename evals/{investigate-logs-002-tool-selection-hosted => investigate-logs-002-query-logs-tool-selection}/EVAL.ts (100%) create mode 100644 evals/investigate-logs-002-query-logs-tool-selection/PROMPT.md delete mode 100644 evals/investigate-logs-002-tool-selection-hosted/PROMPT.md delete mode 100644 evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts delete mode 100644 evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md diff --git a/evals/investigate-logs-002-tool-selection-hosted/EVAL.ts b/evals/investigate-logs-002-query-logs-tool-selection/EVAL.ts similarity index 100% rename from evals/investigate-logs-002-tool-selection-hosted/EVAL.ts rename to evals/investigate-logs-002-query-logs-tool-selection/EVAL.ts diff --git a/evals/investigate-logs-002-query-logs-tool-selection/PROMPT.md b/evals/investigate-logs-002-query-logs-tool-selection/PROMPT.md new file mode 100644 index 00000000..8a4c3b5e --- /dev/null +++ b/evals/investigate-logs-002-query-logs-tool-selection/PROMPT.md @@ -0,0 +1,12 @@ +--- +stage: investigate +suite: regression +interface: mcp +product: + - edge-functions +topic: + - observability +motivation: supabase/mcp#333 review (Rodriguespn) — verify the agent successfully uses query_logs to investigate edge function logs. get_logs is hidden from tools/list on this platform (it only shows up when the platform lacks ClickHouse-backed querying, which platform-lite here does not simulate), so this exercises query_logs end-to-end rather than a choice between two listed tools. +--- + +Pull the edge function logs from the last 15 minutes so I can see what's been happening. diff --git a/evals/investigate-logs-002-tool-selection-hosted/PROMPT.md b/evals/investigate-logs-002-tool-selection-hosted/PROMPT.md deleted file mode 100644 index 35566f33..00000000 --- a/evals/investigate-logs-002-tool-selection-hosted/PROMPT.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -stage: investigate -suite: regression -interface: mcp -product: - - edge-functions -topic: - - observability -motivation: supabase/mcp#333 review (Rodriguespn) — verify the agent selects query_logs on hosted projects, driven by the log tool descriptions. ---- - -This project runs on hosted Supabase (production, on supabase.com). Pull the edge -function logs from the last 15 minutes so I can see what's been happening. diff --git a/evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts b/evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts deleted file mode 100644 index d9e458da..00000000 --- a/evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { CheckResult, ToolScorer } from "@supabase-evals/core"; - -const scorer: ToolScorer = async (ctx) => { - const usedGetLogs = ctx.toolCalls.some((tc) => tc.endpoint === "get_logs"); - const usedQueryLogs = ctx.toolCalls.some((tc) => tc.endpoint === "query_logs"); - - const checks: CheckResult[] = [ - { name: "used get_logs on a self-hosted project", passed: usedGetLogs }, - { - name: "did not attempt query_logs (unavailable on self-hosted)", - passed: !usedQueryLogs, - }, - ]; - - return { passed: checks.every((check) => check.passed), checks }; -}; - -export default scorer; diff --git a/evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md b/evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md deleted file mode 100644 index c8ec522f..00000000 --- a/evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -stage: investigate -suite: regression -interface: mcp -product: - - edge-functions -topic: - - observability - - self-hosting -motivation: supabase/mcp#333 review (Rodriguespn) — verify the agent falls back to get_logs on self-hosted projects, where ClickHouse-backed query_logs is unavailable. ---- - -This is a self-hosted Supabase instance (running via the self-hosting Docker -setup, not on supabase.com). Pull the edge function logs from the last 15 minutes -so I can see what's been happening.