diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 29c574bb..ab4d4e35 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -158,13 +158,16 @@ async function runOne( expName: string, exp: ExperimentConfig, ev: EvalManifest -): Promise { +): Promise< + ScoreResult & { attempts: number; toolCalls: unknown[]; agentReport: string; stoppedReason: string } +> { const skillContext = loadSkills(exp.skills); const prompt = readFileSync(ev.promptPath, "utf8"); const scorer = (await import(pathToFileURL(ev.evalPath).href)).default as ProjectScorer | ToolScorer; let last: ScoreResult = { passed: false, score: 0, notes: "no attempts" }; let lastToolCalls: unknown[] = []; let lastAgentReport = ""; + let lastStoppedReason = "not_started"; for (let attempt = 1; attempt <= RUNS; attempt += 1) { if (ev.mode === "project") { @@ -186,6 +189,7 @@ async function runOne( lastToolCalls = run.toolCalls; lastAgentReport = run.agentReport; + lastStoppedReason = run.stoppedReason; last = await (scorer as ProjectScorer)({ workspace, projectResult: { build, vitest }, @@ -194,7 +198,13 @@ async function runOne( }); if (STOP_ON_PASS && last.passed) { - return { ...last, attempts: attempt, toolCalls: run.toolCalls, agentReport: run.agentReport }; + return { + ...last, + attempts: attempt, + toolCalls: run.toolCalls, + agentReport: run.agentReport, + stoppedReason: run.stoppedReason, + }; } continue; } @@ -219,6 +229,7 @@ async function runOne( lastToolCalls = run.toolCalls; lastAgentReport = run.agentReport; + lastStoppedReason = run.stoppedReason; last = await (scorer as ToolScorer)({ ...session.scoringContext, toolCalls: run.toolCalls, @@ -226,14 +237,26 @@ async function runOne( }); if (STOP_ON_PASS && last.passed) { - return { ...last, attempts: attempt, toolCalls: run.toolCalls, agentReport: run.agentReport }; + return { + ...last, + attempts: attempt, + toolCalls: run.toolCalls, + agentReport: run.agentReport, + stoppedReason: run.stoppedReason, + }; } } finally { await session.close(); } } - return { ...last, attempts: RUNS, toolCalls: lastToolCalls, agentReport: lastAgentReport }; + return { + ...last, + attempts: RUNS, + toolCalls: lastToolCalls, + agentReport: lastAgentReport, + stoppedReason: lastStoppedReason, + }; } function normalizeExperimentName(s: string): string { diff --git a/evals/detect-reliability-001-error-rate-spike/EVAL.ts b/evals/detect-reliability-001-error-rate-spike/EVAL.ts index 57d427e8..3d257bdd 100644 --- a/evals/detect-reliability-001-error-rate-spike/EVAL.ts +++ b/evals/detect-reliability-001-error-rate-spike/EVAL.ts @@ -1,5 +1,19 @@ import type { ToolScorer } from "@supabase-evals/core"; +function reportedErrorRateAboveFivePercent(report: string): boolean { + const percentMatches = report.matchAll(/\b(\d+(?:\.\d+)?)\s*%/g); + const hasPercentAboveFive = Array.from(percentMatches).some((match) => Number(match[1]) > 5); + if (hasPercentAboveFive) return true; + + const decimalMatches = report.matchAll(/\b0\.(\d+)\b/g); + const hasDecimalAboveFivePercent = Array.from(decimalMatches).some( + (match) => Number(`0.${match[1]}`) > 0.05 + ); + if (hasDecimalAboveFivePercent) return true; + + return /\b3\s*(?:errors?\s*)?(?:\/|out of)\s*20\b/i.test(report); +} + const scorer: ToolScorer = async (ctx) => { const report = ctx.agentReport ?? ""; @@ -7,10 +21,7 @@ const scorer: ToolScorer = async (ctx) => { { name: "named the affected function", ok: /process-payment/i.test(report) }, { name: "reported an error rate above 5%", - ok: - /\b(1[5-9]|[2-9]\d)%\b/.test(report) || - /\b0\.(1[5-9]|[2-9]\d)\b/.test(report) || - /\b3\s*(?:\/|out of)\s*20\b/i.test(report), + ok: reportedErrorRateAboveFivePercent(report), }, { name: "described the rate as elevated", diff --git a/package-lock.json b/package-lock.json index 73ae6257..a1e7bb5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6501,6 +6501,140 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, + "node_modules/executor": { + "version": "1.4.29", + "resolved": "https://registry.npmjs.org/executor/-/executor-1.4.29.tgz", + "integrity": "sha512-gI0oi95pU5K5M0SiDyNJAXFNeNZPR2Ru0VMgByWttHIUFZtKjvjDvCYhn9J1jf7Rp2rZTQI3baqT2VIOEEWRVQ==", + "license": "MIT", + "bin": { + "executor": "bin/executor" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "executor-darwin-arm64": "npm:executor@1.4.29-darwin-arm64", + "executor-darwin-x64": "npm:executor@1.4.29-darwin-x64", + "executor-linux-arm64": "npm:executor@1.4.29-linux-arm64", + "executor-linux-arm64-musl": "npm:executor@1.4.29-linux-arm64-musl", + "executor-linux-x64": "npm:executor@1.4.29-linux-x64", + "executor-linux-x64-musl": "npm:executor@1.4.29-linux-x64-musl", + "executor-windows-arm64": "npm:executor@1.4.29-windows-arm64", + "executor-windows-x64": "npm:executor@1.4.29-windows-x64" + } + }, + "node_modules/executor-darwin-arm64": { + "name": "executor", + "version": "1.4.29-darwin-arm64", + "resolved": "https://registry.npmjs.org/executor/-/executor-1.4.29-darwin-arm64.tgz", + "integrity": "sha512-0c2C7hMRZx/teNzI3j8BhSdm3CfbnStAY/IC50tsa27yACvp3R0+fOO+apqkb8eZJWsrQT/E6unTXc0+0OoXWg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/executor-darwin-x64": { + "name": "executor", + "version": "1.4.29-darwin-x64", + "resolved": "https://registry.npmjs.org/executor/-/executor-1.4.29-darwin-x64.tgz", + "integrity": "sha512-VyQ9MiE5y5WfB3JZrzL5ImQ63C0DNt0/T7/uAXIaJ3iQo+8WGNWzv3rlTDI5zmvmc+ZGBlplql249HQfGHwNTQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/executor-linux-arm64": { + "name": "executor", + "version": "1.4.29-linux-arm64", + "resolved": "https://registry.npmjs.org/executor/-/executor-1.4.29-linux-arm64.tgz", + "integrity": "sha512-sbG90gQ8N9yQqZtvDeZBhToVrp4MBPzUnawl87gZVLYcjBDpAF1g3JFZ003gMXqS8vdfw3Zl9iyV4xClLvVq9A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/executor-linux-arm64-musl": { + "name": "executor", + "version": "1.4.29-linux-arm64-musl", + "resolved": "https://registry.npmjs.org/executor/-/executor-1.4.29-linux-arm64-musl.tgz", + "integrity": "sha512-2AW3r1g3cILLvOZi1jX/4XRIR3JPvwgUisCYFgY36VVx3GswgOBkT3vimy4G+9hSvUUOtY4YDKFJAvYW/2Wjdg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/executor-linux-x64": { + "name": "executor", + "version": "1.4.29-linux-x64", + "resolved": "https://registry.npmjs.org/executor/-/executor-1.4.29-linux-x64.tgz", + "integrity": "sha512-br8MvFD3pncyaEWxKmOO2rp3QF8YYao70dvaIRg6Re2vG0rz18cEJ91kS394R0dRHUGCphkF+oY7kSFYAzVS/w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/executor-linux-x64-musl": { + "name": "executor", + "version": "1.4.29-linux-x64-musl", + "resolved": "https://registry.npmjs.org/executor/-/executor-1.4.29-linux-x64-musl.tgz", + "integrity": "sha512-OifvSvvx6DWolSYJt6l9RQwTAMeMC+JVbP/dOPlXMewugeZNLEttRcCG+PabnOBwCAtmBaB4fRmU45NWWs/ZEA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/executor-windows-arm64": { + "name": "executor", + "version": "1.4.29-windows-arm64", + "resolved": "https://registry.npmjs.org/executor/-/executor-1.4.29-windows-arm64.tgz", + "integrity": "sha512-TlY4Mclqrh60NGGgf4/hCkMi9+BBYVENfo5KBjEWNrGvDTVhmaggSkgqQqQKCtvYlG0ZRicCc/ArWEUCuNJs4w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/executor-windows-x64": { + "name": "executor", + "version": "1.4.29-windows-x64", + "resolved": "https://registry.npmjs.org/executor/-/executor-1.4.29-windows-x64.tgz", + "integrity": "sha512-wMkmcXsbXYuPet9K6RJSy+FJWPYujvODiVyVtjSvG6kgKn/9IzVikr4pxloe+t2Pa8fOdLFrAzpyXZxWRJC+Dw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -11240,6 +11374,7 @@ "@supabase-evals/platform-lite": "*", "@supabase/supabase-js": "^2.105.1", "ai": "^6.0.174", + "executor": "1.4.29", "typescript": "^5.6.0" } }, diff --git a/packages/core/package.json b/packages/core/package.json index 08c2e691..ee14877a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -13,6 +13,7 @@ "@supabase/supabase-js": "^2.105.1", "@supabase-evals/platform-lite": "*", "ai": "^6.0.174", + "executor": "1.4.29", "typescript": "^5.6.0" } } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7761e090..2bbe31cb 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,8 +1,12 @@ import vm from "node:vm"; -import { createHmac } from "node:crypto"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import { createHash, createHmac } from "node:crypto"; +import { execFile } from "node:child_process"; +import { createServer } from "node:net"; +import { promisify } from "node:util"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; import { createClient, type SupabaseClient } from "@supabase/supabase-js"; import { createMCPClient } from "@ai-sdk/mcp"; import { Experimental_StdioMCPTransport as StdioMCPTransport } from "@ai-sdk/mcp/mcp-stdio"; @@ -17,6 +21,13 @@ import { type ServerHandle, } from "@supabase-evals/platform-lite"; +const EXECUTOR_BIN = join( + dirname(fileURLToPath(import.meta.resolve("executor/package.json"))), + "bin", + "executor" +); +const execFileAsync = promisify(execFile); + export type { SupabaseClient }; export type { ManagementApiClient }; @@ -346,44 +357,150 @@ export function executorMcpServer(): McpServerDefinition { async createConfig({ apiUrl, accessToken }) { const scopeDir = mkdtempSync(join(tmpdir(), "eval-executor-scope-")); const dataDir = mkdtempSync(join(tmpdir(), "eval-executor-data-")); + // Keep source registration isolated from any user daemon already listening on 4788. + const daemonUrl = `http://localhost:${await getAvailablePort()}`; - writeFileSync( - join(scopeDir, "executor.jsonc"), - JSON.stringify({ - sources: [ - { - kind: "openapi", - namespace: "platform", - spec: `${apiUrl}/openapi.json`, - baseUrl: apiUrl, - headers: { Authorization: `Bearer ${accessToken}` }, - }, - ], - }) - ); + try { + // executor.jsonc#sources is no longer replayed; see https://github.com/RhysSullivan/executor/pull/807. + await addExecutorOpenApiSource({ + scopeDir, + dataDir, + daemonUrl, + spec: `${apiUrl}/openapi.json`, + baseUrl: apiUrl, + namespace: "platform", + headers: { Authorization: `Bearer ${accessToken}` }, + }); + } catch (err) { + await cleanupExecutorResources({ scopeDir, dataDir, daemonUrl }); + throw err; + } return { config: { - command: "executor", - args: ["mcp", "--scope", scopeDir], + command: process.execPath, + args: [EXECUTOR_BIN, "mcp", "--scope", scopeDir], env: { EXECUTOR_DATA_DIR: dataDir }, }, cleanup: async () => { - const errors: unknown[] = []; - for (const dir of [scopeDir, dataDir]) { - try { - rmSync(dir, { recursive: true, force: true }); - } catch (err) { - errors.push(err); - } - } - throwIfCloseErrors(errors, "failed to close executor MCP resources"); + await cleanupExecutorResources({ scopeDir, dataDir, daemonUrl }); }, }; }, }; } +async function addExecutorOpenApiSource(input: { + scopeDir: string; + dataDir: string; + daemonUrl: string; + spec: string; + baseUrl: string; + namespace: string; + headers: Record; +}) { + const sourceConfig = { + scope: executorScopeId(input.scopeDir), + spec: input.spec, + namespace: input.namespace, + baseUrl: input.baseUrl, + headers: input.headers, + }; + + const { stdout } = await execFileAsync( + process.execPath, + [ + EXECUTOR_BIN, + "call", + "executor", + "openapi", + "addSource", + JSON.stringify(sourceConfig), + "--scope", + input.scopeDir, + "--base-url", + input.daemonUrl, + ], + { env: executorEnv(input.dataDir) } + ); + + const executionId = extractExecutorExecutionId(stdout); + if (!executionId) return; + + await execFileAsync( + process.execPath, + [ + EXECUTOR_BIN, + "resume", + "--execution-id", + executionId, + "--action", + "accept", + "--content", + "{}", + "--scope", + input.scopeDir, + "--base-url", + input.daemonUrl, + ], + { env: executorEnv(input.dataDir) } + ); +} + +async function cleanupExecutorResources(input: { + scopeDir: string; + dataDir: string; + daemonUrl: string; +}): Promise { + const errors: unknown[] = []; + try { + await execFileAsync( + process.execPath, + [EXECUTOR_BIN, "daemon", "stop", "--base-url", input.daemonUrl], + { env: executorEnv(input.dataDir) } + ); + } catch (err) { + errors.push(err); + } + + for (const dir of [input.scopeDir, input.dataDir]) { + try { + rmSync(dir, { recursive: true, force: true }); + } catch (err) { + errors.push(err); + } + } + throwIfCloseErrors(errors, "failed to close executor MCP resources"); +} + +function executorEnv(dataDir: string): Record { + return { ...definedEnv(process.env), EXECUTOR_DATA_DIR: dataDir }; +} + +function extractExecutorExecutionId(stdout: string): string | undefined { + return stdout.match(/(?:^|\s)executionId:\s*(\S+)/)?.[1]; +} + +function executorScopeId(scopeDir: string): string { + const folder = basename(scopeDir) || scopeDir; + const hash = createHash("sha256").update(scopeDir).digest("hex").slice(0, 8); + return `${folder}-${hash}`; +} + +async function getAvailablePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.on("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + server.close(() => { + if (typeof address === "object" && address) resolve(address.port); + else reject(new Error("failed to allocate executor daemon port")); + }); + }); + }); +} + export const ACCESS_TOKEN = "eval-token"; export const MCP_SERVER_VERSION = "0.8.1"; @@ -456,7 +573,9 @@ export async function bootPlatformBackend(opts: { } } -const MAX_STEPS = 30; +// Executor-backed API calls may require a separate resume step after the +// initial tool call, so mutation-heavy evals need more headroom than direct MCP. +const MAX_STEPS = 60; const MAX_OUTPUT_TOKENS = 4096; const RUNTIME_URL = "http://supabase-evals.local";