fix: executor experiment config, execution, and scoring

apps/framework/harness/run-eval.ts

@@ -158,13 +158,16 @@ async function runOne(
   expName: string,
   exp: ExperimentConfig,
   ev: EvalManifest
-): Promise<ScoreResult & { attempts: number; toolCalls: unknown[]; agentReport: string }> {
+): Promise<
+  ScoreResult & { attempts: number; toolCalls: unknown[]; agentReport: string; stoppedReason: string }
+> {
   const skillContext = loadSkills(exp.skills);
   const prompt = readFileSync(ev.promptPath, "utf8");
   const scorer = (await import(pathToFileURL(ev.evalPath).href)).default as ProjectScorer | ToolScorer;
   let last: ScoreResult = { passed: false, score: 0, notes: "no attempts" };
   let lastToolCalls: unknown[] = [];
   let lastAgentReport = "";
+  let lastStoppedReason = "not_started";
 
   for (let attempt = 1; attempt <= RUNS; attempt += 1) {
     if (ev.mode === "project") {
@@ -186,6 +189,7 @@ async function runOne(
 
       lastToolCalls = run.toolCalls;
       lastAgentReport = run.agentReport;
+      lastStoppedReason = run.stoppedReason;
       last = await (scorer as ProjectScorer)({
         workspace,
         projectResult: { build, vitest },
@@ -194,7 +198,13 @@ async function runOne(
       });
 
       if (STOP_ON_PASS && last.passed) {
-        return { ...last, attempts: attempt, toolCalls: run.toolCalls, agentReport: run.agentReport };
+        return {
+          ...last,
+          attempts: attempt,
+          toolCalls: run.toolCalls,
+          agentReport: run.agentReport,
+          stoppedReason: run.stoppedReason,
+        };
       }
       continue;
     }
@@ -219,21 +229,34 @@ async function runOne(
 
       lastToolCalls = run.toolCalls;
       lastAgentReport = run.agentReport;
+      lastStoppedReason = run.stoppedReason;
       last = await (scorer as ToolScorer)({
         ...session.scoringContext,
         toolCalls: run.toolCalls,
         agentReport: run.agentReport,
       });
 
       if (STOP_ON_PASS && last.passed) {
-        return { ...last, attempts: attempt, toolCalls: run.toolCalls, agentReport: run.agentReport };
+        return {
+          ...last,
+          attempts: attempt,
+          toolCalls: run.toolCalls,
+          agentReport: run.agentReport,
+          stoppedReason: run.stoppedReason,
+        };
       }
     } finally {
       await session.close();
     }
   }
 
-  return { ...last, attempts: RUNS, toolCalls: lastToolCalls, agentReport: lastAgentReport };
+  return {
+    ...last,
+    attempts: RUNS,
+    toolCalls: lastToolCalls,
+    agentReport: lastAgentReport,
+    stoppedReason: lastStoppedReason,
+  };
 }
 
 function normalizeExperimentName(s: string): string {

evals/detect-reliability-001-error-rate-spike/EVAL.ts

@@ -1,16 +1,27 @@
 import type { ToolScorer } from "@supabase-evals/core";
 
+function reportedErrorRateAboveFivePercent(report: string): boolean {
+  const percentMatches = report.matchAll(/\b(\d+(?:\.\d+)?)\s*%/g);
+  const hasPercentAboveFive = Array.from(percentMatches).some((match) => Number(match[1]) > 5);
+  if (hasPercentAboveFive) return true;
+
+  const decimalMatches = report.matchAll(/\b0\.(\d+)\b/g);
+  const hasDecimalAboveFivePercent = Array.from(decimalMatches).some(
+    (match) => Number(`0.${match[1]}`) > 0.05
+  );
+  if (hasDecimalAboveFivePercent) return true;
+
+  return /\b3\s*(?:errors?\s*)?(?:\/|out of)\s*20\b/i.test(report);
+}
+
 const scorer: ToolScorer = async (ctx) => {
   const report = ctx.agentReport ?? "";
 
   const checks = [
     { name: "named the affected function", ok: /process-payment/i.test(report) },
     {
       name: "reported an error rate above 5%",
-      ok:
-        /\b(1[5-9]|[2-9]\d)%\b/.test(report) ||
-        /\b0\.(1[5-9]|[2-9]\d)\b/.test(report) ||
-        /\b3\s*(?:\/|out of)\s*20\b/i.test(report),
+      ok: reportedErrorRateAboveFivePercent(report),
     },
     {
       name: "described the rate as elevated",

packages/core/package.json

@@ -13,6 +13,7 @@
     "@supabase/supabase-js": "^2.105.1",
     "@supabase-evals/platform-lite": "*",
     "ai": "^6.0.174",
+    "executor": "1.4.29",
     "typescript": "^5.6.0"
   }
 }