feat(eval): eval harness + first failure-mode fixtures

.gitignore

@@ -141,3 +141,6 @@ dist
 vite.config.js.timestamp-*
 vite.config.ts.timestamp-*
 .vite/
+
+# Eval outputs (scorecard is regenerated every run; history is recorded deliberately)
+eval/scorecard.json

TRACKER.md

@@ -5,8 +5,8 @@
 ## Status
 
 - **Current phase:** 0 — Foundations
-- **Next step:** 0.2 — Eval harness + first fixtures
-- **Done:** 0.1
+- **Next step:** 0.3 — Graph storage & versioning
+- **Done:** 0.1, 0.2
 - **Gates passed:** none yet (Gate 0 completes with 0.4)
 
 ## What CodeRadar is
@@ -58,7 +58,7 @@ The v0.1 schema is definition-only, which is *wrong* for C1 — fix before build
 - Demo-app scan shows `UserCard` with 1 definition + 1 instance (parent `UserList`).
 - `matchComponentsByText` / `traceLineage` return `QueryResult` envelopes.
 
-### [ ] 0.2 Eval harness + first fixtures
+### [x] 0.2 Eval harness + first fixtures
 **Failure modes:** infrastructure for all; first fixtures C1, A4
 **Build:** `eval/run.ts` (a workspace package `@coderadar/eval`):
 - Discovers `eval/fixtures/*/`, scans each `app/` dir, diffs the graph + query results against `golden.json` (format in testing-strategy.md, including `forbidden` entries).

docs/testing-strategy.md

@@ -48,27 +48,39 @@ eval/
 └── run.ts                          # runner: scan fixtures, diff vs golden, emit scorecard
 ```
 
-`golden.json` format (per fixture):
+`golden.json` format (per fixture; full contract in `eval/src/golden.ts`):
 
 ```json
 {
   "failureMode": "C1",
+  "app": "./app",
   "expect": {
     "components": [{ "name": "DataTable", "instances": 2 }],
     "attributions": [
-      { "instanceAt": "pages/Users.tsx", "endpoint": "/api/users" },
-      { "instanceAt": "pages/Invoices.tsx", "endpoint": "/api/invoices" }
+      { "component": "DataTable", "instanceAt": "pages/UsersPage.tsx",
+        "endpoints": ["/api/users"],
+        "expectedFail": "phase-2: per-instance attribution requires prop-flow (step 2.2)" }
     ],
     "forbidden": [
-      { "note": "definition-level attribution would claim DataTable calls both APIs",
-        "attribution": { "definition": "DataTable", "endpoint": "*" } }
+      { "component": "DataTable", "instanceAt": "pages/UsersPage.tsx",
+        "endpoint": "/api/invoices",
+        "note": "poison: invoices API attributed to the users-page table" }
+    ],
+    "queries": [
+      { "terms": ["Save"], "status": "ambiguous" },
+      { "terms": ["All Users"], "status": "ok", "top": "UsersPage" }
     ]
   }
 }
 ```
 
-`forbidden` entries catch the *specific wrong answer* each failure mode produces —
-passing isn't just finding the right edges, it's not emitting the poisonous ones.
+- `forbidden` entries catch the *specific wrong answer* each failure mode produces —
+  passing isn't just finding the right edges, it's not emitting the poisonous ones.
+  Forbidden checks never carry `expectedFail`: poison gates in every phase.
+- `expectedFail` (per check) gives xfail semantics: a failing check reports `xfail`
+  and doesn't gate; a *passing* check still carrying the marker reports
+  `unexpected-pass` and **does** gate — stale markers are removed the moment the
+  capability lands, so capability arrival is always an explicit, reviewed event.
 
 ## Metrics
 
@@ -103,7 +115,8 @@ ratchet upward as reality informs them):
 
 ## Monitoring direction over time
 
-- `eval/run.ts` appends every run's scorecard to `eval/history.jsonl` (committed weekly).
+- `pnpm eval -- --record` appends the run's summary to `eval/history.jsonl`
+  (recorded deliberately — e.g. at each step completion — and committed).
   A shrinking metric between phases is a **regression investigation**, not noise.
 - From Phase 4, the corrections store (G4) is periodically folded into `eval/tickets/` —
   real mismatches from the pipeline become permanent eval cases. This is the flywheel:

eval/fixtures/a4-generic-text/app/BillingForm.tsx

@@ -0,0 +1,9 @@
+export function BillingForm() {
+  return (
+    <form>
+      <h2>Card number</h2>
+      <input placeholder="4242 4242 4242 4242" />
+      <button type="submit">Save</button>
+    </form>
+  );
+}

eval/fixtures/a4-generic-text/app/ProfileForm.tsx

@@ -0,0 +1,9 @@
+export function ProfileForm() {
+  return (
+    <form>
+      <h2>Display name</h2>
+      <input placeholder="How should we address you?" />
+      <button type="submit">Save</button>
+    </form>
+  );
+}

eval/fixtures/a4-generic-text/app/SettingsForm.tsx

@@ -0,0 +1,12 @@
+export function SettingsForm() {
+  return (
+    <form>
+      <h2>Notification preferences</h2>
+      <label>
+        Email alerts
+        <input type="checkbox" />
+      </label>
+      <button type="submit">Save</button>
+    </form>
+  );
+}

eval/fixtures/a4-generic-text/golden.json

@@ -0,0 +1,18 @@
+{
+  "failureMode": "A4",
+  "note": "Generic text collisions: 'Save' appears in three components. A lone generic term must yield an honest ambiguous (with a disambiguation question), never a coin-flip ok. Adding one distinctive term must resolve it.",
+  "expect": {
+    "components": [
+      { "name": "SettingsForm", "instances": 0 },
+      { "name": "ProfileForm", "instances": 0 },
+      { "name": "BillingForm", "instances": 0 }
+    ],
+    "queries": [
+      { "terms": ["Save"], "status": "ambiguous" },
+      { "terms": ["Save", "Card number"], "status": "ok", "top": "BillingForm" },
+      { "terms": ["Save", "Notification preferences"], "status": "ok", "top": "SettingsForm" },
+      { "terms": ["Save", "Display name"], "status": "ok", "top": "ProfileForm" },
+      { "terms": ["Purchase history"], "status": "declined" }
+    ]
+  }
+}

eval/fixtures/c1-shared-datatable/app/components/DataTable.tsx

@@ -0,0 +1,29 @@
+export interface Column {
+  key: string;
+  label: string;
+}
+
+export function DataTable({ rows, columns }: { rows: Record<string, string>[]; columns: Column[] }) {
+  if (rows.length === 0) return <p>No records found</p>;
+
+  return (
+    <table>
+      <thead>
+        <tr>
+          {columns.map((col) => (
+            <th key={col.key}>{col.label}</th>
+          ))}
+        </tr>
+      </thead>
+      <tbody>
+        {rows.map((row, i) => (
+          <tr key={i}>
+            {columns.map((col) => (
+              <td key={col.key}>{row[col.key]}</td>
+            ))}
+          </tr>
+        ))}
+      </tbody>
+    </table>
+  );
+}

eval/fixtures/c1-shared-datatable/app/pages/InvoicesPage.tsx

@@ -0,0 +1,26 @@
+import { useEffect, useState } from "react";
+
+import { DataTable } from "../components/DataTable";
+
+export function InvoicesPage() {
+  const [rows, setRows] = useState<Record<string, string>[]>([]);
+
+  useEffect(() => {
+    fetch("/api/invoices")
+      .then((res) => res.json())
+      .then(setRows);
+  }, []);
+
+  return (
+    <main>
+      <h1>All Invoices</h1>
+      <DataTable
+        rows={rows}
+        columns={[
+          { key: "number", label: "Invoice Number" },
+          { key: "total", label: "Total" },
+        ]}
+      />
+    </main>
+  );
+}

eval/fixtures/c1-shared-datatable/app/pages/UsersPage.tsx

@@ -0,0 +1,26 @@
+import { useEffect, useState } from "react";
+
+import { DataTable } from "../components/DataTable";
+
+export function UsersPage() {
+  const [rows, setRows] = useState<Record<string, string>[]>([]);
+
+  useEffect(() => {
+    fetch("/api/users")
+      .then((res) => res.json())
+      .then(setRows);
+  }, []);
+
+  return (
+    <main>
+      <h1>All Users</h1>
+      <DataTable
+        rows={rows}
+        columns={[
+          { key: "name", label: "Name" },
+          { key: "email", label: "Email" },
+        ]}
+      />
+    </main>
+  );
+}

eval/fixtures/c1-shared-datatable/golden.json

@@ -0,0 +1,57 @@
+{
+  "failureMode": "C1",
+  "note": "The headline case: one shared DataTable, two pages, two different APIs. Attribution must be per instance. The forbidden entries catch the poison answer a definition-level trace would give (both endpoints attributed to every usage).",
+  "expect": {
+    "components": [
+      { "name": "DataTable", "instances": 2 },
+      { "name": "UsersPage", "instances": 0 },
+      { "name": "InvoicesPage", "instances": 0 }
+    ],
+    "attributions": [
+      {
+        "component": "DataTable",
+        "instanceAt": "pages/UsersPage.tsx",
+        "endpoints": ["/api/users"],
+        "expectedFail": "phase-2: per-instance attribution requires prop-flow (step 2.2)"
+      },
+      {
+        "component": "DataTable",
+        "instanceAt": "pages/InvoicesPage.tsx",
+        "endpoints": ["/api/invoices"],
+        "expectedFail": "phase-2: per-instance attribution requires prop-flow (step 2.2)"
+      },
+      {
+        "component": "UsersPage",
+        "endpoints": ["/api/users"]
+      },
+      {
+        "component": "InvoicesPage",
+        "endpoints": ["/api/invoices"]
+      }
+    ],
+    "forbidden": [
+      {
+        "component": "DataTable",
+        "instanceAt": "pages/UsersPage.tsx",
+        "endpoint": "/api/invoices",
+        "note": "poison: invoices API attributed to the users-page table"
+      },
+      {
+        "component": "DataTable",
+        "instanceAt": "pages/InvoicesPage.tsx",
+        "endpoint": "/api/users",
+        "note": "poison: users API attributed to the invoices-page table"
+      },
+      {
+        "component": "UsersPage",
+        "endpoint": "/api/invoices",
+        "note": "poison: cross-page endpoint bleed at definition level"
+      }
+    ],
+    "queries": [
+      { "terms": ["All Users"], "status": "ok", "top": "UsersPage" },
+      { "terms": ["All Invoices"], "status": "ok", "top": "InvoicesPage" },
+      { "terms": ["No records found"], "status": "ok", "top": "DataTable" }
+    ]
+  }
+}

eval/fixtures/demo-app/golden.json

@@ -0,0 +1,25 @@
+{
+  "failureMode": "baseline",
+  "note": "The bundled example app, kept green from day one. App source lives in examples/demo-app so the README quick-start and this fixture share one copy.",
+  "app": "../../../examples/demo-app/src",
+  "expect": {
+    "components": [
+      { "name": "UserList", "instances": 0 },
+      { "name": "UserCard", "instances": 1 }
+    ],
+    "attributions": [
+      {
+        "component": "UserList",
+        "endpoints": ["/api/users", "/api/users/${user.id}"]
+      },
+      {
+        "component": "UserCard",
+        "endpoints": ["/api/users/${user.id}"]
+      }
+    ],
+    "queries": [
+      { "terms": ["Team Members"], "status": "ok", "top": "UserList" },
+      { "terms": ["Remove member"], "status": "ok", "top": "UserCard" }
+    ]
+  }
+}

eval/history.jsonl

@@ -0,0 +1 @@
+{"generatedAt":"2026-07-12T20:53:38.653Z","commitSha":"25e035d50ddddb72c9e8c02febc9785e7a64bb3c","pass":25,"fail":0,"xfail":2,"unexpectedPass":0,"lineagePrecision":1,"lineageRecall":0.714,"matchAccuracy":1}

eval/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@coderadar/eval",
+  "version": "0.1.0",
+  "private": true,
+  "description": "CodeRadar eval harness — scans failure-mode fixtures, diffs against golden outputs, emits the scorecard that gates every phase.",
+  "license": "MIT",
+  "type": "module",
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit"
+  },
+  "dependencies": {
+    "@coderadar/core": "workspace:*",
+    "@coderadar/parser-react": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "^22.20.1",
+    "typescript": "^5.7.0"
+  }
+}