From 39814b1850edad8e1c181648bd0f84ff44c576f2 Mon Sep 17 00:00:00 2001 From: Deep Kumar Singh Kushwah Date: Mon, 13 Jul 2026 02:23:38 +0530 Subject: [PATCH] feat(eval): eval harness + first failure-mode fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 0.2 (TRACKER). The measurement loop every later phase is gated by: - @coderadar/eval: fixture discovery, golden.json diffing, scorecard.json, history.jsonl (--record), thresholds gate with non-zero exit. - Per-check expectedFail (xfail semantics): failing marked checks report xfail and don't gate; passing marked checks report unexpected-pass and DO gate, so capability arrival is an explicit reviewed event. Forbidden (poison) checks never xfail. - Fixtures: c1-shared-datatable (headline case — per-instance attributions xfail'd until step 2.2, poison assertions active now), a4-generic-text (ambiguity honesty on 'Save' collisions), demo-app (baseline, app shared with examples/). - Current scorecard: 25 pass / 0 fail / 2 xfail / 0 unexpected-pass; lineage precision 1.000, recall 0.714 (xfail gap), match accuracy 1.000. Co-Authored-By: Claude Fable 5 --- .gitignore | 3 + TRACKER.md | 6 +- docs/testing-strategy.md | 29 ++- .../a4-generic-text/app/BillingForm.tsx | 9 + .../a4-generic-text/app/ProfileForm.tsx | 9 + .../a4-generic-text/app/SettingsForm.tsx | 12 ++ eval/fixtures/a4-generic-text/golden.json | 18 ++ .../app/components/DataTable.tsx | 29 +++ .../app/pages/InvoicesPage.tsx | 26 +++ .../app/pages/UsersPage.tsx | 26 +++ eval/fixtures/c1-shared-datatable/golden.json | 57 ++++++ eval/fixtures/demo-app/golden.json | 25 +++ eval/history.jsonl | 1 + eval/package.json | 20 +++ eval/src/checks.ts | 130 ++++++++++++++ eval/src/golden.ts | 97 ++++++++++ eval/src/run.ts | 165 ++++++++++++++++++ eval/thresholds.json | 5 + eval/tsconfig.json | 8 + package.json | 3 +- pnpm-lock.yaml | 16 ++ pnpm-workspace.yaml | 1 + 22 files changed, 683 insertions(+), 12 deletions(-) create mode 100644 eval/fixtures/a4-generic-text/app/BillingForm.tsx create mode 100644 eval/fixtures/a4-generic-text/app/ProfileForm.tsx create mode 100644 eval/fixtures/a4-generic-text/app/SettingsForm.tsx create mode 100644 eval/fixtures/a4-generic-text/golden.json create mode 100644 eval/fixtures/c1-shared-datatable/app/components/DataTable.tsx create mode 100644 eval/fixtures/c1-shared-datatable/app/pages/InvoicesPage.tsx create mode 100644 eval/fixtures/c1-shared-datatable/app/pages/UsersPage.tsx create mode 100644 eval/fixtures/c1-shared-datatable/golden.json create mode 100644 eval/fixtures/demo-app/golden.json create mode 100644 eval/history.jsonl create mode 100644 eval/package.json create mode 100644 eval/src/checks.ts create mode 100644 eval/src/golden.ts create mode 100644 eval/src/run.ts create mode 100644 eval/thresholds.json create mode 100644 eval/tsconfig.json diff --git a/.gitignore b/.gitignore index 872d5f6..4c08c23 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,6 @@ dist vite.config.js.timestamp-* vite.config.ts.timestamp-* .vite/ + +# Eval outputs (scorecard is regenerated every run; history is recorded deliberately) +eval/scorecard.json diff --git a/TRACKER.md b/TRACKER.md index 00c237a..aac2ecb 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -5,8 +5,8 @@ ## Status - **Current phase:** 0 — Foundations -- **Next step:** 0.2 — Eval harness + first fixtures -- **Done:** 0.1 +- **Next step:** 0.3 — Graph storage & versioning +- **Done:** 0.1, 0.2 - **Gates passed:** none yet (Gate 0 completes with 0.4) ## What CodeRadar is @@ -58,7 +58,7 @@ The v0.1 schema is definition-only, which is *wrong* for C1 — fix before build - Demo-app scan shows `UserCard` with 1 definition + 1 instance (parent `UserList`). - `matchComponentsByText` / `traceLineage` return `QueryResult` envelopes. -### [ ] 0.2 Eval harness + first fixtures +### [x] 0.2 Eval harness + first fixtures **Failure modes:** infrastructure for all; first fixtures C1, A4 **Build:** `eval/run.ts` (a workspace package `@coderadar/eval`): - Discovers `eval/fixtures/*/`, scans each `app/` dir, diffs the graph + query results against `golden.json` (format in testing-strategy.md, including `forbidden` entries). diff --git a/docs/testing-strategy.md b/docs/testing-strategy.md index 1d6000c..e9e95aa 100644 --- a/docs/testing-strategy.md +++ b/docs/testing-strategy.md @@ -48,27 +48,39 @@ eval/ └── run.ts # runner: scan fixtures, diff vs golden, emit scorecard ``` -`golden.json` format (per fixture): +`golden.json` format (per fixture; full contract in `eval/src/golden.ts`): ```json { "failureMode": "C1", + "app": "./app", "expect": { "components": [{ "name": "DataTable", "instances": 2 }], "attributions": [ - { "instanceAt": "pages/Users.tsx", "endpoint": "/api/users" }, - { "instanceAt": "pages/Invoices.tsx", "endpoint": "/api/invoices" } + { "component": "DataTable", "instanceAt": "pages/UsersPage.tsx", + "endpoints": ["/api/users"], + "expectedFail": "phase-2: per-instance attribution requires prop-flow (step 2.2)" } ], "forbidden": [ - { "note": "definition-level attribution would claim DataTable calls both APIs", - "attribution": { "definition": "DataTable", "endpoint": "*" } } + { "component": "DataTable", "instanceAt": "pages/UsersPage.tsx", + "endpoint": "/api/invoices", + "note": "poison: invoices API attributed to the users-page table" } + ], + "queries": [ + { "terms": ["Save"], "status": "ambiguous" }, + { "terms": ["All Users"], "status": "ok", "top": "UsersPage" } ] } } ``` -`forbidden` entries catch the *specific wrong answer* each failure mode produces — -passing isn't just finding the right edges, it's not emitting the poisonous ones. +- `forbidden` entries catch the *specific wrong answer* each failure mode produces — + passing isn't just finding the right edges, it's not emitting the poisonous ones. + Forbidden checks never carry `expectedFail`: poison gates in every phase. +- `expectedFail` (per check) gives xfail semantics: a failing check reports `xfail` + and doesn't gate; a *passing* check still carrying the marker reports + `unexpected-pass` and **does** gate — stale markers are removed the moment the + capability lands, so capability arrival is always an explicit, reviewed event. ## Metrics @@ -103,7 +115,8 @@ ratchet upward as reality informs them): ## Monitoring direction over time -- `eval/run.ts` appends every run's scorecard to `eval/history.jsonl` (committed weekly). +- `pnpm eval -- --record` appends the run's summary to `eval/history.jsonl` + (recorded deliberately — e.g. at each step completion — and committed). A shrinking metric between phases is a **regression investigation**, not noise. - From Phase 4, the corrections store (G4) is periodically folded into `eval/tickets/` — real mismatches from the pipeline become permanent eval cases. This is the flywheel: diff --git a/eval/fixtures/a4-generic-text/app/BillingForm.tsx b/eval/fixtures/a4-generic-text/app/BillingForm.tsx new file mode 100644 index 0000000..a9cfc3b --- /dev/null +++ b/eval/fixtures/a4-generic-text/app/BillingForm.tsx @@ -0,0 +1,9 @@ +export function BillingForm() { + return ( +
+

Card number

+ + +
+ ); +} diff --git a/eval/fixtures/a4-generic-text/app/ProfileForm.tsx b/eval/fixtures/a4-generic-text/app/ProfileForm.tsx new file mode 100644 index 0000000..3de0602 --- /dev/null +++ b/eval/fixtures/a4-generic-text/app/ProfileForm.tsx @@ -0,0 +1,9 @@ +export function ProfileForm() { + return ( +
+

Display name

+ + +
+ ); +} diff --git a/eval/fixtures/a4-generic-text/app/SettingsForm.tsx b/eval/fixtures/a4-generic-text/app/SettingsForm.tsx new file mode 100644 index 0000000..c0d812a --- /dev/null +++ b/eval/fixtures/a4-generic-text/app/SettingsForm.tsx @@ -0,0 +1,12 @@ +export function SettingsForm() { + return ( +
+

Notification preferences

+ + +
+ ); +} diff --git a/eval/fixtures/a4-generic-text/golden.json b/eval/fixtures/a4-generic-text/golden.json new file mode 100644 index 0000000..fdadced --- /dev/null +++ b/eval/fixtures/a4-generic-text/golden.json @@ -0,0 +1,18 @@ +{ + "failureMode": "A4", + "note": "Generic text collisions: 'Save' appears in three components. A lone generic term must yield an honest ambiguous (with a disambiguation question), never a coin-flip ok. Adding one distinctive term must resolve it.", + "expect": { + "components": [ + { "name": "SettingsForm", "instances": 0 }, + { "name": "ProfileForm", "instances": 0 }, + { "name": "BillingForm", "instances": 0 } + ], + "queries": [ + { "terms": ["Save"], "status": "ambiguous" }, + { "terms": ["Save", "Card number"], "status": "ok", "top": "BillingForm" }, + { "terms": ["Save", "Notification preferences"], "status": "ok", "top": "SettingsForm" }, + { "terms": ["Save", "Display name"], "status": "ok", "top": "ProfileForm" }, + { "terms": ["Purchase history"], "status": "declined" } + ] + } +} diff --git a/eval/fixtures/c1-shared-datatable/app/components/DataTable.tsx b/eval/fixtures/c1-shared-datatable/app/components/DataTable.tsx new file mode 100644 index 0000000..7518933 --- /dev/null +++ b/eval/fixtures/c1-shared-datatable/app/components/DataTable.tsx @@ -0,0 +1,29 @@ +export interface Column { + key: string; + label: string; +} + +export function DataTable({ rows, columns }: { rows: Record[]; columns: Column[] }) { + if (rows.length === 0) return

No records found

; + + return ( + + + + {columns.map((col) => ( + + ))} + + + + {rows.map((row, i) => ( + + {columns.map((col) => ( + + ))} + + ))} + +
{col.label}
{row[col.key]}
+ ); +} diff --git a/eval/fixtures/c1-shared-datatable/app/pages/InvoicesPage.tsx b/eval/fixtures/c1-shared-datatable/app/pages/InvoicesPage.tsx new file mode 100644 index 0000000..f05bd8d --- /dev/null +++ b/eval/fixtures/c1-shared-datatable/app/pages/InvoicesPage.tsx @@ -0,0 +1,26 @@ +import { useEffect, useState } from "react"; + +import { DataTable } from "../components/DataTable"; + +export function InvoicesPage() { + const [rows, setRows] = useState[]>([]); + + useEffect(() => { + fetch("/api/invoices") + .then((res) => res.json()) + .then(setRows); + }, []); + + return ( +
+

All Invoices

+ +
+ ); +} diff --git a/eval/fixtures/c1-shared-datatable/app/pages/UsersPage.tsx b/eval/fixtures/c1-shared-datatable/app/pages/UsersPage.tsx new file mode 100644 index 0000000..d607e57 --- /dev/null +++ b/eval/fixtures/c1-shared-datatable/app/pages/UsersPage.tsx @@ -0,0 +1,26 @@ +import { useEffect, useState } from "react"; + +import { DataTable } from "../components/DataTable"; + +export function UsersPage() { + const [rows, setRows] = useState[]>([]); + + useEffect(() => { + fetch("/api/users") + .then((res) => res.json()) + .then(setRows); + }, []); + + return ( +
+

All Users

+ +
+ ); +} diff --git a/eval/fixtures/c1-shared-datatable/golden.json b/eval/fixtures/c1-shared-datatable/golden.json new file mode 100644 index 0000000..04424b9 --- /dev/null +++ b/eval/fixtures/c1-shared-datatable/golden.json @@ -0,0 +1,57 @@ +{ + "failureMode": "C1", + "note": "The headline case: one shared DataTable, two pages, two different APIs. Attribution must be per instance. The forbidden entries catch the poison answer a definition-level trace would give (both endpoints attributed to every usage).", + "expect": { + "components": [ + { "name": "DataTable", "instances": 2 }, + { "name": "UsersPage", "instances": 0 }, + { "name": "InvoicesPage", "instances": 0 } + ], + "attributions": [ + { + "component": "DataTable", + "instanceAt": "pages/UsersPage.tsx", + "endpoints": ["/api/users"], + "expectedFail": "phase-2: per-instance attribution requires prop-flow (step 2.2)" + }, + { + "component": "DataTable", + "instanceAt": "pages/InvoicesPage.tsx", + "endpoints": ["/api/invoices"], + "expectedFail": "phase-2: per-instance attribution requires prop-flow (step 2.2)" + }, + { + "component": "UsersPage", + "endpoints": ["/api/users"] + }, + { + "component": "InvoicesPage", + "endpoints": ["/api/invoices"] + } + ], + "forbidden": [ + { + "component": "DataTable", + "instanceAt": "pages/UsersPage.tsx", + "endpoint": "/api/invoices", + "note": "poison: invoices API attributed to the users-page table" + }, + { + "component": "DataTable", + "instanceAt": "pages/InvoicesPage.tsx", + "endpoint": "/api/users", + "note": "poison: users API attributed to the invoices-page table" + }, + { + "component": "UsersPage", + "endpoint": "/api/invoices", + "note": "poison: cross-page endpoint bleed at definition level" + } + ], + "queries": [ + { "terms": ["All Users"], "status": "ok", "top": "UsersPage" }, + { "terms": ["All Invoices"], "status": "ok", "top": "InvoicesPage" }, + { "terms": ["No records found"], "status": "ok", "top": "DataTable" } + ] + } +} diff --git a/eval/fixtures/demo-app/golden.json b/eval/fixtures/demo-app/golden.json new file mode 100644 index 0000000..c027f35 --- /dev/null +++ b/eval/fixtures/demo-app/golden.json @@ -0,0 +1,25 @@ +{ + "failureMode": "baseline", + "note": "The bundled example app, kept green from day one. App source lives in examples/demo-app so the README quick-start and this fixture share one copy.", + "app": "../../../examples/demo-app/src", + "expect": { + "components": [ + { "name": "UserList", "instances": 0 }, + { "name": "UserCard", "instances": 1 } + ], + "attributions": [ + { + "component": "UserList", + "endpoints": ["/api/users", "/api/users/${user.id}"] + }, + { + "component": "UserCard", + "endpoints": ["/api/users/${user.id}"] + } + ], + "queries": [ + { "terms": ["Team Members"], "status": "ok", "top": "UserList" }, + { "terms": ["Remove member"], "status": "ok", "top": "UserCard" } + ] + } +} diff --git a/eval/history.jsonl b/eval/history.jsonl new file mode 100644 index 0000000..6dc53e8 --- /dev/null +++ b/eval/history.jsonl @@ -0,0 +1 @@ +{"generatedAt":"2026-07-12T20:53:38.653Z","commitSha":"25e035d50ddddb72c9e8c02febc9785e7a64bb3c","pass":25,"fail":0,"xfail":2,"unexpectedPass":0,"lineagePrecision":1,"lineageRecall":0.714,"matchAccuracy":1} diff --git a/eval/package.json b/eval/package.json new file mode 100644 index 0000000..12aecaf --- /dev/null +++ b/eval/package.json @@ -0,0 +1,20 @@ +{ + "name": "@coderadar/eval", + "version": "0.1.0", + "private": true, + "description": "CodeRadar eval harness — scans failure-mode fixtures, diffs against golden outputs, emits the scorecard that gates every phase.", + "license": "MIT", + "type": "module", + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "@coderadar/core": "workspace:*", + "@coderadar/parser-react": "workspace:*" + }, + "devDependencies": { + "@types/node": "^22.20.1", + "typescript": "^5.7.0" + } +} diff --git a/eval/src/checks.ts b/eval/src/checks.ts new file mode 100644 index 0000000..f23e443 --- /dev/null +++ b/eval/src/checks.ts @@ -0,0 +1,130 @@ +/** Run one fixture's golden checks against its scanned graph. */ + +import { + type LineageGraph, + matchComponentsByText, + traceLineage, +} from "@coderadar/core"; + +import type { CheckResult, FixtureResult, Golden } from "./golden.js"; + +export function runChecks(fixture: string, golden: Golden, graph: LineageGraph): FixtureResult { + const checks: CheckResult[] = []; + const attribution = { truePositives: 0, falsePositives: 0, falseNegatives: 0 }; + + const finalize = ( + kind: CheckResult["kind"], + id: string, + passed: boolean, + marker: string | undefined, + detail?: string, + ): void => { + let status: CheckResult["status"]; + if (marker !== undefined) { + status = passed ? "unexpected-pass" : "xfail"; + detail = passed + ? `passed but marked expectedFail (${marker}) — remove the stale marker` + : marker; + } else { + status = passed ? "pass" : "fail"; + } + checks.push({ id, kind, status, detail }); + }; + + for (const expected of golden.expect.components ?? []) { + const definition = graph.nodes.find( + (n) => n.kind === "component" && n.name === expected.name, + ); + const instances = graph.nodes.filter( + (n) => n.kind === "instance" && n.name === expected.name, + ); + const passed = definition !== undefined && instances.length === expected.instances; + finalize( + "components", + `components:${expected.name}`, + passed, + expected.expectedFail, + definition === undefined + ? "definition not found" + : instances.length !== expected.instances + ? `expected ${expected.instances} instances, found ${instances.length}` + : undefined, + ); + } + + for (const expected of golden.expect.attributions ?? []) { + const id = `attribution:${expected.component}${expected.instanceAt !== undefined ? `@${expected.instanceAt}` : ""}`; + const found = traceEndpoints(graph, expected.component, expected.instanceAt); + if (found === null) { + attribution.falseNegatives += expected.endpoints.length; + finalize("attributions", id, false, expected.expectedFail, "trace target not found in graph"); + continue; + } + const want = new Set(expected.endpoints); + const got = new Set(found); + const missing = [...want].filter((e) => !got.has(e)); + const extra = [...got].filter((e) => !want.has(e)); + attribution.truePositives += want.size - missing.length; + attribution.falseNegatives += missing.length; + attribution.falsePositives += extra.length; + finalize( + "attributions", + id, + missing.length === 0 && extra.length === 0, + expected.expectedFail, + missing.length + extra.length > 0 + ? `missing: [${missing.join(", ")}] extra: [${extra.join(", ")}]` + : undefined, + ); + } + + for (const forbidden of golden.expect.forbidden ?? []) { + const id = `forbidden:${forbidden.component}${forbidden.instanceAt !== undefined ? `@${forbidden.instanceAt}` : ""}!${forbidden.endpoint}`; + const found = traceEndpoints(graph, forbidden.component, forbidden.instanceAt) ?? []; + const poisoned = found.includes(forbidden.endpoint); + // Forbidden checks never xfail: poison must gate in every phase. + checks.push({ + id, + kind: "forbidden", + status: poisoned ? "fail" : "pass", + detail: poisoned ? `POISON: ${forbidden.note ?? "forbidden attribution present"}` : undefined, + }); + if (poisoned) attribution.falsePositives += 1; + } + + for (const query of golden.expect.queries ?? []) { + const id = `query:${query.terms.join("+")}`; + const result = matchComponentsByText(graph, query.terms); + let passed = result.status === query.status; + let detail: string | undefined; + if (!passed) { + detail = `expected status ${query.status}, got ${result.status}`; + } else if (query.status === "ok" && query.top !== undefined) { + const top = result.candidates[0]?.value.component.name; + passed = top === query.top; + if (!passed) detail = `expected top ${query.top}, got ${top ?? "none"}`; + } + finalize("queries", id, passed, query.expectedFail, detail); + } + + return { fixture, failureMode: golden.failureMode, checks, attribution }; +} + +/** Endpoints reached from a definition or a specific instance. Null if the target is missing. */ +function traceEndpoints( + graph: LineageGraph, + component: string, + instanceAt: string | undefined, +): string[] | null { + const target = + instanceAt !== undefined + ? graph.nodes.find( + (n) => n.kind === "instance" && n.name === component && n.loc.file === instanceAt, + ) + : graph.nodes.find((n) => n.kind === "component" && n.name === component); + if (target === undefined) return null; + const result = traceLineage(graph, target.id); + const lineage = result.candidates[0]?.value; + if (result.status !== "ok" || lineage === undefined) return null; + return lineage.dataSources.map((d) => d.endpoint); +} diff --git a/eval/src/golden.ts b/eval/src/golden.ts new file mode 100644 index 0000000..6dd00b1 --- /dev/null +++ b/eval/src/golden.ts @@ -0,0 +1,97 @@ +/** The golden.json contract every fixture follows. */ + +/** + * xfail semantics, available on any individual check via `expectedFail`: + * a failing check reports xfail (does not gate); a PASSING check still marked + * expectedFail reports unexpected-pass and DOES gate, so stale markers are + * removed the moment the capability lands. + */ +export interface GoldenComponent { + name: string; + /** Expected number of InstanceNodes for this definition. */ + instances: number; + expectedFail?: string; +} + +export interface GoldenAttribution { + component: string; + /** + * When set: trace the *instance* of `component` located in this file. + * When absent: trace the definition. + */ + instanceAt?: string; + /** Exact set of endpoints the trace must yield. */ + endpoints: string[]; + expectedFail?: string; +} + +export interface GoldenForbidden { + component: string; + instanceAt?: string; + /** This endpoint must NOT appear in the trace — it is the poison answer. */ + endpoint: string; + note?: string; +} + +export interface GoldenQuery { + terms: string[]; + status: "ok" | "ambiguous" | "declined"; + /** Required top-1 component name when status is "ok". */ + top?: string; + expectedFail?: string; +} + +export interface Golden { + failureMode: string; + note?: string; + /** App directory relative to the fixture dir. Default "./app". */ + app?: string; + expect: { + components?: GoldenComponent[]; + attributions?: GoldenAttribution[]; + forbidden?: GoldenForbidden[]; + queries?: GoldenQuery[]; + }; +} + +export type CheckStatus = "pass" | "fail" | "xfail" | "unexpected-pass"; + +export interface CheckResult { + /** e.g. "components:DataTable", "attribution:DataTable@pages/UsersPage.tsx" */ + id: string; + kind: "components" | "attributions" | "forbidden" | "queries"; + status: CheckStatus; + detail?: string; +} + +export interface FixtureResult { + fixture: string; + failureMode: string; + checks: CheckResult[]; + /** Attribution tallies for lineage precision/recall. */ + attribution: { truePositives: number; falsePositives: number; falseNegatives: number }; +} + +export interface Scorecard { + generatedAt: string; + commitSha: string | null; + fixtures: FixtureResult[]; + summary: { + pass: number; + fail: number; + xfail: number; + unexpectedPass: number; + lineagePrecision: number | null; + lineageRecall: number | null; + matchAccuracy: number | null; + }; +} + +export interface Thresholds { + maxFail: number; + maxUnexpectedPass: number; + /** Optional metric floors; null-valued metrics are not gated. */ + minLineagePrecision?: number; + minLineageRecall?: number; + minMatchAccuracy?: number; +} diff --git a/eval/src/run.ts b/eval/src/run.ts new file mode 100644 index 0000000..fae645c --- /dev/null +++ b/eval/src/run.ts @@ -0,0 +1,165 @@ +/** + * CodeRadar eval runner. + * + * Scans every fixture under eval/fixtures/, diffs against golden.json, prints + * a per-failure-mode scorecard, writes eval/scorecard.json, and exits non-zero + * when eval/thresholds.json is violated. + * + * Usage: node eval/dist/run.js [--record] + * --record append this run's summary to eval/history.jsonl (for trend tracking) + */ + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { resolveHookEdges, scanReact } from "@coderadar/parser-react"; + +import { runChecks } from "./checks.js"; +import type { FixtureResult, Golden, Scorecard, Thresholds } from "./golden.js"; + +const evalDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const fixturesDir = path.join(evalDir, "fixtures"); + +function main(): void { + const record = process.argv.includes("--record"); + const fixtureNames = fs + .readdirSync(fixturesDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort(); + + const results: FixtureResult[] = []; + for (const name of fixtureNames) { + const fixtureDir = path.join(fixturesDir, name); + const goldenPath = path.join(fixtureDir, "golden.json"); + if (!fs.existsSync(goldenPath)) { + console.warn(`skipping ${name}: no golden.json`); + continue; + } + const golden = JSON.parse(fs.readFileSync(goldenPath, "utf-8")) as Golden; + const appDir = path.resolve(fixtureDir, golden.app ?? "./app"); + const graph = resolveHookEdges(scanReact({ root: appDir })); + results.push(runChecks(name, golden, graph)); + } + + const scorecard = buildScorecard(results); + fs.writeFileSync(path.join(evalDir, "scorecard.json"), JSON.stringify(scorecard, null, 2)); + if (record) { + fs.appendFileSync( + path.join(evalDir, "history.jsonl"), + JSON.stringify({ generatedAt: scorecard.generatedAt, commitSha: scorecard.commitSha, ...scorecard.summary }) + "\n", + ); + } + + print(scorecard); + gate(scorecard); +} + +function buildScorecard(results: FixtureResult[]): Scorecard { + let pass = 0; + let fail = 0; + let xfail = 0; + let unexpectedPass = 0; + let queryPass = 0; + let queryTotal = 0; + const attr = { truePositives: 0, falsePositives: 0, falseNegatives: 0 }; + + for (const result of results) { + for (const check of result.checks) { + if (check.status === "pass") pass += 1; + else if (check.status === "fail") fail += 1; + else if (check.status === "xfail") xfail += 1; + else unexpectedPass += 1; + if (check.kind === "queries" && check.status !== "xfail") { + queryTotal += 1; + if (check.status === "pass") queryPass += 1; + } + } + attr.truePositives += result.attribution.truePositives; + attr.falsePositives += result.attribution.falsePositives; + attr.falseNegatives += result.attribution.falseNegatives; + } + + const attributed = attr.truePositives + attr.falsePositives; + const golden = attr.truePositives + attr.falseNegatives; + return { + generatedAt: new Date().toISOString(), + commitSha: gitSha(), + fixtures: results, + summary: { + pass, + fail, + xfail, + unexpectedPass, + lineagePrecision: attributed > 0 ? round(attr.truePositives / attributed) : null, + lineageRecall: golden > 0 ? round(attr.truePositives / golden) : null, + matchAccuracy: queryTotal > 0 ? round(queryPass / queryTotal) : null, + }, + }; +} + +function print(scorecard: Scorecard): void { + const icons = { pass: "✓", fail: "✗", xfail: "…", "unexpected-pass": "!" } as const; + for (const fixture of scorecard.fixtures) { + console.log(`\n[${fixture.failureMode}] ${fixture.fixture}`); + for (const check of fixture.checks) { + const suffix = check.detail !== undefined ? ` — ${check.detail}` : ""; + console.log(` ${icons[check.status]} ${check.id}${suffix}`); + } + } + const s = scorecard.summary; + console.log( + `\n${s.pass} pass · ${s.fail} fail · ${s.xfail} xfail · ${s.unexpectedPass} unexpected-pass`, + ); + console.log( + `lineage precision=${fmt(s.lineagePrecision)} recall=${fmt(s.lineageRecall)} · match accuracy=${fmt(s.matchAccuracy)}`, + ); +} + +function gate(scorecard: Scorecard): void { + const thresholds = JSON.parse( + fs.readFileSync(path.join(evalDir, "thresholds.json"), "utf-8"), + ) as Thresholds; + const s = scorecard.summary; + const violations: string[] = []; + if (s.fail > thresholds.maxFail) violations.push(`fail ${s.fail} > max ${thresholds.maxFail}`); + if (s.unexpectedPass > thresholds.maxUnexpectedPass) { + violations.push(`unexpected-pass ${s.unexpectedPass} > max ${thresholds.maxUnexpectedPass}`); + } + checkFloor(violations, "lineagePrecision", s.lineagePrecision, thresholds.minLineagePrecision); + checkFloor(violations, "lineageRecall", s.lineageRecall, thresholds.minLineageRecall); + checkFloor(violations, "matchAccuracy", s.matchAccuracy, thresholds.minMatchAccuracy); + + if (violations.length > 0) { + console.error(`\nEVAL GATE FAILED:\n ${violations.join("\n ")}`); + process.exitCode = 1; + } else { + console.log("\neval gate: OK"); + } +} + +function checkFloor( + violations: string[], + name: string, + value: number | null, + floor: number | undefined, +): void { + if (floor !== undefined && value !== null && value < floor) { + violations.push(`${name} ${value} < min ${floor}`); + } +} + +function gitSha(): string | null { + try { + return execFileSync("git", ["rev-parse", "HEAD"], { cwd: evalDir, encoding: "utf-8" }).trim(); + } catch { + return null; + } +} + +const round = (n: number): number => Math.round(n * 1000) / 1000; +const fmt = (n: number | null): string => (n === null ? "n/a" : n.toFixed(3)); + +main(); diff --git a/eval/thresholds.json b/eval/thresholds.json new file mode 100644 index 0000000..cfebf43 --- /dev/null +++ b/eval/thresholds.json @@ -0,0 +1,5 @@ +{ + "maxFail": 0, + "maxUnexpectedPass": 0, + "minMatchAccuracy": 1.0 +} diff --git a/eval/tsconfig.json b/eval/tsconfig.json new file mode 100644 index 0000000..c17b043 --- /dev/null +++ b/eval/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src"] +} diff --git a/package.json b/package.json index b64c690..3f4f410 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "build": "pnpm -r build", "test": "pnpm -r test", "lint": "pnpm -r lint", - "typecheck": "pnpm -r typecheck" + "typecheck": "pnpm -r typecheck", + "eval": "pnpm -r build && node eval/dist/run.js" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0eac416..b783df1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,22 @@ importers: .: {} + eval: + dependencies: + '@coderadar/core': + specifier: workspace:* + version: link:../packages/core + '@coderadar/parser-react': + specifier: workspace:* + version: link:../packages/parser-react + devDependencies: + '@types/node': + specifier: ^22.20.1 + version: 22.20.1 + typescript: + specifier: ^5.7.0 + version: 5.9.3 + packages/cli: dependencies: '@coderadar/core': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7e24990..249a333 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,5 @@ packages: - "packages/*" + - "eval" allowBuilds: esbuild: true