diff --git a/TRACKER.md b/TRACKER.md index 184436a..cd39621 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -5,7 +5,7 @@ ## Status - **Current phase:** 6F — Field hardening, feedback round 1 (runs before 6.1–6.5) -- **Next step:** 6F.6 test-coverage hardening (blocked: needs the field's actual failing variant — Grafana produced 1,009 covered-by edges, so detection works on real code; the fixture's custom-wrapper shape already passes) +- **Next step:** publish 0.4.1 to the tester (all shippable fixes: 6F.6–6F.10 + visualizer). 6F.6 detection half remains blocked on a real failing test file — its defensive `coverage-unmapped` half shipped. - **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.6, 4.1–4.6, 5.1–5.7, 6F.1–6F.5, 6F.7–6F.10 · **released 0.4.0** (tag v0.4.0, PR #50 to main; publish to npm pending). **Self-validated on Grafana frontend** (6,461 files, 15,334 nodes / 18,367 edges in 72 s): 55 RTK-query data sources, 32 routes, 1,009 coverage edges, gibberish declines — all previously 0/1 in the field run. - **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) · Gate 4 (A4 rarity · A10 fuzzy/OCR · A1 structural · A6 subtree · E3 vision annotations · E2 aliases · G4 corrections — high-conf correct 1.000, ambiguity honesty 1.000, poison rate 0.000) · Gate 5 (F1 context bundle · F2 blast radius · F3 test coverage · F4 response schema · F5 git history · MCP server over stdio — scorecard 265/0/0, all honesty metrics 1.000; **M5 reached** — ticket in → budgeted context bundle out, over MCP) @@ -477,7 +477,7 @@ unmarked, so **the fixture is fully green with zero marks and Gate 6F's extracto are met**. 4 new parser tests (129 total); eval 290 pass / 0 fail / 0 xfail / 0 unexpected-pass, gate OK, metrics 1.000. -### [ ] 6F.6 Test-coverage detection hardening +### [~] 6F.6 Test-coverage detection hardening **Failure modes:** F3 **Build:** the field run found 1 `covered-by` edge app-wide, making `untested` warnings near-universal noise. Handle custom render wrappers (`renderWithProviders()` — resolve @@ -488,6 +488,16 @@ through to the JSX argument), test files importing through the same alias/barrel **Accept:** field-patterns fixture: wrapped renders produce `covered-by` edges; a near-empty coverage graph emits `coverage-unmapped` instead of per-component `untested`; enable the 6F.2 checks. +**Done (defensive half):** the `coverage-unmapped` downgrade shipped — `buildBundle` now, when +test files exist but < 5% of components carry a `covered-by` edge, emits one graph-level +`coverage-unmapped — only N/M components have mapped test coverage` note instead of a +near-universal false `untested`. A genuinely test-free repo (no test nodes) keeps the accurate +per-component `untested`. 3 agent-sdk unit tests (near-empty → downgrade · healthy → keep · +no-tests → keep); verified on the real Grafana graph (34% coverage → per-component `untested` +preserved). **Detection half still open** (custom-wrapper/alias/outside-`__tests__` resolution): +could NOT reproduce the field failure — the fixture's `renderWithProviders` shape already maps, +and Grafana produced 1,009 `covered-by` edges, so detection works on real code. Blocked on a +real failing test file from the tester before building the wrong thing. ### [x] 6F.7 Scoring & result-surface polish **Failure modes:** A4, D2, D6 diff --git a/packages/agent-sdk/src/bundle.test.ts b/packages/agent-sdk/src/bundle.test.ts index 7d4ed67..b1f6591 100644 --- a/packages/agent-sdk/src/bundle.test.ts +++ b/packages/agent-sdk/src/bundle.test.ts @@ -97,4 +97,81 @@ describe("buildBundle (TRACKER 5.2, F1)", () => { expect(ood.match).toEqual([]); expect(ood.warnings.some((w) => w.includes("out-of-scope"))).toBe(true); }); + + it("reports a plain 'untested' when the repo genuinely has no tests", () => { + // graphWithLineage has 1 component, 0 test nodes → untested is accurate. + const bundle = buildBundle(graph, { text: "the Big card overview" }, { budgetTokens: 8000 }); + expect(bundle.warnings.some((w) => w.startsWith("untested —"))).toBe(true); + expect(bundle.warnings.some((w) => w.startsWith("coverage-unmapped"))).toBe(false); + }); +}); + +describe("coverage-unmapped downgrade (TRACKER 6F.6, F3)", () => { + // A graph where test files EXIST but almost none map to a component — the + // field-found signature (tests present, ~0 covered-by edges). The matched + // component's missing coverage must read as "unmapped", not a false "untested". + function sparselyCoveredGraph(componentCount: number, coveredCount: number): LineageGraph { + const nodes: LineageGraph["nodes"] = []; + const edges: LineageEdge[] = []; + for (let i = 0; i < componentCount; i += 1) { + const name = i === 0 ? "SilencePanel" : `Widget${i}`; + const text = i === 0 ? "Silence alert notifications" : `Widget ${i} label`; + nodes.push({ + id: nodeId("component", `${name}.tsx`, name), + kind: "component", + name, + loc: loc(`${name}.tsx`), + exportName: name, + props: [], + renderedText: [{ text, source: "jsx" }], + rendersComponents: [], + structure: { + table: 0, columns: 0, form: 0, input: 0, button: 0, + link: 0, image: 0, heading: 0, list: 0, repeated: 0, + }, + }); + } + // Several test files exist, but they only cover the LAST few components — + // never the matched SilencePanel (index 0). + for (let t = 0; t < 5; t += 1) { + const testId = nodeId("test", `Widget.test.tsx`, `test${t}`); + nodes.push({ + id: testId, + kind: "test", + name: `test${t}`, + loc: loc(`__tests__/Widget${t}.test.tsx`), + framework: "vitest", + }); + if (t < coveredCount) { + edges.push({ from: nodes[componentCount - 1 - t]!.id, to: testId, kind: "covered-by" }); + } + } + return { + version: 2, + root: "/app", + generatedAt: "2026-01-01T00:00:00.000Z", + generator: "test", + nodes, + edges, + }; + } + + it("emits coverage-unmapped instead of untested when coverage is near-empty", () => { + // 40 components, tests present, only 1 covered (2.5% < 5% floor). + const g = sparselyCoveredGraph(40, 1); + const bundle = buildBundle(g, { text: "Silence alert notifications" }, { budgetTokens: 8000 }); + expect(bundle.match[0]?.component).toBe("SilencePanel"); + expect(bundle.warnings.some((w) => w.startsWith("coverage-unmapped"))).toBe(true); + expect(bundle.warnings.some((w) => w.startsWith("untested —"))).toBe(false); + }); + + it("keeps per-component untested when coverage is healthy", () => { + // 10 components, 5 covered (50% ≥ 5%) — the matched SilencePanel is not one + // of them, so untested is the accurate, useful signal. + const g = sparselyCoveredGraph(10, 5); + const bundle = buildBundle(g, { text: "Silence alert notifications" }, { budgetTokens: 8000 }); + expect(bundle.match[0]?.component).toBe("SilencePanel"); + expect(bundle.warnings.some((w) => w.startsWith("untested —"))).toBe(true); + expect(bundle.warnings.some((w) => w.startsWith("coverage-unmapped"))).toBe(false); + }); }); diff --git a/packages/agent-sdk/src/bundle.ts b/packages/agent-sdk/src/bundle.ts index 2b6c70d..7293b16 100644 --- a/packages/agent-sdk/src/bundle.ts +++ b/packages/agent-sdk/src/bundle.ts @@ -186,7 +186,29 @@ export function buildBundle( if (test !== undefined) testFiles.add(test.loc.file); } bundle.tests = [...testFiles].sort().map((file) => ({ file })); - if (!topCovered) bundle.warnings.push(`untested — no test renders ${top.component.name}`); + if (!topCovered) { + // Graph-level coverage health (6F.6, F3): when test files exist but almost + // no component carries a covered-by edge, coverage mapping is effectively + // blind — a per-component "untested" is then a near-universal FALSE signal + // (the field-found noise: 1 covered-by edge across a whole app). Downgrade + // to one honest graph-level note. A genuinely test-free repo (no test + // nodes at all) keeps the accurate "untested". + let componentCount = 0; + let testCount = 0; + for (const node of graph.nodes) { + if (node.kind === "component") componentCount += 1; + else if (node.kind === "test") testCount += 1; + } + const coveredComponents = new Set(); + for (const edge of graph.edges) if (edge.kind === "covered-by") coveredComponents.add(edge.from); + const coverageUnmapped = + testCount > 0 && componentCount > 0 && coveredComponents.size / componentCount < 0.05; + bundle.warnings.push( + coverageUnmapped + ? `coverage-unmapped — only ${coveredComponents.size}/${componentCount} components have mapped test coverage; "untested" is unreliable for this graph` + : `untested — no test renders ${top.component.name}`, + ); + } // Git history (5.6): recent commits touching the matched component's files. const files = new Set([top.component.loc.file]);