diff --git a/packages/loopover-engine/src/calibration/provider-track-record.ts b/packages/loopover-engine/src/calibration/provider-track-record.ts new file mode 100644 index 0000000000..57e9e1a212 --- /dev/null +++ b/packages/loopover-engine/src/calibration/provider-track-record.ts @@ -0,0 +1,161 @@ +// Per-provider reviewer track records (#8228, epic #8211 track F). Dual-reviewer consensus events exist +// (reviewer-consensus-calibration.ts) and reversal labels now say which calls were RIGHT; this module joins +// the two: measured precision per reviewer identity, per repo and overall, over decided cases. Providers are +// opaque ids — no provider names hardcoded, no config coupling. Mirrors the consensus module's ingestion +// discipline (typed inputs, explicit vote vocabulary) and the #8085 scorer's null-below-the-sample-floor +// rule: a slice that decided nothing reports null, never 0. +// +// JOIN SEMANTICS (documented once, tested as invariants): +// • A provider signal joins a labeled BacktestCase by exact `targetKey`. A signal whose target carries no +// decided label is counted (`signals`) but contributes to no rate — undecided is not evidence. +// • A provider "supported the firing" when it voted `fail` (the defect-flagging vote in the consensus +// vocabulary). `precision` = P(label "confirmed" | this provider voted fail) — the same +// correct-firing-as-numerator discipline as computeRulePrecision, at reviewer grain. +// • `agreementRate` = share of this provider's decided votes that MATCHED the human label (fail↔confirmed, +// pass/warn↔reversed) — a symmetric accuracy measure precision alone can't give a rarely-failing provider. +// • `consensusRate` = share of this provider's signals on targets that another provider ALSO reviewed +// where the two votes agreed (both-fail or both-non-fail); `splitRate` is its complement. Null when the +// provider shares no targets — one-provider corpora have no consensus to measure. +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads. + +import type { BacktestCase } from "./backtest-corpus.js"; +import type { ReviewerConsensusVote } from "../reviewer-consensus-calibration.js"; + +export type ProviderReviewSignal = { + /** Opaque reviewer identity — an id, never a hardcoded provider name. */ + provider: string; + repoFullName: string; + /** Joins to {@link BacktestCase.targetKey} (`owner/repo#N`). */ + targetKey: string; + vote: ReviewerConsensusVote; +}; + +export type ProviderTrackRecord = { + provider: string; + /** The repo this row aggregates, or null for the provider's overall rollup across every repo. */ + repoFullName: string | null; + signals: number; + decided: number; + confirmed: number; + reversed: number; + precision: number | null; + agreementRate: number | null; + consensusRate: number | null; + splitRate: number | null; +}; + +type MutableStats = { + signals: number; + decided: number; + confirmed: number; + reversed: number; + failDecided: number; + failConfirmed: number; + agreed: number; + shared: number; + consensus: number; +}; + +function emptyStats(): MutableStats { + return { signals: 0, decided: 0, confirmed: 0, reversed: 0, failDecided: 0, failConfirmed: 0, agreed: 0, shared: 0, consensus: 0 }; +} + +function toRecord(provider: string, repoFullName: string | null, stats: MutableStats): ProviderTrackRecord { + return { + provider, + repoFullName, + signals: stats.signals, + decided: stats.decided, + confirmed: stats.confirmed, + reversed: stats.reversed, + precision: stats.failDecided > 0 ? stats.failConfirmed / stats.failDecided : null, + agreementRate: stats.decided > 0 ? stats.agreed / stats.decided : null, + consensusRate: stats.shared > 0 ? stats.consensus / stats.shared : null, + splitRate: stats.shared > 0 ? (stats.shared - stats.consensus) / stats.shared : null, + }; +} + +/** + * Compute per-(provider, repo) and per-provider-overall track records from reviewer signals joined against + * a labeled corpus, per the join semantics documented in this module's header. Deterministic ordering: + * providers ascending, and within each provider the overall rollup (repoFullName null) first, then repos + * ascending. Aggregates only — provider ids, repo names, and numbers; never target keys or vote payloads. + */ +export function computeProviderTrackRecords( + signals: readonly ProviderReviewSignal[], + cases: readonly BacktestCase[], +): ProviderTrackRecord[] { + const labelByTarget = new Map(); + for (const backtestCase of cases) labelByTarget.set(backtestCase.targetKey, backtestCase.label); + + // Which providers reviewed each target, with their fail/non-fail stance — the consensus/split join. + const stancesByTarget = new Map>(); + for (const signal of signals) { + let stances = stancesByTarget.get(signal.targetKey); + if (stances === undefined) { + stances = new Map(); + stancesByTarget.set(signal.targetKey, stances); + } + stances.set(signal.provider, signal.vote === "fail"); + } + + const perRepo = new Map>(); // provider → repo → stats + const overall = new Map(); + for (const signal of signals) { + let repos = perRepo.get(signal.provider); + if (repos === undefined) { + repos = new Map(); + perRepo.set(signal.provider, repos); + } + let repoStats = repos.get(signal.repoFullName); + if (repoStats === undefined) { + repoStats = emptyStats(); + repos.set(signal.repoFullName, repoStats); + } + let overallStats = overall.get(signal.provider); + if (overallStats === undefined) { + overallStats = emptyStats(); + overall.set(signal.provider, overallStats); + } + + const label = labelByTarget.get(signal.targetKey); + const votedFail = signal.vote === "fail"; + const stances = stancesByTarget.get(signal.targetKey)!; + for (const stats of [repoStats, overallStats]) { + stats.signals += 1; + if (label !== undefined) { + stats.decided += 1; + if (label === "confirmed") stats.confirmed += 1; + else stats.reversed += 1; + if (votedFail) { + stats.failDecided += 1; + if (label === "confirmed") stats.failConfirmed += 1; + } + // Matched the human: a fail vote on a confirmed firing, or a non-fail vote on a reversed one. + if (votedFail === (label === "confirmed")) stats.agreed += 1; + } + if (stances.size > 1) { + stats.shared += 1; + let agreeingOthers = 0; + let others = 0; + for (const [otherProvider, otherFail] of stances) { + if (otherProvider === signal.provider) continue; + others += 1; + if (otherFail === votedFail) agreeingOthers += 1; + } + if (agreeingOthers === others) stats.consensus += 1; + } + } + } + + const records: ProviderTrackRecord[] = []; + for (const provider of [...overall.keys()].sort()) { + records.push(toRecord(provider, null, overall.get(provider)!)); + const repos = perRepo.get(provider)!; + for (const repoFullName of [...repos.keys()].sort()) { + records.push(toRecord(provider, repoFullName, repos.get(repoFullName)!)); + } + } + return records; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index bea83db60d..0471a4ee67 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -175,6 +175,7 @@ export * from "./calibration/backtest-track-record.js"; // same way scripts/backtest-corpus-export.ts already imports BacktestCase. export * from "./calibration/backtest-split.js"; export * from "./calibration/backtest-threshold.js"; +export * from "./calibration/provider-track-record.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/provider-track-record.test.ts b/packages/loopover-engine/test/provider-track-record.test.ts new file mode 100644 index 0000000000..7a882032f8 --- /dev/null +++ b/packages/loopover-engine/test/provider-track-record.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { computeProviderTrackRecords, type BacktestCase, type ProviderReviewSignal } from "../dist/index.js"; + +function labeled(targetKey: string, label: BacktestCase["label"]): BacktestCase { + return { + ruleId: "ai_consensus_defect", + targetKey, + outcome: "close", + label, + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + }; +} + +function signal(provider: string, targetKey: string, vote: ProviderReviewSignal["vote"]): ProviderReviewSignal { + return { provider, repoFullName: "acme/widgets", targetKey, vote }; +} + +test("barrel: the public entrypoint re-exports the provider track-record computation (#8228)", () => { + assert.equal(typeof computeProviderTrackRecords, "function"); +}); + +test("both-provider round-trip: precision + agreement + consensus rates land per provider", () => { + const records = computeProviderTrackRecords( + [ + signal("a", "acme/widgets#1", "fail"), + signal("b", "acme/widgets#1", "fail"), + signal("a", "acme/widgets#2", "fail"), + signal("b", "acme/widgets#2", "pass"), + ], + [labeled("acme/widgets#1", "confirmed"), labeled("acme/widgets#2", "reversed")], + ); + const aOverall = records.find((r) => r.provider === "a" && r.repoFullName === null)!; + assert.equal(aOverall.precision, 0.5); + assert.equal(aOverall.consensusRate, 0.5); + const bOverall = records.find((r) => r.provider === "b" && r.repoFullName === null)!; + assert.equal(bOverall.precision, 1); + assert.equal(bOverall.agreementRate, 1); +}); + +test("null discipline: no fail votes -> null precision; no shared targets -> null consensus/split", () => { + const records = computeProviderTrackRecords( + [signal("solo", "acme/widgets#1", "pass")], + [labeled("acme/widgets#1", "reversed")], + ); + const overall = records.find((r) => r.repoFullName === null)!; + assert.equal(overall.precision, null); + assert.equal(overall.consensusRate, null); + assert.equal(overall.splitRate, null); + assert.equal(overall.agreementRate, 1); +}); diff --git a/test/unit/provider-track-record-engine.test.ts b/test/unit/provider-track-record-engine.test.ts new file mode 100644 index 0000000000..87b30498bf --- /dev/null +++ b/test/unit/provider-track-record-engine.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; + +// Import the engine SOURCE directly (not the built dist) -- coverage.include lists +// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in +// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace +// suite). Same pattern as backtest-corpus-engine.test.ts / repo-corpus-engine.test.ts. +import { + computeProviderTrackRecords, + type ProviderReviewSignal, +} from "../../packages/loopover-engine/src/calibration/provider-track-record"; +import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus"; + +function labeled(targetKey: string, label: BacktestCase["label"]): BacktestCase { + return { + ruleId: "ai_consensus_defect", + targetKey, + outcome: "close", + label, + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + }; +} + +function signal(provider: string, targetKey: string, vote: ProviderReviewSignal["vote"], repoFullName = "acme/widgets"): ProviderReviewSignal { + return { provider, repoFullName, targetKey, vote }; +} + +describe("computeProviderTrackRecords (#8228)", () => { + it("computes precision, agreement, and consensus/split rates for a both-provider corpus, per repo and overall", () => { + const cases = [ + labeled("acme/widgets#1", "confirmed"), + labeled("acme/widgets#2", "reversed"), + labeled("acme/widgets#3", "confirmed"), + ]; + const signals = [ + // #1: both fail on a confirmed firing — consensus, both correct. + signal("provider-a", "acme/widgets#1", "fail"), + signal("provider-b", "acme/widgets#1", "fail"), + // #2: split — a fails (wrong: label reversed), b passes (right). + signal("provider-a", "acme/widgets#2", "fail"), + signal("provider-b", "acme/widgets#2", "pass"), + // #3: only a reviews it, warns (non-fail on a confirmed firing — disagreed with the human). + signal("provider-a", "acme/widgets#3", "warn"), + ]; + const records = computeProviderTrackRecords(signals, cases); + + const aOverall = records.find((r) => r.provider === "provider-a" && r.repoFullName === null)!; + expect(aOverall).toMatchObject({ + signals: 3, + decided: 3, + confirmed: 2, + reversed: 1, + precision: 0.5, // of a's 2 fail votes, 1 hit a confirmed firing + agreementRate: 1 / 3, // matched the human only on #1 + consensusRate: 0.5, // shared #1 (agreed) and #2 (split) + splitRate: 0.5, + }); + const bOverall = records.find((r) => r.provider === "provider-b" && r.repoFullName === null)!; + expect(bOverall).toMatchObject({ signals: 2, decided: 2, precision: 1, agreementRate: 1, consensusRate: 0.5, splitRate: 0.5 }); + + // Single-repo corpus: each provider's per-repo row equals its overall rollup. + const aRepo = records.find((r) => r.provider === "provider-a" && r.repoFullName === "acme/widgets")!; + expect(aRepo).toMatchObject({ signals: aOverall.signals, decided: aOverall.decided, precision: aOverall.precision }); + }); + + it("keeps a one-provider corpus's consensus/split rates null — no shared targets, no consensus to measure", () => { + const records = computeProviderTrackRecords( + [signal("solo", "acme/widgets#1", "fail"), signal("solo", "acme/widgets#2", "pass")], + [labeled("acme/widgets#1", "confirmed"), labeled("acme/widgets#2", "reversed")], + ); + const overall = records.find((r) => r.repoFullName === null)!; + expect(overall.consensusRate).toBeNull(); + expect(overall.splitRate).toBeNull(); + expect(overall.precision).toBe(1); + expect(overall.agreementRate).toBe(1); + }); + + it("reports null (never 0) precision below the sample floor: undecided targets and providers that never voted fail", () => { + const records = computeProviderTrackRecords( + [ + signal("quiet", "acme/widgets#9", "pass"), // undecided target — no label exists + signal("quiet", "acme/widgets#1", "warn"), // decided, but never a fail vote + ], + [labeled("acme/widgets#1", "reversed")], + ); + const overall = records.find((r) => r.repoFullName === null)!; + expect(overall).toMatchObject({ signals: 2, decided: 1, precision: null, agreementRate: 1 }); + }); + + it("rolls per-repo rows up into the overall row exactly, with deterministic provider→overall→repo ordering", () => { + const cases = [labeled("acme/widgets#1", "confirmed"), labeled("acme/gadgets#2", "confirmed")]; + const signals = [ + signal("zeta", "acme/widgets#1", "fail", "acme/widgets"), + signal("zeta", "acme/gadgets#2", "fail", "acme/gadgets"), + signal("alpha", "acme/widgets#1", "fail", "acme/widgets"), + ]; + const records = computeProviderTrackRecords(signals, cases); + expect(records.map((r) => [r.provider, r.repoFullName])).toEqual([ + ["alpha", null], + ["alpha", "acme/widgets"], + ["zeta", null], + ["zeta", "acme/gadgets"], + ["zeta", "acme/widgets"], + ]); + const zetaOverall = records.find((r) => r.provider === "zeta" && r.repoFullName === null)!; + const zetaRepos = records.filter((r) => r.provider === "zeta" && r.repoFullName !== null); + expect(zetaRepos.reduce((sum, r) => sum + r.decided, 0)).toBe(zetaOverall.decided); + expect(zetaRepos.reduce((sum, r) => sum + r.signals, 0)).toBe(zetaOverall.signals); + // Determinism: identical inputs yield the identical result. + expect(computeProviderTrackRecords(signals, cases)).toEqual(records); + }); + + it("never leaks target keys into any returned shape — provider ids, repo names, and numbers only", () => { + const records = computeProviderTrackRecords( + [signal("provider-a", "acme/widgets#42", "fail")], + [labeled("acme/widgets#42", "confirmed")], + ); + expect(JSON.stringify(records)).not.toContain("#42"); + }); + + it("reports null agreement (never 0) for a provider whose every signal is undecided", () => { + const records = computeProviderTrackRecords([signal("unjoined", "acme/widgets#404", "fail")], []); + const overall = records.find((r) => r.repoFullName === null)!; + expect(overall).toMatchObject({ signals: 1, decided: 0, precision: null, agreementRate: null }); + }); + + it("returns an empty list for empty inputs", () => { + expect(computeProviderTrackRecords([], [])).toEqual([]); + }); +});