diff --git a/packages/loopover-engine/src/calibration/reliability-curve.ts b/packages/loopover-engine/src/calibration/reliability-curve.ts new file mode 100644 index 0000000000..5e95fb2f4a --- /dev/null +++ b/packages/loopover-engine/src/calibration/reliability-curve.ts @@ -0,0 +1,160 @@ +// Per-rule reliability curve + derived threshold suggestion (#8226, epic #8211 track E). Knob evaluation +// (src/services/loosening-knobs.ts) steps down hand-picked candidate ladders; the labeled corpus supports +// something strictly better: bucket a rule's decided cases by their CLAIMED confidence (metadata.confidence, +// the same channel buildConfidenceThresholdClassifier reads, #8138), measure each bucket's EMPIRICAL +// precision against the human verdicts, and let the optimal floor fall out of the curve instead of being +// guessed. This module is the pure math only -- no advisor/registry integration (maintainer follow-on). +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads. + +import type { BacktestCase } from "./backtest-corpus.js"; + +/** One claimed-confidence bucket of a {@link ReliabilityCurve}: its `[floor, ceiling)` confidence range + * (the curve's TOP bucket is ceiling-inclusive so a claimed confidence of exactly 1 is bucketable), the + * decided cases whose claimed confidence landed in it, their confirmed/reversed verdict split, and the + * bucket's empirical precision (`confirmed / cases`) -- null, never 0, when `cases` sits below the curve's + * sample floor, the same "unknown stays unknown" discipline as RulePrecisionReport.precision (#8085). */ +export type ReliabilityBucket = { + floor: number; + ceiling: number; + cases: number; + confirmed: number; + reversed: number; + precision: number | null; +}; + +/** A rule's claimed-confidence reliability curve: `buckets` ascending by `floor`, plus the `sampleFloor` + * the per-bucket precisions were computed under -- carried so {@link deriveThresholdSuggestion} can apply + * the SAME never-on-noise floor to its pooled counts. */ +export type ReliabilityCurve = { + sampleFloor: number; + buckets: ReliabilityBucket[]; +}; + +/** Default bucket edges: one catch-all below 0.3, then 0.05-wide buckets up to 1 -- the SAME granularity + * the loosenable-knob registry's candidate ladders step at (loosening-knobs.ts: [0.45, 0.4, 0.35, 0.3] + * and [0.9, 0.85]), so every floor the registry could actually adopt, both hard minimums (0.3, 0.85) + * included, is exactly a bucket floor a suggestion can land on. No shipped floor lives below 0.3, hence + * the single catch-all there. Sparse corpora keep their honesty either way: a thin bucket reports null + * precision, and {@link deriveThresholdSuggestion} pools at-or-above buckets before judging density. */ +export const DEFAULT_RELIABILITY_BUCKET_EDGES: readonly number[] = [ + 0, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1, +]; + +/** Minimum decided cases before a bucket (or a pooled suggestion window) reports a real precision -- + * below it the value is null, never 0. 5 mirrors the registry's smallest never-on-noise floor + * (loosening-knobs.ts minHeldOutCases: 5) and MIN_CALIBRATION_SAMPLES (contributor-calibration.ts). */ +export const RELIABILITY_BUCKET_SAMPLE_FLOOR = 5; + +/** Index of the bucket containing `claimed` under half-open `[floor, ceiling)` edges with a + * ceiling-INCLUSIVE top bucket, or -1 when it lands in none (below the first edge, above the last, or + * NaN -- the negated first guard makes NaN fail closed into -1 rather than landing in a bucket). */ +function bucketIndexFor(claimed: number, bucketEdges: readonly number[]): number { + if (!(claimed >= bucketEdges[0]!)) return -1; + for (let i = 1; i < bucketEdges.length; i++) { + if (claimed < bucketEdges[i]!) return i - 1; + } + return claimed === bucketEdges[bucketEdges.length - 1]! ? bucketEdges.length - 2 : -1; +} + +/** + * Bucket `cases` by their CLAIMED confidence (`metadata.confidence`) and report each bucket's empirical + * precision against the human verdicts. A case with no numeric claimed confidence contributes to no bucket: + * this deliberately DIVERGES from buildConfidenceThresholdClassifier's degrade-to-1 fallback (#8138) -- + * that function must DECIDE every case, this one MEASURES claim reliability, and fabricating a confidence-1 + * claim would corrupt the top bucket's evidence (same "drop rather than guess" posture as + * repo-corpus-slice's unparseable-key handling). An out-of-range claim (below the first edge, above the + * last) is likewise dropped, never clamped into a bucket. A bucket below `sampleFloor` reports null + * precision, never 0. Throws on malformed `bucketEdges` (fewer than 2, out of [0, 1], or not strictly + * ascending) or a `sampleFloor` below 1 -- caller bugs, mirroring splitBacktestCorpus's guard; the negated + * compound forms make NaN fail closed into the throw. Pure and deterministic. + */ +export function computeReliabilityCurve( + cases: readonly BacktestCase[], + bucketEdges: readonly number[] = DEFAULT_RELIABILITY_BUCKET_EDGES, + sampleFloor: number = RELIABILITY_BUCKET_SAMPLE_FLOOR, +): ReliabilityCurve { + if (bucketEdges.length < 2) { + throw new Error(`invalid_bucket_edges: need at least 2 edges, got ${bucketEdges.length}`); + } + for (let i = 0; i < bucketEdges.length; i++) { + if (!(bucketEdges[i]! >= 0 && bucketEdges[i]! <= 1)) { + throw new Error(`invalid_bucket_edges: edge outside [0, 1]: ${bucketEdges[i]}`); + } + if (i > 0 && !(bucketEdges[i]! > bucketEdges[i - 1]!)) { + throw new Error(`invalid_bucket_edges: edges must be strictly ascending at index ${i}`); + } + } + if (!(sampleFloor >= 1)) { + throw new Error(`invalid_sample_floor: ${sampleFloor}`); + } + const counts = bucketEdges.slice(0, -1).map(() => ({ cases: 0, confirmed: 0, reversed: 0 })); + for (const backtestCase of cases) { + const claimed = backtestCase.metadata?.confidence; + if (typeof claimed !== "number") continue; + const index = bucketIndexFor(claimed, bucketEdges); + if (index === -1) continue; + const bucket = counts[index]!; + bucket.cases += 1; + if (backtestCase.label === "confirmed") bucket.confirmed += 1; + else bucket.reversed += 1; + } + return { + sampleFloor, + buckets: counts.map((count, i) => ({ + floor: bucketEdges[i]!, + ceiling: bucketEdges[i + 1]!, + cases: count.cases, + confirmed: count.confirmed, + reversed: count.reversed, + // sampleFloor >= 1 (validated above), so a passing count.cases is never 0 -- no divide-by-zero arm. + precision: count.cases >= sampleFloor ? count.confirmed / count.cases : null, + })), + }; +} + +/** + * Derive the LOOSEST confidence floor the curve's evidence supports: the lowest bucket floor at or above + * `hardMinimum` whose at-or-above buckets' POOLED precision (pooled confirmed / pooled cases, raw counts -- + * a bucket individually below the sample floor still contributes its cases to the pool) meets + * `targetPrecision`, with the pool itself subject to the curve's own `sampleFloor` (a pooled window below + * it is unknown, not 0, so it can never qualify). Null when no candidate floor qualifies -- including when + * the only precision-meeting floors sit below `hardMinimum` (a suggestion is never clamped UP to a floor + * whose own pooled evidence was not checked) or when pooled density is insufficient everywhere. + * Conservative by construction and deterministic: same curve + parameters, same suggestion. Throws when + * `targetPrecision` or `hardMinimum` is outside [0, 1] (negated compound guards, so NaN fails closed) -- + * caller bugs, mirroring splitBacktestCorpus. + */ +export function deriveThresholdSuggestion( + curve: ReliabilityCurve, + targetPrecision: number, + hardMinimum: number, +): number | null { + if (!(targetPrecision >= 0 && targetPrecision <= 1)) { + throw new Error(`invalid_target_precision: ${targetPrecision}`); + } + if (!(hardMinimum >= 0 && hardMinimum <= 1)) { + throw new Error(`invalid_hard_minimum: ${hardMinimum}`); + } + const { buckets, sampleFloor } = curve; + // Suffix-pooled raw counts: pooledCases[i]/pooledConfirmed[i] cover every bucket whose floor is at or + // above buckets[i].floor (buckets ascend by floor, so the pool for candidate i is the suffix from i). + const pooledCases: number[] = new Array(buckets.length).fill(0); + const pooledConfirmed: number[] = new Array(buckets.length).fill(0); + let cases = 0; + let confirmed = 0; + for (let i = buckets.length - 1; i >= 0; i--) { + cases += buckets[i]!.cases; + confirmed += buckets[i]!.confirmed; + pooledCases[i] = cases; + pooledConfirmed[i] = confirmed; + } + for (let i = 0; i < buckets.length; i++) { + if (buckets[i]!.floor < hardMinimum) continue; + // Suffix pools only shrink as the floor tightens, so once density fails here it fails for every later + // candidate too -- the uniform guard just lets the loop run out to the null below. + if (pooledCases[i]! < sampleFloor) continue; + if (pooledConfirmed[i]! / pooledCases[i]! >= targetPrecision) return buckets[i]!.floor; + } + return null; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index beef2399fc..8b46c9c363 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -178,6 +178,7 @@ export * from "./calibration/backtest-track-record.js"; export * from "./calibration/backtest-split.js"; export * from "./calibration/backtest-threshold.js"; export * from "./calibration/provider-track-record.js"; +export * from "./calibration/reliability-curve.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/reliability-curve.test.ts b/packages/loopover-engine/test/reliability-curve.test.ts new file mode 100644 index 0000000000..388f24766e --- /dev/null +++ b/packages/loopover-engine/test/reliability-curve.test.ts @@ -0,0 +1,286 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + computeReliabilityCurve, + deriveThresholdSuggestion, + DEFAULT_RELIABILITY_BUCKET_EDGES, + RELIABILITY_BUCKET_SAMPLE_FLOOR, + type BacktestCase, + type ReliabilityCurve, +} from "../dist/index.js"; + +function caseWith(label: BacktestCase["label"], confidence?: unknown): BacktestCase { + const backtestCase: BacktestCase = { + ruleId: "linked_issue_scope_mismatch", + targetKey: "acme/widgets#1", + outcome: "block", + label, + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + }; + if (confidence !== undefined) backtestCase.metadata = { confidence }; + return backtestCase; +} + +test("barrel: the public entrypoint re-exports the reliability-curve primitives (#8226)", () => { + assert.equal(typeof computeReliabilityCurve, "function"); + assert.equal(typeof deriveThresholdSuggestion, "function"); + assert.ok(Array.isArray(DEFAULT_RELIABILITY_BUCKET_EDGES)); + assert.equal(typeof RELIABILITY_BUCKET_SAMPLE_FLOOR, "number"); +}); + +test("computeReliabilityCurve: buckets decided cases by claimed confidence and reports per-bucket precision", () => { + const curve = computeReliabilityCurve( + [ + caseWith("confirmed", 0.2), + caseWith("reversed", 0.3), + caseWith("confirmed", 0.7), + caseWith("confirmed", 0.9), + caseWith("reversed", 0.6), + ], + [0, 0.5, 1], + 1, + ); + assert.deepEqual(curve, { + sampleFloor: 1, + buckets: [ + { floor: 0, ceiling: 0.5, cases: 2, confirmed: 1, reversed: 1, precision: 0.5 }, + { floor: 0.5, ceiling: 1, cases: 3, confirmed: 2, reversed: 1, precision: 2 / 3 }, + ], + }); +}); + +test("computeReliabilityCurve: uses the documented default edges and sample floor when omitted", () => { + const curve = computeReliabilityCurve([caseWith("confirmed", 0.95)]); + assert.equal(curve.sampleFloor, RELIABILITY_BUCKET_SAMPLE_FLOOR); + assert.deepEqual( + curve.buckets.map((bucket) => bucket.floor), + DEFAULT_RELIABILITY_BUCKET_EDGES.slice(0, -1), + ); + assert.deepEqual( + curve.buckets.map((bucket) => bucket.ceiling), + DEFAULT_RELIABILITY_BUCKET_EDGES.slice(1), + ); + // Every registry candidate ladder value (loosening-knobs.ts) is a landable bucket floor by default. + for (const registryValue of [0.3, 0.35, 0.4, 0.45, 0.85, 0.9]) { + assert.ok(curve.buckets.some((bucket) => bucket.floor === registryValue), `no bucket floor at ${registryValue}`); + } + const top = curve.buckets[curve.buckets.length - 1]!; + assert.deepEqual([top.floor, top.ceiling], [0.95, 1]); + assert.equal(top.cases, 1); + // 1 case sits below the default sample floor of 5 -> null, never a fabricated 1.0 precision. + assert.equal(top.precision, null); +}); + +test("computeReliabilityCurve: an interior edge is floor-inclusive -- a claim exactly at it lands in the higher bucket", () => { + const curve = computeReliabilityCurve([caseWith("confirmed", 0.5)], [0, 0.5, 1], 1); + assert.equal(curve.buckets[0]!.cases, 0); + assert.equal(curve.buckets[1]!.cases, 1); +}); + +test("computeReliabilityCurve: the first edge is inclusive and the TOP edge is ceiling-inclusive", () => { + const curve = computeReliabilityCurve([caseWith("confirmed", 0), caseWith("reversed", 1)], [0, 0.5, 1], 1); + assert.equal(curve.buckets[0]!.cases, 1); + assert.equal(curve.buckets[1]!.cases, 1); +}); + +test("computeReliabilityCurve: drops cases with no numeric claimed confidence instead of fabricating one (diverges from #8138's degrade-to-1)", () => { + const curve = computeReliabilityCurve( + [ + caseWith("confirmed"), // no metadata at all + { ...caseWith("confirmed"), metadata: {} }, // metadata without confidence + caseWith("confirmed", "high"), // non-numeric claim + ], + [0, 0.5, 1], + 1, + ); + assert.deepEqual( + curve.buckets.map((bucket) => bucket.cases), + [0, 0], + ); +}); + +test("computeReliabilityCurve: drops out-of-range and NaN claims, never clamping them into a bucket", () => { + const curve = computeReliabilityCurve( + [caseWith("confirmed", -0.1), caseWith("confirmed", 1.5), caseWith("confirmed", Number.NaN)], + [0, 0.5, 1], + 1, + ); + assert.deepEqual( + curve.buckets.map((bucket) => bucket.cases), + [0, 0], + ); +}); + +test("computeReliabilityCurve: a bucket below the sample floor reports null precision, never 0", () => { + const curve = computeReliabilityCurve( + [caseWith("reversed", 0.7), caseWith("reversed", 0.7)], + [0, 0.5, 1], + 3, + ); + // 2 all-reversed cases: a coerced precision would read 0 -- the N/A-over-zero rule keeps it null. + assert.equal(curve.buckets[1]!.cases, 2); + assert.equal(curve.buckets[1]!.precision, null); +}); + +test("computeReliabilityCurve: a bucket exactly AT the sample floor reports its real precision", () => { + const curve = computeReliabilityCurve( + [caseWith("confirmed", 0.7), caseWith("confirmed", 0.8), caseWith("reversed", 0.9)], + [0, 0.5, 1], + 3, + ); + assert.equal(curve.buckets[1]!.precision, 2 / 3); +}); + +test("computeReliabilityCurve: an empty corpus yields all-zero buckets with null precision everywhere", () => { + const curve = computeReliabilityCurve([], [0, 0.5, 1], 1); + assert.deepEqual(curve.buckets, [ + { floor: 0, ceiling: 0.5, cases: 0, confirmed: 0, reversed: 0, precision: null }, + { floor: 0.5, ceiling: 1, cases: 0, confirmed: 0, reversed: 0, precision: null }, + ]); +}); + +test("computeReliabilityCurve: throws on malformed bucket edges", () => { + assert.throws(() => computeReliabilityCurve([], [0.5], 1), /invalid_bucket_edges/); + assert.throws(() => computeReliabilityCurve([], [0, 0.5, 0.5, 1], 1), /invalid_bucket_edges/); + assert.throws(() => computeReliabilityCurve([], [0, 0.7, 0.5, 1], 1), /invalid_bucket_edges/); + assert.throws(() => computeReliabilityCurve([], [-0.1, 0.5, 1], 1), /invalid_bucket_edges/); + assert.throws(() => computeReliabilityCurve([], [0, 0.5, 1.5], 1), /invalid_bucket_edges/); + assert.throws(() => computeReliabilityCurve([], [0, Number.NaN, 1], 1), /invalid_bucket_edges/); +}); + +test("computeReliabilityCurve: throws on a sample floor below 1 (NaN fails closed into the throw)", () => { + assert.throws(() => computeReliabilityCurve([], [0, 1], 0), /invalid_sample_floor/); + assert.throws(() => computeReliabilityCurve([], [0, 1], Number.NaN), /invalid_sample_floor/); +}); + +test("computeReliabilityCurve: deterministic -- identical inputs yield an equal curve", () => { + const cases = [caseWith("confirmed", 0.4), caseWith("reversed", 0.8)]; + assert.deepEqual(computeReliabilityCurve(cases, [0, 0.5, 1], 1), computeReliabilityCurve(cases, [0, 0.5, 1], 1)); +}); + +function curveOf(sampleFloor: number, buckets: Array<[floor: number, ceiling: number, confirmed: number, reversed: number]>): ReliabilityCurve { + return { + sampleFloor, + buckets: buckets.map(([floor, ceiling, confirmed, reversed]) => ({ + floor, + ceiling, + cases: confirmed + reversed, + confirmed, + reversed, + precision: confirmed + reversed >= sampleFloor ? confirmed / (confirmed + reversed) : null, + })), + }; +} + +test("deriveThresholdSuggestion: suggests the LOOSEST floor whose at-or-above pooled precision meets the target", () => { + // Pooled from 0: (4 + 5 confirmed) / 10 = 0.9 -- already at target, so the loosest floor wins. + const curve = curveOf(5, [ + [0, 0.5, 4, 1], + [0.5, 1, 5, 0], + ]); + assert.equal(deriveThresholdSuggestion(curve, 0.9, 0), 0); +}); + +test("deriveThresholdSuggestion: a weak low bucket dilutes the pool and pushes the suggestion up", () => { + // Pooled from 0: 6/10 = 0.6 < 0.9; pooled from 0.5: 5/5 = 1 >= 0.9. + const curve = curveOf(5, [ + [0, 0.5, 1, 4], + [0.5, 1, 5, 0], + ]); + assert.equal(deriveThresholdSuggestion(curve, 0.9, 0), 0.5); +}); + +test("deriveThresholdSuggestion: buckets below the per-bucket sample floor still contribute raw counts to the pool", () => { + // Each bucket alone is below the floor of 5 (3 and 4 cases -> null bucket precision), but the pooled + // window from 0.5 has 7 cases -- enough density for a real, qualifying pooled precision. + const curve = curveOf(5, [ + [0.5, 0.8, 3, 0], + [0.8, 1, 4, 0], + ]); + assert.equal(curve.buckets[0]!.precision, null); + assert.equal(curve.buckets[1]!.precision, null); + assert.equal(deriveThresholdSuggestion(curve, 0.9, 0), 0.5); +}); + +test("deriveThresholdSuggestion: never suggests below the hard minimum -- an at-minimum bucket floor is the loosest candidate", () => { + const curve = curveOf(5, [ + [0, 0.5, 5, 0], + [0.5, 1, 5, 0], + ]); + // Floor 0 qualifies on evidence but sits below the hard minimum, so the at-minimum floor 0.5 is suggested. + assert.equal(deriveThresholdSuggestion(curve, 0.9, 0.5), 0.5); +}); + +test("deriveThresholdSuggestion: a hard minimum inside a bucket excludes that bucket's floor as a candidate", () => { + const curve = curveOf(5, [ + [0.5, 0.7, 5, 0], + [0.7, 1, 5, 0], + ]); + assert.equal(deriveThresholdSuggestion(curve, 0.9, 0.6), 0.7); +}); + +test("deriveThresholdSuggestion: null when the only qualifying floors sit below the hard minimum", () => { + // Pooled from 0: 9/10 = 0.9 qualifies; pooled from 0.5 alone: 4/5 = 0.8 does not. No clamping up. + const curve = curveOf(5, [ + [0, 0.5, 5, 0], + [0.5, 1, 4, 1], + ]); + assert.equal(deriveThresholdSuggestion(curve, 0.9, 0.5), null); +}); + +test("deriveThresholdSuggestion: null when pooled density is insufficient everywhere (N/A over a fabricated qualifier)", () => { + const curve = curveOf(5, [ + [0, 0.5, 2, 0], + [0.5, 1, 2, 0], + ]); + // Every pooled window has 4 < 5 cases: perfect-looking precision, but the evidence is too thin to act on. + assert.equal(deriveThresholdSuggestion(curve, 0.5, 0), null); +}); + +test("deriveThresholdSuggestion: null when no pooled window meets the target precision", () => { + const curve = curveOf(5, [ + [0, 0.5, 3, 2], + [0.5, 1, 3, 2], + ]); + assert.equal(deriveThresholdSuggestion(curve, 0.95, 0), null); +}); + +test("deriveThresholdSuggestion: null for an empty-corpus curve", () => { + assert.equal(deriveThresholdSuggestion(computeReliabilityCurve([], [0, 0.5, 1], 5), 0.5, 0), null); +}); + +test("deriveThresholdSuggestion: throws when targetPrecision or hardMinimum is outside [0, 1] (NaN fails closed)", () => { + const curve = curveOf(1, [[0, 1, 1, 0]]); + assert.throws(() => deriveThresholdSuggestion(curve, -0.1, 0), /invalid_target_precision/); + assert.throws(() => deriveThresholdSuggestion(curve, 1.5, 0), /invalid_target_precision/); + assert.throws(() => deriveThresholdSuggestion(curve, Number.NaN, 0), /invalid_target_precision/); + assert.throws(() => deriveThresholdSuggestion(curve, 0.9, -0.1), /invalid_hard_minimum/); + assert.throws(() => deriveThresholdSuggestion(curve, 0.9, 1.5), /invalid_hard_minimum/); + assert.throws(() => deriveThresholdSuggestion(curve, 0.9, Number.NaN), /invalid_hard_minimum/); +}); + +test("deriveThresholdSuggestion: invariant -- as the target rises the suggestion only tightens (never loosens), never dips below the hard minimum, and is always a bucket floor", () => { + const curve = curveOf(3, [ + [0, 0.3, 2, 3], + [0.3, 0.5, 3, 2], + [0.5, 0.7, 4, 1], + [0.7, 0.9, 4, 0], + [0.9, 1, 3, 0], + ]); + const floors = curve.buckets.map((bucket) => bucket.floor); + for (const hardMinimum of [0, 0.3, 0.5, 0.9]) { + let previous = -Infinity; + for (const target of [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 1]) { + const suggestion = deriveThresholdSuggestion(curve, target, hardMinimum); + const effective = suggestion ?? Infinity; // null = "no floor qualifies" = tighter than any floor + assert.ok(effective >= previous, `target ${target} loosened the suggestion (${suggestion}) under minimum ${hardMinimum}`); + if (suggestion !== null) { + assert.ok(suggestion >= hardMinimum, `suggestion ${suggestion} fell below the hard minimum ${hardMinimum}`); + assert.ok(floors.includes(suggestion), `suggestion ${suggestion} is not one of the curve's bucket floors`); + } + previous = effective; + } + } +}); diff --git a/test/unit/reliability-curve-engine.test.ts b/test/unit/reliability-curve-engine.test.ts new file mode 100644 index 0000000000..f79ce72e2e --- /dev/null +++ b/test/unit/reliability-curve-engine.test.ts @@ -0,0 +1,267 @@ +import { describe, expect, it } from "vitest"; +// Direct src-path import (not the `@loopover/engine` package barrel, which resolves to dist and is NOT in +// vitest's coverage.include): the engine's own node:test suite runs against dist and is invisible to Codecov, +// so this vitest mirror is what gives packages/loopover-engine/src/calibration/reliability-curve.ts its +// codecov/patch coverage (the "engine blind-spot rule"). The companion +// packages/loopover-engine/test/reliability-curve.test.ts is the node:test that gates the engine workspace's +// own `npm run test`. Vite resolves the `.js` specifier to the sibling `.ts` on disk. +import { + computeReliabilityCurve, + deriveThresholdSuggestion, + DEFAULT_RELIABILITY_BUCKET_EDGES, + RELIABILITY_BUCKET_SAMPLE_FLOOR, + type ReliabilityCurve, +} from "../../packages/loopover-engine/src/calibration/reliability-curve.js"; +import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus.js"; + +function caseWith(label: BacktestCase["label"], confidence?: unknown): BacktestCase { + const backtestCase: BacktestCase = { + ruleId: "linked_issue_scope_mismatch", + targetKey: "acme/widgets#1", + outcome: "block", + label, + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + }; + if (confidence !== undefined) backtestCase.metadata = { confidence }; + return backtestCase; +} + +function curveOf(sampleFloor: number, buckets: Array<[floor: number, ceiling: number, confirmed: number, reversed: number]>): ReliabilityCurve { + return { + sampleFloor, + buckets: buckets.map(([floor, ceiling, confirmed, reversed]) => ({ + floor, + ceiling, + cases: confirmed + reversed, + confirmed, + reversed, + precision: confirmed + reversed >= sampleFloor ? confirmed / (confirmed + reversed) : null, + })), + }; +} + +describe("computeReliabilityCurve (#8226)", () => { + it("buckets decided cases by claimed confidence and reports per-bucket precision", () => { + const curve = computeReliabilityCurve( + [ + caseWith("confirmed", 0.2), + caseWith("reversed", 0.3), + caseWith("confirmed", 0.7), + caseWith("confirmed", 0.9), + caseWith("reversed", 0.6), + ], + [0, 0.5, 1], + 1, + ); + expect(curve).toEqual({ + sampleFloor: 1, + buckets: [ + { floor: 0, ceiling: 0.5, cases: 2, confirmed: 1, reversed: 1, precision: 0.5 }, + { floor: 0.5, ceiling: 1, cases: 3, confirmed: 2, reversed: 1, precision: 2 / 3 }, + ], + }); + }); + + it("uses the documented default edges and sample floor when omitted", () => { + const curve = computeReliabilityCurve([caseWith("confirmed", 0.95)]); + expect(curve.sampleFloor).toBe(RELIABILITY_BUCKET_SAMPLE_FLOOR); + expect(curve.buckets.map((bucket) => bucket.floor)).toEqual(DEFAULT_RELIABILITY_BUCKET_EDGES.slice(0, -1)); + expect(curve.buckets.map((bucket) => bucket.ceiling)).toEqual(DEFAULT_RELIABILITY_BUCKET_EDGES.slice(1)); + // Every registry candidate ladder value (loosening-knobs.ts) is a landable bucket floor by default. + for (const registryValue of [0.3, 0.35, 0.4, 0.45, 0.85, 0.9]) { + expect(curve.buckets.some((bucket) => bucket.floor === registryValue)).toBe(true); + } + const top = curve.buckets[curve.buckets.length - 1]!; + expect([top.floor, top.ceiling]).toEqual([0.95, 1]); + expect(top.cases).toBe(1); + // 1 case sits below the default sample floor of 5 -> null, never a fabricated 1.0 precision. + expect(top.precision).toBeNull(); + }); + + it("treats an interior edge as floor-inclusive -- a claim exactly at it lands in the higher bucket", () => { + const curve = computeReliabilityCurve([caseWith("confirmed", 0.5)], [0, 0.5, 1], 1); + expect(curve.buckets[0]!.cases).toBe(0); + expect(curve.buckets[1]!.cases).toBe(1); + }); + + it("keeps the first edge inclusive and the TOP edge ceiling-inclusive", () => { + const curve = computeReliabilityCurve([caseWith("confirmed", 0), caseWith("reversed", 1)], [0, 0.5, 1], 1); + expect(curve.buckets[0]!.cases).toBe(1); + expect(curve.buckets[1]!.cases).toBe(1); + }); + + it("drops cases with no numeric claimed confidence instead of fabricating one (diverges from #8138's degrade-to-1)", () => { + const curve = computeReliabilityCurve( + [caseWith("confirmed"), { ...caseWith("confirmed"), metadata: {} }, caseWith("confirmed", "high")], + [0, 0.5, 1], + 1, + ); + expect(curve.buckets.map((bucket) => bucket.cases)).toEqual([0, 0]); + }); + + it("drops out-of-range and NaN claims, never clamping them into a bucket", () => { + const curve = computeReliabilityCurve( + [caseWith("confirmed", -0.1), caseWith("confirmed", 1.5), caseWith("confirmed", Number.NaN)], + [0, 0.5, 1], + 1, + ); + expect(curve.buckets.map((bucket) => bucket.cases)).toEqual([0, 0]); + }); + + it("reports null precision, never 0, for a bucket below the sample floor", () => { + const curve = computeReliabilityCurve([caseWith("reversed", 0.7), caseWith("reversed", 0.7)], [0, 0.5, 1], 3); + // 2 all-reversed cases: a coerced precision would read 0 -- the N/A-over-zero rule keeps it null. + expect(curve.buckets[1]!.cases).toBe(2); + expect(curve.buckets[1]!.precision).toBeNull(); + }); + + it("reports the real precision for a bucket exactly AT the sample floor", () => { + const curve = computeReliabilityCurve( + [caseWith("confirmed", 0.7), caseWith("confirmed", 0.8), caseWith("reversed", 0.9)], + [0, 0.5, 1], + 3, + ); + expect(curve.buckets[1]!.precision).toBe(2 / 3); + }); + + it("yields all-zero buckets with null precision everywhere for an empty corpus", () => { + const curve = computeReliabilityCurve([], [0, 0.5, 1], 1); + expect(curve.buckets).toEqual([ + { floor: 0, ceiling: 0.5, cases: 0, confirmed: 0, reversed: 0, precision: null }, + { floor: 0.5, ceiling: 1, cases: 0, confirmed: 0, reversed: 0, precision: null }, + ]); + }); + + it("throws on malformed bucket edges", () => { + expect(() => computeReliabilityCurve([], [0.5], 1)).toThrow(/invalid_bucket_edges/); + expect(() => computeReliabilityCurve([], [0, 0.5, 0.5, 1], 1)).toThrow(/invalid_bucket_edges/); + expect(() => computeReliabilityCurve([], [0, 0.7, 0.5, 1], 1)).toThrow(/invalid_bucket_edges/); + expect(() => computeReliabilityCurve([], [-0.1, 0.5, 1], 1)).toThrow(/invalid_bucket_edges/); + expect(() => computeReliabilityCurve([], [0, 0.5, 1.5], 1)).toThrow(/invalid_bucket_edges/); + expect(() => computeReliabilityCurve([], [0, Number.NaN, 1], 1)).toThrow(/invalid_bucket_edges/); + }); + + it("throws on a sample floor below 1 (NaN fails closed into the throw)", () => { + expect(() => computeReliabilityCurve([], [0, 1], 0)).toThrow(/invalid_sample_floor/); + expect(() => computeReliabilityCurve([], [0, 1], Number.NaN)).toThrow(/invalid_sample_floor/); + }); + + it("is deterministic -- identical inputs yield an equal curve", () => { + const cases = [caseWith("confirmed", 0.4), caseWith("reversed", 0.8)]; + expect(computeReliabilityCurve(cases, [0, 0.5, 1], 1)).toEqual(computeReliabilityCurve(cases, [0, 0.5, 1], 1)); + }); +}); + +describe("deriveThresholdSuggestion (#8226)", () => { + it("suggests the LOOSEST floor whose at-or-above pooled precision meets the target", () => { + // Pooled from 0: (4 + 5 confirmed) / 10 = 0.9 -- already at target, so the loosest floor wins. + const curve = curveOf(5, [ + [0, 0.5, 4, 1], + [0.5, 1, 5, 0], + ]); + expect(deriveThresholdSuggestion(curve, 0.9, 0)).toBe(0); + }); + + it("pushes the suggestion up when a weak low bucket dilutes the pool", () => { + // Pooled from 0: 6/10 = 0.6 < 0.9; pooled from 0.5: 5/5 = 1 >= 0.9. + const curve = curveOf(5, [ + [0, 0.5, 1, 4], + [0.5, 1, 5, 0], + ]); + expect(deriveThresholdSuggestion(curve, 0.9, 0)).toBe(0.5); + }); + + it("pools raw counts from buckets individually below the per-bucket sample floor", () => { + // Each bucket alone is below the floor of 5 (3 and 4 cases -> null bucket precision), but the pooled + // window from 0.5 has 7 cases -- enough density for a real, qualifying pooled precision. + const curve = curveOf(5, [ + [0.5, 0.8, 3, 0], + [0.8, 1, 4, 0], + ]); + expect(curve.buckets[0]!.precision).toBeNull(); + expect(curve.buckets[1]!.precision).toBeNull(); + expect(deriveThresholdSuggestion(curve, 0.9, 0)).toBe(0.5); + }); + + it("never suggests below the hard minimum -- an at-minimum bucket floor is the loosest candidate", () => { + const curve = curveOf(5, [ + [0, 0.5, 5, 0], + [0.5, 1, 5, 0], + ]); + // Floor 0 qualifies on evidence but sits below the hard minimum, so the at-minimum floor 0.5 is suggested. + expect(deriveThresholdSuggestion(curve, 0.9, 0.5)).toBe(0.5); + }); + + it("excludes a bucket's floor as a candidate when the hard minimum falls inside the bucket", () => { + const curve = curveOf(5, [ + [0.5, 0.7, 5, 0], + [0.7, 1, 5, 0], + ]); + expect(deriveThresholdSuggestion(curve, 0.9, 0.6)).toBe(0.7); + }); + + it("returns null when the only qualifying floors sit below the hard minimum", () => { + // Pooled from 0: 9/10 = 0.9 qualifies; pooled from 0.5 alone: 4/5 = 0.8 does not. No clamping up. + const curve = curveOf(5, [ + [0, 0.5, 5, 0], + [0.5, 1, 4, 1], + ]); + expect(deriveThresholdSuggestion(curve, 0.9, 0.5)).toBeNull(); + }); + + it("returns null when pooled density is insufficient everywhere (N/A over a fabricated qualifier)", () => { + const curve = curveOf(5, [ + [0, 0.5, 2, 0], + [0.5, 1, 2, 0], + ]); + // Every pooled window has 4 < 5 cases: perfect-looking precision, but the evidence is too thin to act on. + expect(deriveThresholdSuggestion(curve, 0.5, 0)).toBeNull(); + }); + + it("returns null when no pooled window meets the target precision", () => { + const curve = curveOf(5, [ + [0, 0.5, 3, 2], + [0.5, 1, 3, 2], + ]); + expect(deriveThresholdSuggestion(curve, 0.95, 0)).toBeNull(); + }); + + it("returns null for an empty-corpus curve", () => { + expect(deriveThresholdSuggestion(computeReliabilityCurve([], [0, 0.5, 1], 5), 0.5, 0)).toBeNull(); + }); + + it("throws when targetPrecision or hardMinimum is outside [0, 1] (NaN fails closed)", () => { + const curve = curveOf(1, [[0, 1, 1, 0]]); + expect(() => deriveThresholdSuggestion(curve, -0.1, 0)).toThrow(/invalid_target_precision/); + expect(() => deriveThresholdSuggestion(curve, 1.5, 0)).toThrow(/invalid_target_precision/); + expect(() => deriveThresholdSuggestion(curve, Number.NaN, 0)).toThrow(/invalid_target_precision/); + expect(() => deriveThresholdSuggestion(curve, 0.9, -0.1)).toThrow(/invalid_hard_minimum/); + expect(() => deriveThresholdSuggestion(curve, 0.9, 1.5)).toThrow(/invalid_hard_minimum/); + expect(() => deriveThresholdSuggestion(curve, 0.9, Number.NaN)).toThrow(/invalid_hard_minimum/); + }); + + it("invariant: as the target rises the suggestion only tightens, never dips below the hard minimum, and is always a bucket floor", () => { + const curve = curveOf(3, [ + [0, 0.3, 2, 3], + [0.3, 0.5, 3, 2], + [0.5, 0.7, 4, 1], + [0.7, 0.9, 4, 0], + [0.9, 1, 3, 0], + ]); + const floors = curve.buckets.map((bucket) => bucket.floor); + for (const hardMinimum of [0, 0.3, 0.5, 0.9]) { + let previous = -Infinity; + for (const target of [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 1]) { + const suggestion = deriveThresholdSuggestion(curve, target, hardMinimum); + const effective = suggestion ?? Infinity; // null = "no floor qualifies" = tighter than any floor + expect(effective).toBeGreaterThanOrEqual(previous); + if (suggestion !== null) { + expect(suggestion).toBeGreaterThanOrEqual(hardMinimum); + expect(floors).toContain(suggestion); + } + previous = effective; + } + } + }); +});