From 3dd08e5d8b9fc66977fb5df71c7aff4320f74bfe Mon Sep 17 00:00:00 2001 From: michiot05 <281539540+michiot05@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:26:42 +0200 Subject: [PATCH] feat(calibration): pure per-repo corpus slicing + density stats Closes #8215. Adds packages/loopover-engine/src/calibration/repo-corpus-slice.ts with sliceCorpusByRepo (group a BacktestCase[] by owner/repo parsed from targetKey's last '#', dropping unparseable keys, order-preserving) and computeRepoCorpusDensity (per-repo cases/confirmed/reversed plus an eligible flag that applies the same splitBacktestCorpus split + sample-minimum floors on each repo's own slice). Pure, aggregate-only. Barrel export + node:test (engine gate) + root vitest mirror (codecov coverage). --- .../src/calibration/repo-corpus-slice.ts | 80 ++++++++++++++++ packages/loopover-engine/src/index.ts | 1 + .../test/repo-corpus-slice.test.ts | 92 +++++++++++++++++++ test/unit/repo-corpus-slice-engine.test.ts | 85 +++++++++++++++++ 4 files changed, 258 insertions(+) create mode 100644 packages/loopover-engine/src/calibration/repo-corpus-slice.ts create mode 100644 packages/loopover-engine/test/repo-corpus-slice.test.ts create mode 100644 test/unit/repo-corpus-slice-engine.test.ts diff --git a/packages/loopover-engine/src/calibration/repo-corpus-slice.ts b/packages/loopover-engine/src/calibration/repo-corpus-slice.ts new file mode 100644 index 0000000000..40a861f686 --- /dev/null +++ b/packages/loopover-engine/src/calibration/repo-corpus-slice.ts @@ -0,0 +1,80 @@ +// Per-repo corpus slicing + density stats (#8215, epic #8211 track B) -- the pure building block per-repo +// autonomy needs before any per-repo evaluation can run. Every BacktestCase carries its repo inside +// `targetKey` (`owner/repo#N`), but the calibration primitives only ever evaluate globally; these two +// functions carve a corpus into per-repo slices and report which repos have enough labeled density to be +// evaluable at all. +// +// PURE, storage-agnostic, aggregate-only: no IO, no store reads, no registry access; every returned shape is +// repo names + counts + a boolean -- never a targetKey or metadata -- the same public-safe discipline the +// rest of the calibration module follows (#8083-#8087). + +import type { BacktestCase } from "./backtest-corpus.js"; +import { splitBacktestCorpus } from "./backtest-split.js"; + +/** Per-repo labeled-density summary: total cases in the repo's slice, their confirmed/reversed breakdown, and + * whether the slice clears the evaluation floors after the standard held-out split. Aggregate-only. */ +export type RepoCorpusDensity = { + cases: number; + confirmed: number; + reversed: number; + eligible: boolean; +}; + +/** Parse the `owner/repo` prefix from a `BacktestCase.targetKey` shaped `owner/repo#N`. Returns null for a key + * with no `#`, or with an empty repo portion before the last `#` -- the caller drops those rather than + * guessing a repo (`lastIndexOf` so a stray `#` inside an issue fragment can't truncate the repo early). */ +function repoFromTargetKey(targetKey: string): string | null { + const hashIndex = targetKey.lastIndexOf("#"); + if (hashIndex <= 0) return null; + return targetKey.slice(0, hashIndex); +} + +/** + * Slice a corpus into per-repo groups keyed by `owner/repo`, deterministically. A case whose `targetKey` has + * no parseable repo (no `#`, or an empty prefix) is dropped, never guessed. Case order within each slice + * preserves the input order; Map key order is first-seen-repo order -- both deterministic for a given input. + */ +export function sliceCorpusByRepo(cases: readonly BacktestCase[]): Map { + const slices = new Map(); + for (const backtestCase of cases) { + const repo = repoFromTargetKey(backtestCase.targetKey); + if (repo === null) continue; + const existing = slices.get(repo); + if (existing) existing.push(backtestCase); + else slices.set(repo, [backtestCase]); + } + return slices; +} + +/** + * Per-repo density report. For each repo slice: total `cases`, the `confirmed`/`reversed` label split, and + * `eligible` -- true only when the repo's OWN slice, split by the same `splitBacktestCorpus` seed/fraction the + * knob evaluators use, yields at least `minVisible` visible and `minHeldOut` held-out cases. A repo is + * evaluable only if its own slice clears both floors, so a large global corpus can't lend density to a + * sparse repo. Aggregate-only output (repo names + numbers). Deterministic for a given input + params. + */ +export function computeRepoCorpusDensity( + cases: readonly BacktestCase[], + minVisible: number, + minHeldOut: number, + heldOutFraction: number, + splitSeed: string, +): Map { + const density = new Map(); + for (const [repo, repoCases] of sliceCorpusByRepo(cases)) { + let confirmed = 0; + let reversed = 0; + for (const backtestCase of repoCases) { + if (backtestCase.label === "confirmed") confirmed += 1; + else reversed += 1; + } + const { visible, heldOut } = splitBacktestCorpus(repoCases, heldOutFraction, splitSeed); + density.set(repo, { + cases: repoCases.length, + confirmed, + reversed, + eligible: visible.length >= minVisible && heldOut.length >= minHeldOut, + }); + } + return density; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index bea83db60d..1b533b0bad 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -164,6 +164,7 @@ export * from "./governor/action-mode.js"; export * from "./governor/chokepoint.js"; export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; +export * from "./calibration/repo-corpus-slice.js"; export * from "./calibration/ams-prediction-corpus.js"; export * from "./calibration/backtest-score.js"; export * from "./calibration/backtest-compare.js"; diff --git a/packages/loopover-engine/test/repo-corpus-slice.test.ts b/packages/loopover-engine/test/repo-corpus-slice.test.ts new file mode 100644 index 0000000000..c17f33a28e --- /dev/null +++ b/packages/loopover-engine/test/repo-corpus-slice.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { computeRepoCorpusDensity, sliceCorpusByRepo, type BacktestCase } from "../dist/index.js"; + +function bcase(targetKey: string, label: BacktestCase["label"] = "confirmed"): BacktestCase { + return { + ruleId: "missing_linked_issue", + targetKey, + outcome: "block", + label, + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + }; +} + +test("barrel: the public entrypoint re-exports the repo-corpus-slice primitives (#8215)", () => { + assert.equal(typeof sliceCorpusByRepo, "function"); + assert.equal(typeof computeRepoCorpusDensity, "function"); +}); + +test("sliceCorpusByRepo: groups by owner/repo, preserving input order within each slice and first-seen key order", () => { + const slices = sliceCorpusByRepo([ + bcase("acme/widgets#1"), + bcase("acme/gadgets#5"), + bcase("acme/widgets#2"), + bcase("acme/gadgets#9"), + ]); + assert.deepEqual([...slices.keys()], ["acme/widgets", "acme/gadgets"]); + assert.deepEqual(slices.get("acme/widgets")!.map((c) => c.targetKey), ["acme/widgets#1", "acme/widgets#2"]); + assert.deepEqual(slices.get("acme/gadgets")!.map((c) => c.targetKey), ["acme/gadgets#5", "acme/gadgets#9"]); +}); + +test("sliceCorpusByRepo: drops targetKeys with no '#' or an empty repo prefix, never guessing", () => { + const slices = sliceCorpusByRepo([bcase("acme/widgets#1"), bcase("no-hash"), bcase("#123")]); + assert.deepEqual([...slices.keys()], ["acme/widgets"]); + assert.equal(slices.get("acme/widgets")!.length, 1); +}); + +test("sliceCorpusByRepo: parses the repo from the LAST '#' so an issue-fragment hash can't truncate it early", () => { + const slices = sliceCorpusByRepo([bcase("acme/widgets#12#note")]); + assert.deepEqual([...slices.keys()], ["acme/widgets#12"]); +}); + +test("computeRepoCorpusDensity: reports per-repo case counts and the confirmed/reversed label split", () => { + const density = computeRepoCorpusDensity( + [bcase("acme/widgets#1", "confirmed"), bcase("acme/widgets#2", "reversed"), bcase("acme/widgets#3", "confirmed")], + 0, + 0, + 0, + "seed", + ); + assert.deepEqual(density.get("acme/widgets"), { cases: 3, confirmed: 2, reversed: 1, eligible: true }); +}); + +test("computeRepoCorpusDensity: eligible=true when a repo's own slice clears both floors after the split", () => { + // heldOutFraction 0 -> every case is visible, none held out; minHeldOut 0 keeps the held-out floor satisfied. + const density = computeRepoCorpusDensity([bcase("acme/widgets#1"), bcase("acme/widgets#2")], 2, 0, 0, "seed"); + assert.equal(density.get("acme/widgets")!.eligible, true); +}); + +test("computeRepoCorpusDensity: eligible=false when the visible floor is not met (left side of the &&)", () => { + const density = computeRepoCorpusDensity([bcase("acme/widgets#1"), bcase("acme/widgets#2")], 5, 0, 0, "seed"); + assert.equal(density.get("acme/widgets")!.eligible, false); +}); + +test("computeRepoCorpusDensity: eligible=false when the held-out floor is not met (right side of the &&)", () => { + // fraction 0 -> heldOut is empty, so a minHeldOut of 1 fails even though the visible floor passes. + const density = computeRepoCorpusDensity([bcase("acme/widgets#1"), bcase("acme/widgets#2")], 1, 1, 0, "seed"); + assert.equal(density.get("acme/widgets")!.eligible, false); +}); + +test("computeRepoCorpusDensity: a large global corpus cannot lend density to a sparse repo (own-slice floors)", () => { + const cases = [ + ...Array.from({ length: 8 }, (_unused, i) => bcase(`acme/big#${i}`)), + bcase("acme/small#1"), + ]; + const density = computeRepoCorpusDensity(cases, 3, 0, 0, "seed"); + assert.equal(density.get("acme/big")!.eligible, true); + assert.equal(density.get("acme/small")!.eligible, false); +}); + +test("computeRepoCorpusDensity: deterministic — same corpus + params yields an equal report", () => { + const cases = [bcase("acme/widgets#1", "confirmed"), bcase("acme/gadgets#2", "reversed")]; + const a = computeRepoCorpusDensity(cases, 1, 0, 0.5, "seed"); + const b = computeRepoCorpusDensity(cases, 1, 0, 0.5, "seed"); + assert.deepEqual([...a.entries()], [...b.entries()]); +}); + +test("computeRepoCorpusDensity: an empty corpus yields an empty density map", () => { + assert.equal(computeRepoCorpusDensity([], 1, 1, 0.5, "seed").size, 0); +}); diff --git a/test/unit/repo-corpus-slice-engine.test.ts b/test/unit/repo-corpus-slice-engine.test.ts new file mode 100644 index 0000000000..0e6a640257 --- /dev/null +++ b/test/unit/repo-corpus-slice-engine.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +// Direct src-path import (not the `@loopover/engine` package barrel, which resolves to dist and is NOT in +// vitest's coverage.include): the engine's own node:test suite runs against dist and is invisible to Codecov, +// so this vitest mirror is what gives packages/loopover-engine/src/calibration/repo-corpus-slice.ts its +// codecov/patch coverage (the "engine blind-spot rule"). The companion +// packages/loopover-engine/test/repo-corpus-slice.test.ts is the node:test that gates the engine workspace's +// own `npm run test`. Vite resolves the `.js` specifier to the sibling `.ts` on disk. +import { computeRepoCorpusDensity, sliceCorpusByRepo } from "../../packages/loopover-engine/src/calibration/repo-corpus-slice.js"; +import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus.js"; + +function bcase(targetKey: string, label: BacktestCase["label"] = "confirmed"): BacktestCase { + return { + ruleId: "missing_linked_issue", + targetKey, + outcome: "block", + label, + firedAt: "2026-07-22T00:00:00.000Z", + decidedAt: "2026-07-22T01:00:00.000Z", + }; +} + +describe("sliceCorpusByRepo (#8215)", () => { + it("groups by owner/repo, preserving input order within each slice and first-seen key order", () => { + const slices = sliceCorpusByRepo([bcase("acme/widgets#1"), bcase("acme/gadgets#5"), bcase("acme/widgets#2")]); + expect([...slices.keys()]).toEqual(["acme/widgets", "acme/gadgets"]); + expect(slices.get("acme/widgets")!.map((c) => c.targetKey)).toEqual(["acme/widgets#1", "acme/widgets#2"]); + expect(slices.get("acme/gadgets")!.map((c) => c.targetKey)).toEqual(["acme/gadgets#5"]); + }); + + it("drops targetKeys with no '#' or an empty repo prefix, never guessing", () => { + const slices = sliceCorpusByRepo([bcase("acme/widgets#1"), bcase("no-hash"), bcase("#123")]); + expect([...slices.keys()]).toEqual(["acme/widgets"]); + expect(slices.get("acme/widgets")!).toHaveLength(1); + }); + + it("parses the repo from the LAST '#' so an issue-fragment hash can't truncate it early", () => { + expect([...sliceCorpusByRepo([bcase("acme/widgets#12#note")]).keys()]).toEqual(["acme/widgets#12"]); + }); +}); + +describe("computeRepoCorpusDensity (#8215)", () => { + it("reports per-repo case counts and the confirmed/reversed label split", () => { + const density = computeRepoCorpusDensity( + [bcase("acme/widgets#1", "confirmed"), bcase("acme/widgets#2", "reversed"), bcase("acme/widgets#3", "confirmed")], + 0, + 0, + 0, + "seed", + ); + expect(density.get("acme/widgets")).toEqual({ cases: 3, confirmed: 2, reversed: 1, eligible: true }); + }); + + it("eligible=true when a repo's own slice clears both floors after the split", () => { + const density = computeRepoCorpusDensity([bcase("acme/widgets#1"), bcase("acme/widgets#2")], 2, 0, 0, "seed"); + expect(density.get("acme/widgets")!.eligible).toBe(true); + }); + + it("eligible=false when the visible floor is not met (left side of the &&)", () => { + const density = computeRepoCorpusDensity([bcase("acme/widgets#1"), bcase("acme/widgets#2")], 5, 0, 0, "seed"); + expect(density.get("acme/widgets")!.eligible).toBe(false); + }); + + it("eligible=false when the held-out floor is not met (right side of the &&)", () => { + const density = computeRepoCorpusDensity([bcase("acme/widgets#1"), bcase("acme/widgets#2")], 1, 1, 0, "seed"); + expect(density.get("acme/widgets")!.eligible).toBe(false); + }); + + it("a large global corpus cannot lend density to a sparse repo (own-slice floors)", () => { + const cases = [...Array.from({ length: 8 }, (_unused, i) => bcase(`acme/big#${i}`)), bcase("acme/small#1")]; + const density = computeRepoCorpusDensity(cases, 3, 0, 0, "seed"); + expect(density.get("acme/big")!.eligible).toBe(true); + expect(density.get("acme/small")!.eligible).toBe(false); + }); + + it("is deterministic — same corpus + params yields an equal report", () => { + const cases = [bcase("acme/widgets#1", "confirmed"), bcase("acme/gadgets#2", "reversed")]; + expect([...computeRepoCorpusDensity(cases, 1, 0, 0.5, "seed").entries()]).toEqual( + [...computeRepoCorpusDensity(cases, 1, 0, 0.5, "seed").entries()], + ); + }); + + it("an empty corpus yields an empty density map", () => { + expect(computeRepoCorpusDensity([], 1, 1, 0.5, "seed").size).toBe(0); + }); +});