feat(calibration): pure per-repo corpus slicing and labeled-density stats

packages/loopover-engine/src/calibration/repo-corpus.ts

@@ -0,0 +1,75 @@
+// Per-repo corpus slicing + labeled-density stats (#8215, epic #8211 track B). Every BacktestCase carries
+// its repo inside `targetKey` (`owner/repo#N` — host-defined by signal-tracking's own convention), but the
+// calibration primitives only ever evaluate globally; per-repo autonomy needs these two pure building blocks
+// first: slice a corpus by repo, and know which repos have enough labeled density to evaluate AT ALL under
+// the same split + sample-minimum discipline the knob evaluators already apply globally.
+//
+// Aggregates only in every returned shape (repo names + numbers) — no target keys, no metadata — so a
+// consumer can surface these results without re-auditing the public/private boundary. Same purity contract
+// as the rest of this module family: no IO, no randomness, no wall-clock reads.
+
+import type { BacktestCase } from "./backtest-corpus.js";
+import { splitBacktestCorpus } from "./backtest-split.js";
+
+/**
+ * Slice a corpus by repo: the repo is everything before `targetKey`'s LAST `#` (an issue/PR key like
+ * `owner/repo#123` — the last-`#` parse keeps a `#` inside an owner/repo name from truncating the repo).
+ * A key with no `#`, or nothing before it, is dropped — never guessed into a slice. Deterministic:
+ * insertion order follows each repo's first appearance, and case order is preserved within each slice.
+ */
+export function sliceCorpusByRepo(cases: readonly BacktestCase[]): Map<string, BacktestCase[]> {
+  const slices = new Map<string, BacktestCase[]>();
+  for (const backtestCase of cases) {
+    const hashIndex = backtestCase.targetKey.lastIndexOf("#");
+    if (hashIndex <= 0) continue; // unparseable target key — drop, never guess
+    const repoFullName = backtestCase.targetKey.slice(0, hashIndex);
+    let slice = slices.get(repoFullName);
+    if (slice === undefined) {
+      slice = [];
+      slices.set(repoFullName, slice);
+    }
+    slice.push(backtestCase);
+  }
+  return slices;
+}
+
+/** Aggregate-only labeled-density stats for one repo's slice. `eligible` applies the SAME seeded-split +
+ *  sample-minimum discipline the knob evaluators use: a repo is evaluable only if ITS OWN slice still clears
+ *  both floors after splitting. */
+export type RepoCorpusDensity = {
+  cases: number;
+  confirmed: number;
+  reversed: number;
+  eligible: boolean;
+};
+
+/**
+ * Compute per-repo labeled density over `cases`, keyed by repo (via {@link sliceCorpusByRepo}, so
+ * unparseable keys are dropped here too). Each repo's `eligible` flag re-runs the deterministic split on
+ * that repo's OWN slice with the caller's seed/fraction and requires both the visible and held-out sides to
+ * clear their floors — the exact never-on-noise bar `evaluateKnobLoosening`/`evaluateKnobDrift` apply to the
+ * global corpus. Pure and deterministic: same corpus + parameters ⇒ same map, in first-appearance order.
+ */
+export function computeRepoCorpusDensity(
+  cases: readonly BacktestCase[],
+  minVisible: number,
+  minHeldOut: number,
+  heldOutFraction: number,
+  splitSeed: string,
+): Map<string, RepoCorpusDensity> {
+  const densities = new Map<string, RepoCorpusDensity>();
+  for (const [repoFullName, slice] of sliceCorpusByRepo(cases)) {
+    let confirmed = 0;
+    for (const backtestCase of slice) {
+      if (backtestCase.label === "confirmed") confirmed += 1;
+    }
+    const { visible, heldOut } = splitBacktestCorpus(slice, heldOutFraction, splitSeed);
+    densities.set(repoFullName, {
+      cases: slice.length,
+      confirmed,
+      reversed: slice.length - confirmed,
+      eligible: visible.length >= minVisible && heldOut.length >= minHeldOut,
+    });
+  }
+  return densities;
+}

packages/loopover-engine/src/index.ts

@@ -175,6 +175,7 @@ export * from "./calibration/backtest-track-record.js";
 // same way scripts/backtest-corpus-export.ts already imports BacktestCase.
 export * from "./calibration/backtest-split.js";
 export * from "./calibration/backtest-threshold.js";
+export * from "./calibration/repo-corpus.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/repo-corpus.test.ts

@@ -0,0 +1,49 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { computeRepoCorpusDensity, sliceCorpusByRepo, splitBacktestCorpus, type BacktestCase } from "../dist/index.js";
+
+function corpusCase(targetKey: string, label: BacktestCase["label"] = "confirmed"): BacktestCase {
+  return {
+    ruleId: "missing_linked_issue",
+    targetKey,
+    outcome: "block",
+    label,
+    firedAt: "2026-07-01T00:00:00.000Z",
+    decidedAt: "2026-07-02T00:00:00.000Z",
+  };
+}
+
+test("barrel: the public entrypoint re-exports the per-repo corpus primitives (#8215)", () => {
+  assert.equal(typeof sliceCorpusByRepo, "function");
+  assert.equal(typeof computeRepoCorpusDensity, "function");
+});
+
+test("sliceCorpusByRepo: slices by the last '#', preserves order, drops unparseable keys", () => {
+  const slices = sliceCorpusByRepo([
+    corpusCase("acme/widgets#1"),
+    corpusCase("beta/tools#3"),
+    corpusCase("acme/widgets#2"),
+    corpusCase("no-hash"),
+    corpusCase("#5"),
+  ]);
+  assert.deepEqual([...slices.keys()], ["acme/widgets", "beta/tools"]);
+  assert.deepEqual(
+    slices.get("acme/widgets")!.map((c) => c.targetKey),
+    ["acme/widgets#1", "acme/widgets#2"],
+  );
+});
+
+test("computeRepoCorpusDensity: per-repo aggregates with the split-floor eligibility bar", () => {
+  const dense = Array.from({ length: 40 }, (_, i) => corpusCase(`acme/dense#${i + 1}`, i % 2 === 0 ? "reversed" : "confirmed"));
+  const sparse = [corpusCase("acme/sparse#1"), corpusCase("acme/sparse#2", "reversed")];
+  const densities = computeRepoCorpusDensity([...dense, ...sparse], 5, 2, 0.25, "repo-density-test-v1");
+
+  const denseDensity = densities.get("acme/dense")!;
+  assert.equal(denseDensity.cases, 40);
+  assert.equal(denseDensity.confirmed + denseDensity.reversed, 40);
+  const denseSplit = splitBacktestCorpus(dense, 0.25, "repo-density-test-v1");
+  assert.equal(denseDensity.eligible, denseSplit.visible.length >= 5 && denseSplit.heldOut.length >= 2);
+
+  assert.deepEqual(densities.get("acme/sparse"), { cases: 2, confirmed: 1, reversed: 1, eligible: false });
+});

test/unit/repo-corpus-engine.test.ts

@@ -0,0 +1,102 @@
+import { describe, expect, it } from "vitest";
+
+// Import the engine SOURCE directly (not the built dist) -- coverage.include lists
+// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in
+// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace
+// suite). Same pattern as backtest-corpus-engine.test.ts / backtest-split-engine.test.ts.
+import { computeRepoCorpusDensity, sliceCorpusByRepo } from "../../packages/loopover-engine/src/calibration/repo-corpus";
+import { splitBacktestCorpus } from "../../packages/loopover-engine/src/calibration/backtest-split";
+import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus";
+
+function corpusCase(targetKey: string, label: BacktestCase["label"] = "confirmed"): BacktestCase {
+  return {
+    ruleId: "missing_linked_issue",
+    targetKey,
+    outcome: "block",
+    label,
+    firedAt: "2026-07-01T00:00:00.000Z",
+    decidedAt: "2026-07-02T00:00:00.000Z",
+  };
+}
+
+describe("sliceCorpusByRepo (#8215)", () => {
+  it("slices a multi-repo corpus by the last '#', preserving per-slice case order and first-appearance repo order", () => {
+    const cases = [
+      corpusCase("acme/widgets#1"),
+      corpusCase("acme/gadgets#7"),
+      corpusCase("acme/widgets#2", "reversed"),
+      corpusCase("beta/tools#3"),
+      corpusCase("acme/widgets#3"),
+    ];
+    const slices = sliceCorpusByRepo(cases);
+    expect([...slices.keys()]).toEqual(["acme/widgets", "acme/gadgets", "beta/tools"]);
+    expect(slices.get("acme/widgets")!.map((c) => c.targetKey)).toEqual(["acme/widgets#1", "acme/widgets#2", "acme/widgets#3"]);
+    expect(slices.get("acme/gadgets")).toHaveLength(1);
+    expect(slices.get("beta/tools")).toHaveLength(1);
+  });
+
+  it("keys a repo containing an embedded '#' by the LAST '#', keeping the full repo name intact", () => {
+    const slices = sliceCorpusByRepo([corpusCase("acme/wid#gets#12")]);
+    expect([...slices.keys()]).toEqual(["acme/wid#gets"]);
+  });
+
+  it("drops unparseable target keys — no '#', or nothing before it — instead of guessing a slice", () => {
+    const good = corpusCase("acme/widgets#1");
+    const slices = sliceCorpusByRepo([corpusCase("no-hash-at-all"), corpusCase("#5"), good]);
+    expect([...slices.keys()]).toEqual(["acme/widgets"]);
+    expect(slices.get("acme/widgets")).toEqual([good]);
+  });
+
+  it("returns an empty map for an empty corpus", () => {
+    expect(sliceCorpusByRepo([])).toEqual(new Map());
+  });
+});
+
+describe("computeRepoCorpusDensity (#8215)", () => {
+  const SEED = "repo-density-test-v1";
+  const FRACTION = 0.25;
+
+  // Probe the real splitter per candidate repo slice so eligibility fixtures are membership-exact rather
+  // than luck-based — the same probing technique test/unit/loosening-knobs.test.ts uses.
+  function denseRepoCases(repoFullName: string, count: number): BacktestCase[] {
+    return Array.from({ length: count }, (_, i) => corpusCase(`${repoFullName}#${i + 1}`, i % 3 === 0 ? "reversed" : "confirmed"));
+  }
+
+  it("reports aggregate-only per-repo counts and applies the split-based eligibility floors to each repo's OWN slice", () => {
+    const dense = denseRepoCases("acme/dense", 40);
+    const sparse = denseRepoCases("acme/sparse", 3);
+    const densities = computeRepoCorpusDensity([...dense, ...sparse], 5, 2, FRACTION, SEED);
+
+    const denseSplit = splitBacktestCorpus(dense, FRACTION, SEED);
+    expect(densities.get("acme/dense")).toEqual({
+      cases: 40,
+      confirmed: dense.filter((c) => c.label === "confirmed").length,
+      reversed: dense.filter((c) => c.label === "reversed").length,
+      eligible: denseSplit.visible.length >= 5 && denseSplit.heldOut.length >= 2,
+    });
+    expect(densities.get("acme/dense")!.eligible).toBe(true); // 40 cases at 0.25 clears 5/2 for this seed
+    expect(densities.get("acme/sparse")!.eligible).toBe(false); // 3 cases can never clear the floors
+    expect(densities.get("acme/sparse")!.cases).toBe(3);
+  });
+
+  it("marks a repo ineligible when its held-out side alone misses the floor, even with a large visible side", () => {
+    const cases = denseRepoCases("acme/lopsided", 40);
+    const { heldOut } = splitBacktestCorpus(cases, FRACTION, SEED);
+    const densities = computeRepoCorpusDensity(cases, 1, heldOut.length + 1, FRACTION, SEED);
+    expect(densities.get("acme/lopsided")!.eligible).toBe(false);
+  });
+
+  it("is deterministic per slice: identical corpus + parameters yield the identical map", () => {
+    const cases = [...denseRepoCases("acme/a", 12), ...denseRepoCases("acme/b", 8)];
+    expect(computeRepoCorpusDensity(cases, 3, 1, FRACTION, SEED)).toEqual(computeRepoCorpusDensity(cases, 3, 1, FRACTION, SEED));
+  });
+
+  it("never leaks target keys or metadata — aggregate numbers and the eligibility flag only", () => {
+    const densities = computeRepoCorpusDensity(denseRepoCases("acme/private", 10), 2, 1, FRACTION, SEED);
+    for (const [repo, density] of densities) {
+      expect(repo).toBe("acme/private");
+      expect(Object.keys(density).sort()).toEqual(["cases", "confirmed", "eligible", "reversed"]);
+      expect(JSON.stringify(density)).not.toContain("#");
+    }
+  });
+});