feat(engine): pure BacktestCase corpus builder from fired/override event pairs

packages/loopover-engine/src/calibration/backtest-corpus.ts

@@ -0,0 +1,96 @@
+// Labeled backtest corpus builder (#8083, part of the #8082 rule-precision backtest epic). Where
+// signal-tracking.ts's computeRulePrecision collapses a rule's fired/override history into ONE aggregate
+// precision number, a backtest needs each PAIRED case kept as an individual labeled record -- a concrete
+// "this rule fired against this target, and a human later said it was right/wrong" row, replayable against
+// a different candidate rule/classifier later.
+//
+// SELF-CONTAINED, STORAGE-AGNOSTIC: pure TypeScript, no IO, no DB, no env -- only the existing
+// RuleFiredEvent/HumanOverrideEvent types from signal-tracking.ts, mirroring that module's own
+// "pure calibration math here, storage at the host layer" posture (see its header comment).
+
+import type { HumanOverrideEvent, RuleFiredEvent } from "./signal-tracking.js";
+
+/** One labeled backtest case: a specific rule firing (`firedAt`, with the fired event's `outcome` and
+ *  optional `metadata`) joined to the human verdict that judged it (`label`, at `decidedAt`). `metadata`
+ *  is omitted entirely (not set to `undefined`) when the fired event carried none -- the same
+ *  optional-property discipline {@link RuleFiredEvent} itself uses. */
+export type BacktestCase = {
+  ruleId: string;
+  targetKey: string;
+  outcome: string;
+  label: "reversed" | "confirmed";
+  firedAt: string;
+  decidedAt: string;
+  metadata?: Record<string, unknown>;
+};
+
+/** True for an override event that targets the same rule as `ruleId` -- mirrors signal-tracking.ts's own
+ *  private `overrideMatchesRule` helper (deliberately NOT exported from there; this issue is additive-only
+ *  in a new file, so the one-line filter is mirrored here instead). */
+function overrideMatchesRule(event: HumanOverrideEvent, ruleId: string): boolean {
+  return event.ruleId === ruleId;
+}
+
+/**
+ * Pick the override that judges `firedEvent`: the one whose `occurredAt` is closest in time strictly AFTER
+ * the fired event's own `occurredAt` (a verdict naturally follows the firing it judges); when none strictly
+ * follows it (e.g. clock skew, or a verdict recorded against an earlier fire of the same target), fall back
+ * to the most recent override by `occurredAt`. Ties keep the first candidate encountered, so the choice is
+ * deterministic for a fixed input order.
+ */
+function pairedOverrideFor(firedEvent: RuleFiredEvent, candidates: readonly HumanOverrideEvent[]): HumanOverrideEvent | null {
+  const firedMs = Date.parse(firedEvent.occurredAt);
+  let nearestAfter: HumanOverrideEvent | null = null;
+  let nearestAfterMs = Number.POSITIVE_INFINITY;
+  let mostRecent: HumanOverrideEvent | null = null;
+  let mostRecentMs = Number.NEGATIVE_INFINITY;
+  for (const candidate of candidates) {
+    const candidateMs = Date.parse(candidate.occurredAt);
+    if (candidateMs > firedMs && candidateMs - firedMs < nearestAfterMs) {
+      nearestAfter = candidate;
+      nearestAfterMs = candidateMs - firedMs;
+    }
+    if (candidateMs > mostRecentMs) {
+      mostRecent = candidate;
+      mostRecentMs = candidateMs;
+    }
+  }
+  return nearestAfter ?? mostRecent;
+}
+
+/**
+ * Build the labeled backtest corpus for `ruleId` from its fired + override events. A fired event and an
+ * override pair into one {@link BacktestCase} when both carry the function's `ruleId` AND the same
+ * `targetKey`. A fired event with NO matching override is excluded from the result (not included as an
+ * unlabeled case) -- mirrors {@link computeRulePrecision}'s "only the decided ones count" discipline.
+ * When a target has been re-fired and re-judged more than once, each fired event pairs with the override
+ * whose `occurredAt` is closest in time strictly after that specific firing; if none strictly follows it,
+ * the most recent override by `occurredAt` stands in (see {@link pairedOverrideFor}) -- one case per fired
+ * event, never duplicates. Only events whose `ruleId` equals the argument are considered; like
+ * computeRulePrecision, a caller MAY pass mixed-rule event lists without filtering first.
+ */
+export function buildBacktestCorpus(
+  ruleId: string,
+  fired: readonly RuleFiredEvent[],
+  overrides: readonly HumanOverrideEvent[],
+): BacktestCase[] {
+  const cases: BacktestCase[] = [];
+  for (const firedEvent of fired) {
+    if (firedEvent.ruleId !== ruleId) continue;
+    const candidates = overrides.filter(
+      (event) => overrideMatchesRule(event, ruleId) && event.targetKey === firedEvent.targetKey,
+    );
+    const override = pairedOverrideFor(firedEvent, candidates);
+    if (!override) continue;
+    cases.push({
+      ruleId,
+      targetKey: firedEvent.targetKey,
+      outcome: firedEvent.outcome,
+      label: override.verdict,
+      firedAt: firedEvent.occurredAt,
+      decidedAt: override.occurredAt,
+      ...(firedEvent.metadata !== undefined ? { metadata: firedEvent.metadata } : {}),
+    });
+  }
+  return cases;
+}

packages/loopover-engine/src/index.ts

@@ -163,6 +163,7 @@ export * from "./governor/kill-switch.js";
 export * from "./governor/action-mode.js";
 export * from "./governor/chokepoint.js";
 export * from "./calibration/signal-tracking.js";
+export * from "./calibration/backtest-corpus.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/backtest-corpus.test.ts

@@ -0,0 +1,108 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { buildBacktestCorpus, type BacktestCase, type HumanOverrideEvent, type RuleFiredEvent } from "../dist/index.js";
+
+function fired(ruleId: string, targetKey: string, overrides: Partial<RuleFiredEvent> = {}): RuleFiredEvent {
+  return { ruleId, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides };
+}
+
+function override(
+  ruleId: string,
+  targetKey: string,
+  verdict: HumanOverrideEvent["verdict"],
+  overrides: Partial<HumanOverrideEvent> = {},
+): HumanOverrideEvent {
+  return { ruleId, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides };
+}
+
+test("barrel: the public entrypoint re-exports the backtest-corpus builder (#8083)", () => {
+  assert.equal(typeof buildBacktestCorpus, "function");
+});
+
+test("buildBacktestCorpus: empty inputs produce an empty corpus", () => {
+  assert.deepEqual(buildBacktestCorpus("missing_linked_issue", [], []), []);
+});
+
+test("buildBacktestCorpus: a fired event with no matching override is excluded, not emitted unlabeled", () => {
+  const corpus = buildBacktestCorpus("missing_linked_issue", [fired("missing_linked_issue", "a#1")], []);
+  assert.deepEqual(corpus, []);
+});
+
+test("buildBacktestCorpus: a single fired+override pair produces one correctly-labeled case", () => {
+  const corpus = buildBacktestCorpus(
+    "missing_linked_issue",
+    [fired("missing_linked_issue", "a#1", { outcome: "closed", metadata: { pr: 1 } })],
+    [override("missing_linked_issue", "a#1", "reversed")],
+  );
+  const expected: BacktestCase[] = [
+    {
+      ruleId: "missing_linked_issue",
+      targetKey: "a#1",
+      outcome: "closed",
+      label: "reversed",
+      firedAt: "2026-07-22T00:00:00.000Z",
+      decidedAt: "2026-07-22T01:00:00.000Z",
+      metadata: { pr: 1 },
+    },
+  ];
+  assert.deepEqual(corpus, expected);
+});
+
+test("buildBacktestCorpus: omits the metadata key entirely when the fired event carries none", () => {
+  const [item] = buildBacktestCorpus(
+    "missing_linked_issue",
+    [fired("missing_linked_issue", "a#1")],
+    [override("missing_linked_issue", "a#1", "confirmed")],
+  );
+  assert.ok(item);
+  assert.equal(item.label, "confirmed");
+  assert.equal(Object.hasOwn(item, "metadata"), false);
+});
+
+test("buildBacktestCorpus: multiple overrides pair each firing with the nearest strictly-following verdict", () => {
+  const corpus = buildBacktestCorpus(
+    "rule",
+    [
+      fired("rule", "a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }),
+      fired("rule", "a#1", { occurredAt: "2026-07-22T04:00:00.000Z" }),
+    ],
+    [
+      override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }),
+      override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T06:00:00.000Z" }),
+    ],
+  );
+  assert.equal(corpus.length, 2);
+  assert.deepEqual(
+    corpus.map((item) => [item.firedAt, item.label, item.decidedAt]),
+    [
+      ["2026-07-22T00:00:00.000Z", "reversed", "2026-07-22T02:00:00.000Z"],
+      ["2026-07-22T04:00:00.000Z", "confirmed", "2026-07-22T06:00:00.000Z"],
+    ],
+  );
+});
+
+test("buildBacktestCorpus: with no strictly-following override, the most recent one stands in", () => {
+  const corpus = buildBacktestCorpus(
+    "rule",
+    [fired("rule", "a#1", { occurredAt: "2026-07-22T10:00:00.000Z" })],
+    [
+      override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+      override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T05:00:00.000Z" }),
+    ],
+  );
+  assert.equal(corpus.length, 1);
+  assert.equal(corpus[0]?.label, "confirmed");
+  assert.equal(corpus[0]?.decidedAt, "2026-07-22T05:00:00.000Z");
+});
+
+test("buildBacktestCorpus: fired and override events for a DIFFERENT ruleId are ignored entirely", () => {
+  const corpus = buildBacktestCorpus(
+    "rule",
+    [fired("other_rule", "a#1"), fired("rule", "a#2")],
+    [override("rule", "a#1", "confirmed"), override("other_rule", "a#2", "confirmed"), override("rule", "a#2", "reversed")],
+  );
+  assert.equal(corpus.length, 1);
+  assert.equal(corpus[0]?.targetKey, "a#2");
+  assert.equal(corpus[0]?.label, "reversed");
+});

test/unit/backtest-corpus.test.ts

@@ -0,0 +1,124 @@
+import { describe, expect, it } from "vitest";
+import {
+  buildBacktestCorpus,
+  type BacktestCase,
+} from "../../packages/loopover-engine/src/calibration/backtest-corpus";
+import type { HumanOverrideEvent, RuleFiredEvent } from "../../packages/loopover-engine/src/calibration/signal-tracking";
+
+function fired(ruleId: string, targetKey: string, overrides: Partial<RuleFiredEvent> = {}): RuleFiredEvent {
+  return { ruleId, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides };
+}
+
+function override(
+  ruleId: string,
+  targetKey: string,
+  verdict: HumanOverrideEvent["verdict"],
+  overrides: Partial<HumanOverrideEvent> = {},
+): HumanOverrideEvent {
+  return { ruleId, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides };
+}
+
+describe("buildBacktestCorpus (#8083) — pure labeled-corpus builder", () => {
+  it("produces an empty corpus from empty inputs", () => {
+    expect(buildBacktestCorpus("missing_linked_issue", [], [])).toEqual([]);
+  });
+
+  it("excludes a fired event with no matching override instead of emitting an unlabeled case", () => {
+    expect(buildBacktestCorpus("missing_linked_issue", [fired("missing_linked_issue", "a#1")], [])).toEqual([]);
+  });
+
+  it("pairs a single fired+override into one correctly-labeled case, carrying outcome and metadata", () => {
+    const corpus = buildBacktestCorpus(
+      "missing_linked_issue",
+      [fired("missing_linked_issue", "a#1", { outcome: "closed", metadata: { pr: 1 } })],
+      [override("missing_linked_issue", "a#1", "reversed")],
+    );
+    const expected: BacktestCase[] = [
+      {
+        ruleId: "missing_linked_issue",
+        targetKey: "a#1",
+        outcome: "closed",
+        label: "reversed",
+        firedAt: "2026-07-22T00:00:00.000Z",
+        decidedAt: "2026-07-22T01:00:00.000Z",
+        metadata: { pr: 1 },
+      },
+    ];
+    expect(corpus).toEqual(expected);
+  });
+
+  it("omits the metadata key entirely (not undefined) when the fired event has none", () => {
+    const [item] = buildBacktestCorpus(
+      "missing_linked_issue",
+      [fired("missing_linked_issue", "a#1")],
+      [override("missing_linked_issue", "a#1", "confirmed")],
+    );
+    expect(item?.label).toBe("confirmed");
+    expect(Object.hasOwn(item ?? {}, "metadata")).toBe(false);
+  });
+
+  it("pairs each firing with the NEAREST strictly-following override when a target was re-fired and re-judged", () => {
+    // Candidate scan order deliberately puts the farther-after verdict first, so the nearer one must
+    // replace it, and a third even-farther verdict must NOT displace the nearest already found.
+    const corpus = buildBacktestCorpus(
+      "rule",
+      [fired("rule", "a#1", { occurredAt: "2026-07-22T00:00:00.000Z" })],
+      [
+        override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T06:00:00.000Z" }),
+        override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T02:00:00.000Z" }),
+        override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T08:00:00.000Z" }),
+      ],
+    );
+    expect(corpus).toHaveLength(1);
+    expect(corpus[0]?.label).toBe("confirmed");
+    expect(corpus[0]?.decidedAt).toBe("2026-07-22T02:00:00.000Z");
+  });
+
+  it("re-fired targets each get their own case, never a duplicate for the same fired event", () => {
+    const corpus = buildBacktestCorpus(
+      "rule",
+      [
+        fired("rule", "a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }),
+        fired("rule", "a#1", { occurredAt: "2026-07-22T04:00:00.000Z" }),
+      ],
+      [
+        override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }),
+        override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T06:00:00.000Z" }),
+      ],
+    );
+    expect(corpus.map((item) => [item.firedAt, item.label])).toEqual([
+      ["2026-07-22T00:00:00.000Z", "reversed"],
+      ["2026-07-22T04:00:00.000Z", "confirmed"],
+    ]);
+  });
+
+  it("falls back to the most recent override when none strictly follows the firing", () => {
+    // Scan order is ascending here, so the later verdict must replace the earlier as most-recent.
+    const corpus = buildBacktestCorpus(
+      "rule",
+      [fired("rule", "a#1", { occurredAt: "2026-07-22T10:00:00.000Z" })],
+      [
+        override("rule", "a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
+        override("rule", "a#1", "confirmed", { occurredAt: "2026-07-22T05:00:00.000Z" }),
+      ],
+    );
+    expect(corpus).toHaveLength(1);
+    expect(corpus[0]?.label).toBe("confirmed");
+    expect(corpus[0]?.decidedAt).toBe("2026-07-22T05:00:00.000Z");
+  });
+
+  it("ignores fired and override events for a different ruleId, and overrides for a different target", () => {
+    const corpus = buildBacktestCorpus(
+      "rule",
+      [fired("other_rule", "a#1"), fired("rule", "a#2")],
+      [
+        override("rule", "a#1", "confirmed"), // right rule, wrong target -- never pairs with a#2
+        override("other_rule", "a#2", "confirmed"), // wrong rule, right target -- filtered out
+        override("rule", "a#2", "reversed"),
+      ],
+    );
+    expect(corpus).toHaveLength(1);
+    expect(corpus[0]?.targetKey).toBe("a#2");
+    expect(corpus[0]?.label).toBe("reversed");
+  });
+});