From 264b82d2d9ef9f870607537aafe2bfdaf28eb2f9 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 7 Aug 2026 15:18:49 -0700 Subject: [PATCH 1/4] fix(drift-sync): record provider deprecations instead of paging a human A family a healthy live /models listing no longer contains is a fact the provider already published, not a decision. drift-sync now records it in a new deprecatedFamilies ledger in model-registry.ts and leaves includeFamilies (and therefore the mock) untouched, instead of writing a needs-human note that reddens the daily cron and emails the owner every morning. --- scripts/drift-sync.ts | 332 ++++++++------------- src/__tests__/drift/logic-pin.test.ts | 14 + src/__tests__/drift/model-registry.test.ts | 98 ++++++ src/__tests__/drift/model-registry.ts | 69 +++++ 4 files changed, 307 insertions(+), 206 deletions(-) diff --git a/scripts/drift-sync.ts b/scripts/drift-sync.ts index faba061c..3808e06d 100644 --- a/scripts/drift-sync.ts +++ b/scripts/drift-sync.ts @@ -39,6 +39,7 @@ import { normalizeModelFamily } from "../src/__tests__/drift/model-family.js"; import { includeFamilies, isClassifiedFamily, + isRecordedDeprecation, NON_MODEL_TOKENS, } from "../src/__tests__/drift/model-registry.js"; import { @@ -338,14 +339,19 @@ export function addChangelogEntry(report: DriftReport, version: string): void { // // - DEPRECATION (classified − live, via a mirror of C4's // `detectDeprecatedFamilies`): a family aimock mocks that a healthy live -// listing no longer contains. Either way the registry is NOT touched — both -// legs route to a family-keyed dedup note file under `drift-proposals/`, -// because `includeFamilies`'s membership is checksum-pinned and re-pinning -// it is a reviewed human decision (see the removal probe in -// `runDriftSyncCore` for the full argument): -// * zero-reference (nothing in aimock's own source still names it) → a -// note PROPOSING the removal, naming the exact two-file edit to make. -// * still-referenced → a note recording that a human must decide at all. +// listing no longer contains. A provider-confirmed deprecation is a FACT, +// not a decision — the provider's own listing already says the family is +// gone — so it NEVER routes to a human. It is RECORDED, mechanically, as a +// new literal in `deprecatedFamilies[provider]` (model-registry.ts), +// comment-marked with the date and whether aimock's own source still +// references it. The mock is left FUNCTIONING: `includeFamilies` is not +// touched, so every fixture and builder for that family keeps serving, and +// `includeFamilies`'s checksum pin stays green. Recording is what stops the +// same deprecation being re-derived as novel drift every morning. +// Dropping a retired family from aimock altogether stays a human's job (it +// needs the `logic-pin.test.ts` re-pin the sync's own changed-file +// allowlist forbids it from making) — but it is optional cleanup, not an +// alert, and nothing is broken while it is undone. // - ADDITION (a genuinely new, UNCLASSIFIED family — matches no include, // exclude, `-preview`, or Gemma rule): NEVER auto-classified. Routed to a // human via the same dedup note-file mechanism. Only once a human edits @@ -399,6 +405,7 @@ export function detectDeprecatedFamiliesForSync( provider: Provider, opts: { isReferenced?: (family: string, provider: Provider) => boolean; + isRecorded?: (family: string, provider: Provider) => boolean; minListingSize?: number; } = {}, ): DeprecationCheckResult { @@ -423,9 +430,17 @@ export function detectDeprecatedFamiliesForSync( // checked BEFORE `isReferenced`: a forward-looking family legitimately has no // source reference either (aimock hasn't built its fixture yet), so relying // on "still referenced" alone can't distinguish it from a genuine retirement. + // Drop retirements already RECORDED in `deprecatedFamilies` (model-registry.ts). + // The provider's listing is not going to start containing them again, so + // re-deriving them is the same news every morning forever — the ledger exists + // precisely so the second sighting is silent. Same shape, and the same + // reasoning, as the forward-looking filter above; the difference is only which + // direction the family is missing IN. + const isRecorded = opts.isRecorded ?? isRecordedDeprecation; const missing = [...classified] .filter((family) => !liveFamilies.has(family)) .filter((family) => !isForwardLookingFamily(family, provider)) + .filter((family) => !isRecorded(family, provider)) .sort(); const isReferenced = opts.isReferenced ?? isFamilyStillReferenced; @@ -458,11 +473,13 @@ export function unclassifiedFamiliesForSync(modelIds: string[], provider: Provid /** Must match `scripts/drift-sync-check.ts`'s `ALLOWED_PREFIXES`. */ export const DRIFT_PROPOSALS_DIR = "drift-proposals"; -export type ProposalKind = - | "new-family" - | "still-referenced-deprecation" - | "zero-reference-deprecation" - | "registry-structural-mismatch"; +/** + * The two things drift-sync genuinely cannot decide alone. Deprecations are + * deliberately NOT here: a provider-confirmed retirement is a fact drift-sync + * records mechanically (see `deprecatedFamilies` in model-registry.ts), so it + * never produces a note and never pages anyone. + */ +export type ProposalKind = "new-family" | "registry-structural-mismatch"; export type ProposalDecision = "pending" | "include"; /** Family-keyed dedup path — re-firing the same alert always resolves to the SAME path. */ @@ -472,14 +489,7 @@ export function proposalNoteRelPath( kind: ProposalKind, ): string { const slug = family.replace(/[^a-z0-9.-]+/gi, "-"); - const kindSlug = - kind === "new-family" - ? "new-family" - : kind === "registry-structural-mismatch" - ? "structural-mismatch" - : kind === "zero-reference-deprecation" - ? "deprecated-unreferenced" - : "deprecated-referenced"; + const kindSlug = kind === "new-family" ? "new-family" : "structural-mismatch"; return `${DRIFT_PROPOSALS_DIR}/${provider}-${slug}-${kindSlug}.md`; } @@ -499,11 +509,7 @@ export function renderProposalNote( const title = kind === "new-family" ? "New / unclassified model family" - : kind === "registry-structural-mismatch" - ? "Registry structural mismatch — mechanical edit could not be applied" - : kind === "zero-reference-deprecation" - ? "Deprecated zero-reference model family — removal PROPOSED, not applied" - : "Deprecated-but-still-referenced model family"; + : "Registry structural mismatch — mechanical edit could not be applied"; const lines = [ `# ${title}: ${family}`, "", @@ -525,40 +531,23 @@ export function renderProposalNote( "", ); } - if (kind === "zero-reference-deprecation") { - lines.push( - "## How to apply", - "", - `1. Delete the \`"${family}"\` entry from \`includeFamilies.${provider}\` in`, - ` \`${MODEL_REGISTRY_REL_PATH}\`.`, - `2. Re-pin \`DATA_FROZEN["includeFamilies.${provider}"]\` in`, - " `src/__tests__/drift/logic-pin.test.ts` with the new membership checksum.", - "3. Delete this note file.", - "", - "All three belong in ONE reviewed commit: step 1 without step 2 leaves the", - "membership pin red, and step 2 without step 1 is a silent canary-silencing edit.", - "That deliberate, reviewed re-pin is a decision the pin reserves for a human, which", - "is exactly why drift-sync proposes this removal instead of applying it.", - "", - ); - } return lines.join("\n"); } // --------------------------------------------------------------------------- // Mechanical registry edits — AST-LOCATED (via the real TypeScript parser, not // a hand-rolled regex/lexer scan) then applied as a single-line text splice. -// The parser is used only to unambiguously find the exact line of the exact -// string-literal element inside `includeFamilies[provider]` / -// `excludeFamilies[provider]`'s array literal (or the array's closing-bracket -// line, for an insert) — the mutation itself is a trivial whole-line -// replace/insert, never a partial-token or multi-line reformat, so it cannot -// silently mangle an adjacent grouping comment or a sibling entry. +// The parser is used only to unambiguously find the array literal inside +// `includeFamilies[provider]` / `excludeFamilies[provider]` / +// `deprecatedFamilies[provider]` and the exact line of its closing bracket — +// the mutation itself is a trivial whole-line insert, never a partial-token or +// multi-line reformat, so it cannot silently mangle an adjacent grouping +// comment or a sibling entry. // --------------------------------------------------------------------------- export const MODEL_REGISTRY_REL_PATH = "src/__tests__/drift/model-registry.ts"; -type RegistrySetName = "includeFamilies" | "excludeFamilies"; +type RegistrySetName = "includeFamilies" | "excludeFamilies" | "deprecatedFamilies"; interface FamilySetLocation { /** family literal text -> 0-based source line index of that literal's own line. */ @@ -605,7 +594,7 @@ function locateFamilySetArray( if (!target) return null; const elementLines = new Map(); - let elementIndent = " "; + let elementIndent: string | null = null; for (const el of target.elements) { if (ts.isStringLiteral(el)) { const { line, character } = sf.getLineAndCharacterOfPosition(el.getStart(sf)); @@ -614,6 +603,17 @@ function locateFamilySetArray( } } const { line: arrayEndLine } = sf.getLineAndCharacterOfPosition(target.getEnd()); + if (elementIndent === null) { + // An array with no string element yet — `deprecatedFamilies`'s three + // comment-seeded arrays on the day the ledger is empty. There is no sibling + // entry to copy an indent from, so derive it from the array's own opening + // line (` : familySet("", [`) plus one 2-space level. + // Cosmetic only: the repo's prettier pre-commit hook normalizes it either + // way, and the AST locator does not care. + const { line: arrayStartLine } = sf.getLineAndCharacterOfPosition(target.getStart(sf)); + const openIndent = sourceText.split("\n")[arrayStartLine]?.match(/^(\s*)/)?.[1] ?? ""; + elementIndent = `${openIndent} `; + } return { elementLines, arrayEndLine, elementIndent }; } @@ -624,49 +624,13 @@ export interface RegistryEditResult { /** * True when the AST locator could NOT find the target array literal in * `model-registry.ts` (structural mismatch). This is distinct from a benign - * no-op (family already-absent for a remove / already-present for an add): - * a locator miss means a real add/remove could not be applied and MUST be - * routed to a human — never collapsed into a silent, clean no-op (G#1). + * no-op (the family is already present, so there is nothing to add): a + * locator miss means a real edit could not be applied and MUST be routed to + * a human — never collapsed into a silent, clean no-op (G#1). */ locatorMiss?: boolean; } -/** Comment-marked removal of `family` from `exportName[provider]`. Never touches any other line. */ -export function removeFamilyLiteralInSource( - sourceText: string, - exportName: RegistrySetName, - provider: Provider, - family: string, - reasonComment: string, -): RegistryEditResult { - const loc = locateFamilySetArray(sourceText, exportName, provider); - if (!loc) { - return { - changed: false, - text: sourceText, - detail: `could not locate ${exportName}.${provider} array in model-registry.ts (structural mismatch — routing to human)`, - locatorMiss: true, - }; - } - const lineIdx = loc.elementLines.get(family); - if (lineIdx === undefined) { - return { - changed: false, - text: sourceText, - detail: `"${family}" is not present in ${exportName}.${provider} — nothing to remove`, - }; - } - const lines = sourceText.split("\n"); - const indentMatch = lines[lineIdx].match(/^(\s*)/); - const indent = indentMatch ? indentMatch[1] : loc.elementIndent; - lines[lineIdx] = `${indent}// ${reasonComment}`; - return { - changed: true, - text: lines.join("\n"), - detail: `removed "${family}" from ${exportName}.${provider} (comment-marked)`, - }; -} - /** Mechanical, comment-marked addition of `family` to `exportName[provider]`. */ export function addFamilyLiteralInSource( sourceText: string, @@ -745,11 +709,9 @@ export interface SyncCoreDeps { } export type FamilyAction = - | "removed" + | "deprecation-recorded" | "added" | "needs-human-new-family" - | "needs-human-still-referenced" - | "needs-human-zero-reference" | "needs-human-structural-mismatch" | "no-op"; @@ -820,125 +782,74 @@ export function runDriftSyncCore( skipped.push({ provider: input.provider, reason: dep.reason }); } else { for (const cand of dep.candidates) { - if (!cand.stillReferenced) { - // The removal is computed but NOT persisted — a PROBE, deliberately. - // - // `includeFamilies[provider]`'s membership is checksum-pinned in - // `logic-pin.test.ts` (`DATA_FROZEN`), and gate-2 re-runs that whole - // file over the edited tree. So a persisted mechanical removal always - // reds its own gate: `pin-check-failed` -> `revertFiles([...touched])` - // -> the registry edit AND every needs-human note written that run are - // wiped -> `reason=gate-failed`, no PR of any class. OBSERVED against - // this repo's own frozen healthy anthropic wave. drift-sync cannot - // repair that itself either: `drift-sync-check`'s changed-file - // allowlist is `model-registry.ts` + `drift-proposals/` ONLY, so it is - // forbidden from touching the pin — and the pin's own message reserves - // the re-pin for "a deliberate, reviewed" human decision. A run that - // silently destroyed its own notes to attempt an edit that can never be - // kept is strictly worse than not attempting it. - // - // So a zero-reference deprecation routes to a note like every other - // decision drift-sync is not authorised to take alone. The registry is - // never mutated, the notes always survive, and the note spells out the - // two-file removal a human applies in one reviewed commit. - // - // The probe is still RUN because its verdict is load-bearing: it is - // what distinguishes "the literal is there and a removal is well-formed" - // from `locatorMiss` (the registry's structure moved — a real fault with - // its own route) and from a clean no-op. - const probe = removeFamilyLiteralInSource( - registrySource, - "includeFamilies", - cand.provider, - cand.family, - `REMOVED ${stamp} (drift-sync): "${cand.family}" no longer in live /models, zero-reference`, - ); - if (probe.changed) { - const zrPath = proposalNoteRelPath( - cand.provider, - cand.family, - "zero-reference-deprecation", - ); - ensureProposalNote( - deps, - zrPath, - () => - renderProposalNote( - cand.provider, - cand.family, - "zero-reference-deprecation", - `"${cand.family}" no longer appears in the live /models listing and nothing in ` + - `aimock's own source still references it, so removing it from ` + - `includeFamilies.${cand.provider} is mechanically safe. drift-sync does not ` + - `apply it: that set's membership is checksum-pinned, and re-pinning is a ` + - `reviewed human decision the sync's own changed-file allowlist forbids it ` + - `from making.`, - stamp, - ), - touchedFiles, - ); - outcomes.push({ - provider: cand.provider, - family: cand.family, - action: "needs-human-zero-reference", - detail: `"${cand.family}" is deprecated and zero-reference — removal proposed to a human (${zrPath})`, - }); - } else if (probe.locatorMiss) { - // G#1: the AST locator could not find includeFamilies[provider] in - // model-registry.ts. A real deprecation could not be applied — this - // must route to a human, NEVER collapse into a silent clean no-op. - const smPath = proposalNoteRelPath( - cand.provider, - cand.family, - "registry-structural-mismatch", - ); - ensureProposalNote( - deps, - smPath, - () => - renderProposalNote( - cand.provider, - cand.family, - "registry-structural-mismatch", - `A zero-reference deprecation was detected for "${cand.family}" but drift-sync ` + - `could not locate the includeFamilies.${cand.provider} array literal in ` + - `${MODEL_REGISTRY_REL_PATH} — the registry's structure changed. A human must ` + - `apply the removal (or fix the locator).`, - stamp, - ), - touchedFiles, - ); - outcomes.push({ - provider: cand.provider, - family: cand.family, - action: "needs-human-structural-mismatch", - detail: `${probe.detail} (${smPath})`, - }); - } else { - outcomes.push({ - provider: cand.provider, - family: cand.family, - action: "no-op", - detail: probe.detail, - }); - } - } else { - const notePath = proposalNoteRelPath( + // A PROVIDER-CONFIRMED DEPRECATION IS A FACT, NOT A DECISION. + // + // The provider's own /models listing already says the family is gone. + // There is nothing here for a human to weigh, so this must never page + // one — it used to route BOTH legs (zero-reference and still-referenced) + // to a needs-human note, which made an unattended cron go red and email + // the repo owner every morning to "decide" ten retirements Anthropic had + // already announced by deleting them from its catalog. + // + // The mechanical action is to RECORD the retirement in + // `deprecatedFamilies[provider]`, not to act on it: + // + // * The mock KEEPS SERVING. `includeFamilies` is untouched, so every + // builder and fixture for the family still answers. Users pin + // retired model ids in their own test suites for years; the upstream + // catalog shrinking is not a reason to break them, and a silent + // removal would. It also keeps `includeFamilies`'s membership + // checksum pin green, so the edit survives gate-2 — a persisted + // REMOVAL never could (the pin reds, `revertFiles` wipes the run, no + // PR of any class; observed). + // * The recording is what makes it stop. `detectDeprecatedFamiliesForSync` + // filters recorded families out of its candidate set, so tomorrow's + // run is quiet instead of re-deriving the same list forever. + // + // `stillReferenced` no longer chooses a ROUTE — both legs record — it + // only annotates the recorded line, because "nothing references this any + // more" is the one fact that tells a human the optional cleanup (drop it + // from `includeFamilies` + re-pin, in one reviewed commit) is safe. + const referenceNote = cand.stillReferenced + ? "still referenced in aimock source — mock retained" + : "no remaining aimock reference — droppable from includeFamilies in a reviewed re-pin"; + const edit = addFamilyLiteralInSource( + registrySource, + "deprecatedFamilies", + cand.provider, + cand.family, + `DEPRECATED ${stamp} (drift-sync): absent from live /models; ${referenceNote}`, + ); + if (edit.changed) { + registrySource = edit.text; + registryChanged = true; + outcomes.push({ + provider: cand.provider, + family: cand.family, + action: "deprecation-recorded", + detail: `"${cand.family}" is absent from the live /models listing — recorded in deprecatedFamilies.${cand.provider} (${referenceNote})`, + }); + } else if (edit.locatorMiss) { + // G#1: the AST locator could not find deprecatedFamilies[provider] in + // model-registry.ts. A real deprecation could not be recorded — this + // must route to a human, NEVER collapse into a silent clean no-op. + const smPath = proposalNoteRelPath( cand.provider, cand.family, - "still-referenced-deprecation", + "registry-structural-mismatch", ); ensureProposalNote( deps, - notePath, + smPath, () => renderProposalNote( cand.provider, cand.family, - "still-referenced-deprecation", - "This family no longer appears in the live /models listing, but aimock's " + - "own source still references it (builders, DEFAULT_MODELS, or fixtures). " + - "drift-sync never silently removes a still-referenced family.", + "registry-structural-mismatch", + `A deprecation was detected for "${cand.family}" but drift-sync could not locate ` + + `the deprecatedFamilies.${cand.provider} array literal in ` + + `${MODEL_REGISTRY_REL_PATH} — the registry's structure changed. A human must ` + + `record the deprecation (or fix the locator).`, stamp, ), touchedFiles, @@ -946,8 +857,15 @@ export function runDriftSyncCore( outcomes.push({ provider: cand.provider, family: cand.family, - action: "needs-human-still-referenced", - detail: `"${cand.family}" is deprecated but still referenced in source — routed to human (${notePath})`, + action: "needs-human-structural-mismatch", + detail: `${edit.detail} (${smPath})`, + }); + } else { + outcomes.push({ + provider: cand.provider, + family: cand.family, + action: "no-op", + detail: edit.detail, }); } } @@ -1367,7 +1285,9 @@ function commitSyncChanges(outcome: SyncCoreOutcome): boolean { (f) => f === MODEL_REGISTRY_REL_PATH || f.startsWith(`${DRIFT_PROPOSALS_DIR}/`), ); if (changed.length === 0) return false; - const applied = outcome.outcomes.filter((o) => o.action === "removed" || o.action === "added"); + const applied = outcome.outcomes.filter( + (o) => o.action === "added" || o.action === "deprecation-recorded", + ); const summary = applied.length > 0 ? applied.map((o) => `${o.action} ${o.provider}/${o.family}`).join(", ") diff --git a/src/__tests__/drift/logic-pin.test.ts b/src/__tests__/drift/logic-pin.test.ts index 7b7b60c0..ea6b99e1 100644 --- a/src/__tests__/drift/logic-pin.test.ts +++ b/src/__tests__/drift/logic-pin.test.ts @@ -35,6 +35,20 @@ * * Keep both lists exact. A guard whose own inventory is wrong is a false * record: it invites the reader to assume a surface is covered when it is not. + * + * DELIBERATELY NOT FROZEN — `model-registry.ts`'s `deprecatedFamilies`. + * DO NOT ADD IT. That set is the ledger `scripts/drift-sync.ts` APPENDS to, + * unattended, every time a provider retires a family aimock mocks, and + * `drift-sync-check`'s gate-2 re-runs this very file over the edited tree — so + * a pin on it would red on the sync's own append, revert it, and leave the + * deprecation to be re-derived and re-reverted every morning forever. The pin + * and the ledger cannot both exist. Nothing is lost by leaving it out: a pin + * defends against a one-line SILENCING edit, and an entry in `deprecatedFamilies` + * silences no alert a human ever sees (it is not consulted by + * `isClassifiedFamily`, so it cannot classify a live family or suppress a + * new-family alert — it only stops a deprecation already recorded from being + * re-recorded). Its invariants are asserted behaviourally in + * `model-registry.test.ts` instead. */ import { describe, it, expect } from "vitest"; import { readFileSync } from "node:fs"; diff --git a/src/__tests__/drift/model-registry.test.ts b/src/__tests__/drift/model-registry.test.ts index 53338cb3..93a72dd4 100644 --- a/src/__tests__/drift/model-registry.test.ts +++ b/src/__tests__/drift/model-registry.test.ts @@ -25,10 +25,15 @@ * relies on, with no live-key dependency. */ import { describe, it, expect } from "vitest"; +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { normalizeModelFamily } from "./model-family.js"; import { includeFamilies, excludeFamilies, + deprecatedFamilies, + isRecordedDeprecation, NON_MODEL_TOKENS, isClassifiedFamily, PREVIEW_FAMILY, @@ -60,6 +65,99 @@ describe("model-registry", () => { } }); + // ── the recorded-deprecation ledger ─────────────────────────────────────── + // + // `deprecatedFamilies` is the one registry set drift-sync WRITES unattended, + // and the one deliberately left out of `logic-pin.test.ts`'s membership + // freeze (pinning it would make every append revert itself at gate-2). These + // are the invariants that stand in for that pin. See the set's own doc. + + it("a recorded deprecation is always a family aimock actually mocks", () => { + for (const provider of ["openai", "anthropic", "gemini"] as const) { + const stray = [...deprecatedFamilies[provider]].filter( + (f) => !includeFamilies[provider].has(f), + ); + expect( + stray, + `deprecatedFamilies.${provider} records ${stray.join(", ")}, which ${ + stray.length === 1 ? "is" : "are" + } not in includeFamilies.${provider}. The ledger records the retirement of a family ` + + `aimock MOCKS; an entry with no include-side counterpart is either a typo or the ` + + `remains of a half-finished cleanup (dropping a family means deleting it from BOTH ` + + `sets in the same reviewed commit as the logic-pin re-pin).`, + ).toEqual([]); + } + }); + + it("recording a deprecation NEVER unclassifies the family — the mock keeps serving", () => { + for (const provider of ["openai", "anthropic", "gemini"] as const) { + for (const family of deprecatedFamilies[provider]) { + expect( + isClassifiedFamily(family, provider), + `${provider}/${family} is recorded as deprecated and stopped being classified. ` + + `Recording what the PROVIDER retired must never change what aimock serves.`, + ).toBe(true); + } + } + }); + + it("isRecordedDeprecation reads the ledger, and only the ledger", () => { + // Non-vacuous while the ledger is empty: it pins the predicate's two + // directions against a family that IS classified (so a body of `return + // isClassifiedFamily(...)`, or `return true`, reddens here) and one that is + // in neither set. + for (const provider of ["openai", "anthropic", "gemini"] as const) { + for (const family of deprecatedFamilies[provider]) { + expect(isRecordedDeprecation(family, provider)).toBe(true); + } + const notRecorded = [...includeFamilies[provider]].filter( + (f) => !deprecatedFamilies[provider].has(f), + ); + expect( + notRecorded.length, + `every ${provider} family is recorded as deprecated — this assertion has nothing left ` + + `to prove and must be rewritten`, + ).toBeGreaterThan(0); + for (const family of notRecorded) { + expect(isRecordedDeprecation(family, provider)).toBe(false); + } + expect(isRecordedDeprecation("zzz-not-a-real-family", provider)).toBe(false); + } + }); + + it("the deprecation ledger never reaches aimock's serving path", () => { + // THE ARCHITECTURAL BOUNDARY, as a test. `deprecatedFamilies` records what + // the PROVIDER retired; it must never become an input to what aimock + // SERVES, or recording a deprecation silently breaks every user whose suite + // still pins that model id. aimock's shipped product source is `src/` + // outside `src/__tests__/` — server.ts's DEFAULT_MODELS, the per-provider + // builders, the routers — so a reference to the ledger appearing anywhere + // in it is that boundary being crossed. + const srcRoot = fileURLToPath(new URL("../../", import.meta.url)); + const offenders: string[] = []; + const stack = [srcRoot]; + while (stack.length > 0) { + const dir = stack.pop()!; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + const rel = full.slice(srcRoot.length).replace(/^[/\\]+/, ""); + if (entry.isDirectory()) { + if (entry.name === "node_modules" || entry.name === "dist") continue; + if (rel === "__tests__") continue; + stack.push(full); + } else if (entry.isFile() && /\.tsx?$/.test(entry.name)) { + if (readFileSync(full, "utf8").includes("deprecatedFamilies")) offenders.push(rel); + } + } + } + expect( + offenders, + `aimock's product source now references deprecatedFamilies (${offenders.join(", ")}). ` + + `The ledger is a record of provider behaviour, not a switch on aimock's own behaviour ` + + `— a retired family must keep being mocked.`, + ).toEqual([]); + }); + it("seeds are idempotent under normalization (already family keys)", () => { for (const provider of ["openai", "anthropic", "gemini"] as const) { for (const family of includeFamilies[provider]) { diff --git a/src/__tests__/drift/model-registry.ts b/src/__tests__/drift/model-registry.ts index 233fe5eb..61e8898a 100644 --- a/src/__tests__/drift/model-registry.ts +++ b/src/__tests__/drift/model-registry.ts @@ -293,6 +293,75 @@ export const excludeFamilies: Record> = { ]), }; +/** + * Families a healthy live `/models` listing NO LONGER CONTAINS — the provider + * retired them. Appended MECHANICALLY by `scripts/drift-sync.ts` on the morning + * it first observes the absence, one entry per family, comment-marked with the + * date and whether aimock's own source still references it. + * + * WHY THIS EXISTS. A provider-confirmed deprecation is a FACT, not a decision: + * the provider's own listing already says the family is gone, so there is + * nothing for a human to decide and no reason to page one. But the detector is + * a pure `includeFamilies − live` diff, so without somewhere to WRITE the fact + * down it re-derives the same ten deprecations every single morning, forever. + * This set is that ledger, and `isRecordedDeprecation` is what takes a recorded + * family out of the candidate set (see `detectDeprecatedFamiliesForSync` in + * `scripts/drift-sync.ts` — the same shape as the `isForwardLookingFamily` + * filter beside it). + * + * RECORDING IS NOT REMOVING, DELIBERATELY. A family recorded here STAYS in + * `includeFamilies` and aimock KEEPS MOCKING IT. Users' tests pin retired model + * ids for years, and dropping a mocked family would break them; the upstream + * catalog shrinking is not a reason for aimock's to. So this set records what + * the PROVIDER did, never what aimock serves — nothing on the serving path may + * read it (asserted in `model-registry.test.ts`). + * + * IT IS NOT A CLASSIFICATION SET. `isClassifiedFamily` does not consult it, so + * an entry here can neither classify a family nor silence a new-family alert. + * Its only effect is to stop re-recording a deprecation already recorded. + * + * DELIBERATELY NOT MEMBERSHIP-PINNED in `logic-pin.test.ts`, unlike + * `includeFamilies`/`excludeFamilies`/`FORWARD_LOOKING_FAMILIES`. Pinning it + * would make every mechanical append fail drift-sync's own gate-2 (the pin + * re-assert) and revert itself — the pin and the ledger are mutually exclusive + * by construction. That costs nothing, because the one-line silencing edit a + * pin defends against has no target here: this set gates no alert a human ever + * sees. The invariant that DOES matter is enforced as a test instead — + * `deprecatedFamilies[p] ⊆ includeFamilies[p]`, i.e. only a family aimock + * actually mocks can be recorded as retired. + * + * CLEANING UP (optional, human, never automatic). Once nothing references a + * recorded family, it may be dropped from aimock entirely: delete it from BOTH + * `includeFamilies[provider]` and this set, then re-pin + * `DATA_FROZEN["includeFamilies."]` in `logic-pin.test.ts`. All three + * edits belong in ONE reviewed commit — the re-pin is exactly the deliberate + * human decision the pin exists to force, which is why drift-sync never makes + * it. Nothing breaks if it is never done; the family just keeps being mocked. + */ +export const deprecatedFamilies: Record> = { + openai: familySet("openai", [ + // drift-sync appends recorded deprecations here. + ]), + anthropic: familySet("anthropic", [ + // drift-sync appends recorded deprecations here. + ]), + gemini: familySet("gemini", [ + // drift-sync appends recorded deprecations here. + ]), +}; + +/** + * True when `family`'s retirement has already been recorded in + * {@link deprecatedFamilies} — i.e. the provider's listing dropped it, the fact + * is written down, and re-detecting it is noise rather than news. + * + * Deliberately separate from {@link isClassifiedFamily}: classification decides + * whether a LIVE family is drift, this decides whether a MISSING family is news. + */ +export function isRecordedDeprecation(family: string, provider: Provider): boolean { + return deprecatedFamilies[provider].has(family); +} + /** * aimock "provider mode" names: internal routing names that reuse a real * upstream provider key but are NOT model ids any provider's `/models` endpoint From f2ab22215ec374234169963c48bf521d399ef9e0 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 7 Aug 2026 15:24:38 -0700 Subject: [PATCH 2/4] test(drift-sync): cover the deprecation ledger; retire the removal-probe tests Adds coverage for the empty-array append, the ten-in-one-run shape, the second-run silence, and the pin staying green; adds the ledger's own registry invariants and the mirror's second bounded divergence. Drops the tests for the zero-reference removal probe along with the probe itself. --- .github/workflows/fix-drift.yml | 32 +- DRIFT.md | 18 +- scripts/drift-sync.ts | 9 + src/__tests__/drift-sync-core.test.ts | 389 +++++++++++------- .../drift-sync-mirror-equivalence.test.ts | 64 +++ 5 files changed, 330 insertions(+), 182 deletions(-) diff --git a/.github/workflows/fix-drift.yml b/.github/workflows/fix-drift.yml index 765bb56d..630f8101 100644 --- a/.github/workflows/fix-drift.yml +++ b/.github/workflows/fix-drift.yml @@ -178,11 +178,15 @@ jobs: # drift-sync.ts fetches each provider's live `/models` listing directly (no # drift-report.json input needed), diffs it against the frozen # `model-registry.ts` classification, and: - # - a zero-reference deprecated family -> mechanical, comment-marked - # removal, gated behind drift-sync-check.ts (allowlist + checksum-pin - # re-assert + clean re-collect) BEFORE the edit is kept, then commits. - # - a still-referenced deprecation, or a genuinely new/unclassified - # family -> NEVER auto-edited; drops a family-keyed dedup note file + # - a deprecated family (of EITHER reference class) -> mechanical, + # comment-marked record in deprecatedFamilies, gated behind + # drift-sync-check.ts (allowlist + checksum-pin re-assert + clean + # re-collect) BEFORE the edit is kept, then commits. The provider's own + # listing already says the family is gone, so this is a fact to write + # down, not a decision to escalate — it NEVER reports needs-human, and + # the mock keeps serving (includeFamilies is untouched). + # - a genuinely new/unclassified family, or a registry structural + # mismatch -> NEVER auto-edited; drops a family-keyed dedup note file # under drift-proposals/ and reports needs-human (the job goes RED # below so a human sees it — and the note is already in the repo on # subsequent runs, so a re-fire is not PR spam). @@ -1555,10 +1559,9 @@ jobs: echo "## Needs a human decision (drift-sync)" echo "" echo "The deterministic, zero-LLM drift-sync found a model-family change it must" - echo "NOT auto-apply (a genuinely new/unclassified family, a deprecation —" - echo "still-referenced or zero-reference — or a registry structural mismatch). It" - echo "wrote the note file(s) below and opened this PR so the decision is REACHABLE" - echo "in the repo." + echo "NOT auto-apply (a genuinely new/unclassified family, or a registry" + echo "structural mismatch). It wrote the note file(s) below and opened this PR so" + echo "the decision is REACHABLE in the repo." echo "" echo "> **Not auto-merged — a human decides.** For a *new-family* note: set the" echo "> note's decision line to \`Decision: include\` (to classify it) or delete the" @@ -1787,8 +1790,11 @@ jobs: fi echo "recorded the suppression ack on PR #${CLOSED_PR} — the delivered notice is not re-posted" - # Alert: the sync found a genuinely-new family or a still-referenced - # deprecation — the two irreducible human decisions. The note was persisted by + # Alert: the sync found a genuinely-new family, or could not locate the + # registry array it had to edit — the two irreducible human decisions. A + # DEPRECATION is deliberately not one of them: the provider's own listing + # already reports it, so drift-sync records it mechanically and this alert + # never fires for one. The note was persisted by # the step above (a distinct PR was opened, or the note was already in the repo # / already proposed in an open PR — no PR spam). This alert fires the job RED # so a human sees it in CI status too. @@ -1817,7 +1823,7 @@ jobs: # A REAL newline — see the cancellation alert above for why a # double-quoted "\n" renders literally in Slack instead. NL=$'\n' - echo "::error::drift-sync found a genuinely-new model family or a deprecation it must not apply itself — needs a human decision (see the needs-human PR / drift-proposals/*.md)" + echo "::error::drift-sync found a genuinely-new model family, or a registry structural mismatch it must not repair itself — needs a human decision (see the needs-human PR / drift-proposals/*.md)" if [ -z "${SLACK_WEBHOOK:-}" ]; then echo "::error::SLACK_WEBHOOK not set — cannot send needs-human alert" exit 1 @@ -1832,7 +1838,7 @@ jobs: else WHERE="No open PR carries this decision (the note may not have been persisted, or its PR was merged/closed). See the drift-sync-log artifact and \`drift-proposals/\`." fi - MSG="🧭 *Drift sync — needs a human decision* — a new/unclassified model family or a still-referenced deprecated family was found. ${WHERE}${NL}Run: https://github.com/${REPO}/actions/runs/${RUN_ID}" + MSG="🧭 *Drift sync — needs a human decision* — a new/unclassified model family, or a registry structural mismatch, was found. ${WHERE}${NL}Run: https://github.com/${REPO}/actions/runs/${RUN_ID}" PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" # Bounded: `--max-time` so a hung POST cannot burn the job's (or a # cancellation's) whole budget, and a small bounded `--retry` so a single diff --git a/DRIFT.md b/DRIFT.md index d3fea748..3c56933b 100644 --- a/DRIFT.md +++ b/DRIFT.md @@ -93,13 +93,9 @@ When a `critical` drift is detected: ## Model Deprecation -The `models.drift.ts` test scrapes model names referenced in aimock's test files, README, and fixtures, then checks each provider's model listing API to verify they still exist. +`models.drift.ts` normalizes each provider's live `GET /models` listing to family keys and subtracts the frozen classification in `model-registry.ts`. Two directions fall out of that subtraction: a live family we do not classify (**new family** — see the automated sync below), and a classified family the listing no longer contains (**deprecation**). -When a model is deprecated: - -1. Update the model name in the affected test files and fixtures -2. Update `src/__tests__/drift/providers.ts` if the cheap test model changed -3. Run `pnpm test` and `pnpm test:drift` +**A deprecation needs nothing from you.** The daily sync records it in `deprecatedFamilies` and aimock keeps mocking the family, so clients pinned to a retired model id keep working. The only thing worth doing by hand is the cheap live test model: if `src/__tests__/drift/providers.ts` names a model that no longer exists, the live drift legs cannot run at all, so point them at a current one and re-run `pnpm test:drift`. ## Adding a New Provider @@ -190,12 +186,12 @@ daily **scheduled cron** (independent of drift-test failure — a retired model family does not, by itself, fail the drift tests): 1. **Sync** — `scripts/drift-sync.ts` fetches each provider's live `/models` listing directly and diffs it against the frozen classification in `src/__tests__/drift/model-registry.ts`: - - a classified family absent from live listings with **zero remaining aimock references** → the removal is mechanically determined but **not applied**: a family-keyed note under `drift-proposals/` proposes it and names the exact two-file edit. `includeFamilies`'s membership is checksum-pinned in `logic-pin.test.ts`, so removing a family means re-pinning it, and that re-pin is the reviewed decision the pin exists to force — a decision the sync's own changed-file allowlist forbids it from making. (A removal the sync applied itself would fail gate 2 on the pin, revert every note the same run wrote, and deliver nothing.) - - a still-referenced deprecated family, or a genuinely new/unclassified family → same route, for the stronger reason that the decision itself is a human's: a family-keyed dedup note file is written under `drift-proposals/` and the run is routed to a human (no PR spam on re-fire) -2. **Gate** — `scripts/drift-sync-check.ts` re-verifies any mechanical edit before (inside `drift-sync.ts`) and after (workflow defense-in-depth) it is kept: a changed-file allowlist (only `model-registry.ts` data literals + `drift-proposals/` notes), a checksum-pin re-assert over the frozen classification logic, and a clean re-collect + - a classified family a healthy live listing no longer contains → **a provider-confirmed deprecation is a fact, not a decision**, so it never routes to a human. drift-sync RECORDS it, mechanically, as a comment-marked entry in `deprecatedFamilies[provider]` (`model-registry.ts`), stamped with the date and with whether aimock's own source still references it. **The mock keeps serving**: `includeFamilies` is untouched, so every builder and fixture for that family still answers — users pin retired model ids in their own suites for years, and the upstream catalog shrinking is not a reason to break them. Recording it is also what makes it stop: the detector filters recorded families out of its candidate set, so the same retirement is not re-derived every morning for ever. Dropping a retired family from aimock altogether stays optional human cleanup (delete it from `includeFamilies` **and** `deprecatedFamilies`, then re-pin `DATA_FROZEN["includeFamilies."]` in `logic-pin.test.ts`, all in one reviewed commit) — the re-pin is the reviewed decision the pin exists to force, which the sync's own changed-file allowlist forbids it from making. Nothing is broken while it is undone. + - a genuinely new/unclassified family, or a registry structural mismatch (the AST locator could not find the array it had to edit) → **not** auto-applied: the decision itself is a human's. A family-keyed dedup note file is written under `drift-proposals/` and the run is routed to a human (no PR spam on re-fire) +2. **Gate** — `scripts/drift-sync-check.ts` re-verifies any mechanical edit before (inside `drift-sync.ts`) and after (workflow defense-in-depth) it is kept: a changed-file allowlist (only `model-registry.ts` data literals + `drift-proposals/` notes), a checksum-pin re-assert over the frozen classification logic, and a clean re-collect. `deprecatedFamilies` is the one registry set deliberately **not** membership-pinned — a pin on the ledger the sync appends to would red on the sync's own append and revert it, every morning, forever. It gates no alert a human sees (`isClassifiedFamily` does not consult it), so there is nothing for a pin to defend; its invariants are asserted behaviourally in `model-registry.test.ts` instead. 3. **PR** — the workflow opens a pull request for a human to review + merge (never auto-merged), unless an open PR already proposes the same changeset or a human has already rejected it. There are two distinct PR classes: - - **`ok-applied`** — a successful mechanical registry edit, i.e. an **addition** a human already approved on a prior run (a deprecation never reaches this class; see above). Pushed onto the `fix/drift-*` branch `drift-sync.ts` committed onto; a human reviews CI + the diff and merges. - - **`needs-human`** — a routed decision. `drift-sync.ts` commits the `drift-proposals/` note file(s), and the workflow pushes a **distinct `drift-needs-human/*` branch** and opens a PR so the note lands in the repo (the job also goes RED + Slack-alerts so the decision is seen). The PR is **never auto-merged**. To approve a _new-family_ note, set its `Decision: include` line and **merge the PR**; the **next** drift-sync run reads the approved note from `main` and applies the mechanical registry edit (an `ok-applied` PR). That two-run hand-off is how the loop closes. + - **`ok-applied`** — a successful mechanical registry edit: a recorded **deprecation**, or an **addition** a human already approved on a prior run. Pushed onto the `fix/drift-*` branch `drift-sync.ts` committed onto; a human reviews CI + the diff and merges. No alert, no red run — it is data-only bookkeeping. + - **`needs-human`** — a routed decision, and now only a genuinely new/unclassified family or a registry structural mismatch. `drift-sync.ts` commits the `drift-proposals/` note file(s), and the workflow pushes a **distinct `drift-needs-human/*` branch** and opens a PR so the note lands in the repo (the job also goes RED + Slack-alerts so the decision is seen). The PR is **never auto-merged**. To approve a _new-family_ note, set its `Decision: include` line and **merge the PR**; the **next** drift-sync run reads the approved note from `main` and applies the mechanical registry edit (an `ok-applied` PR). That two-run hand-off is how the loop closes. **Closing a drift-sync PR REJECTS that changeset, permanently.** A CLOSED-but-never-merged PR carrying the `` marker tells the workflow a human decided against that exact changeset, so it stops re-proposing it (a genuinely different drift hashes to a different key and is unaffected; a **merged** PR is an accepted decision and is never read as a rejection). A still-**open** PR carrying the marker always wins over a closed one, so closing a duplicate does not reject the changeset the surviving PR is still proposing. The suppression is **not silent, and not repetitive** — the first run after the closure posts a Slack line naming the closing PR, then records an ack marker in that PR's body so the identical line is not re-posted every morning for as long as the rejection stands (which is for ever: the closure is permanent and the changeset key is date-independent). Delete that ack marker and the next run reports the suppression again. **To un-suppress: REOPEN that PR** — it becomes the pending proposal again, and the registry stays drifted until you do. Deleting the `` marker from the closed PR's body does **not** un-suppress: the marker self-heal now covers closed PRs and puts it back, because that marker going missing is far more often a human rewriting the body (to write down _why_ they declined) than a deliberate un-suppression — and losing it that way used to resurrect the rejected changeset every morning, permanently. Reopening is the deliberate act; a body edit is not. diff --git a/scripts/drift-sync.ts b/scripts/drift-sync.ts index 3808e06d..83bd744d 100644 --- a/scripts/drift-sync.ts +++ b/scripts/drift-sync.ts @@ -691,6 +691,14 @@ export interface SyncCheckResultLike { export interface SyncCoreDeps { isReferenced?: (family: string, provider: Provider) => boolean; + /** + * "Has this retirement already been written down?" Defaults to the real + * ledger (`isRecordedDeprecation` over `deprecatedFamilies`). Injectable + * because the real one reads the COMPILED registry module while the core + * edits registry SOURCE TEXT, so a test cannot otherwise observe a second run + * seeing the first run's record. + */ + isRecorded?: (family: string, provider: Provider) => boolean; readRegistrySource: () => string; writeRegistrySource: (text: string) => void; readProposalNote: (relPath: string) => string | null; @@ -777,6 +785,7 @@ export function runDriftSyncCore( // --- Deprecation half: classified − live (C4's algorithm, mirrored). --- const dep = detectDeprecatedFamiliesForSync(input.liveModelIds, input.provider, { isReferenced: deps.isReferenced, + isRecorded: deps.isRecorded, }); if (dep.status === "skipped") { skipped.push({ provider: input.provider, reason: dep.reason }); diff --git a/src/__tests__/drift-sync-core.test.ts b/src/__tests__/drift-sync-core.test.ts index 8a93b401..14e57240 100644 --- a/src/__tests__/drift-sync-core.test.ts +++ b/src/__tests__/drift-sync-core.test.ts @@ -10,24 +10,24 @@ * RED (observed before this module existed): `scripts/drift-sync.ts` exported * only the C1 git/branch/commit/PR plumbing (todayStamp, exec, getChangedFiles, * buildPrBody, gatedCommitFiles, ...) — none of `runDriftSyncCore`, - * `detectDeprecatedFamiliesForSync`, `removeFamilyLiteralInSource`, etc. - * existed, so a live churn scenario (a new classified model, or a retired - * family) had NO mechanical sync path at all: the only remediation route was - * the LLM freewriter. Verbatim capture of that RED state (this test file - * against the pre-C2 module) is in the slot's final report. + * `detectDeprecatedFamiliesForSync`, `addFamilyLiteralInSource`, etc. existed, + * so a live churn scenario (a new classified model, or a retired family) had NO + * mechanical sync path at all: the only remediation route was the LLM + * freewriter. Verbatim capture of that RED state (this test file against the + * pre-C2 module) is in the slot's final report. */ import { describe, it, expect, vi } from "vitest"; import { execFileSync } from "node:child_process"; import { readFileSync, writeFileSync, existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import ts from "typescript"; -import { includeFamilies } from "./drift/model-registry.js"; +import { includeFamilies, deprecatedFamilies } from "./drift/model-registry.js"; import { MIN_LISTING_SIZE } from "./drift/deprecation-detector.js"; import { detectDeprecatedFamiliesForSync, unclassifiedFamiliesForSync, - removeFamilyLiteralInSource, addFamilyLiteralInSource, proposalNoteRelPath, parseProposalDecision, @@ -65,10 +65,55 @@ function fixtureRegistrySource(): string { ' "gemini-2.5-flash",', " ]),", "};", + // The recorded-deprecation ledger, in the shape the real file ships it: + // three EMPTY arrays held open by a comment. Empty is the shape that + // matters — it is the only one where the insert has no sibling element to + // copy an indent from, and it is what the real registry looks like on the + // first morning a provider retires anything. + "export const deprecatedFamilies = {", + ' openai: set("openai", [', + " // drift-sync appends recorded deprecations here.", + " ]),", + ' anthropic: set("anthropic", [', + " // drift-sync appends recorded deprecations here.", + " ]),", + ' gemini: set("gemini", [', + " // drift-sync appends recorded deprecations here.", + " ]),", + "};", ]; return lines.join("\n"); } +/** + * Re-parse an edited registry source and read back `exportName[provider]`'s + * array members, using the TypeScript parser DIRECTLY rather than the sync's own + * `locateFamilySetArray`. Deliberately independent: an edit that lands in the + * wrong array — or produces text that no longer parses — has to be visible to + * something other than the code that made it. `toEqual` on the member list is + * order-sensitive, so an append that lands in the wrong position shows up too. + */ +function parsedFamilyArray(sourceText: string, exportName: string, provider: string): string[] { + const sf = ts.createSourceFile("registry.ts", sourceText, ts.ScriptTarget.Latest, true); + for (const stmt of sf.statements) { + if (!ts.isVariableStatement(stmt)) continue; + for (const decl of stmt.declarationList.declarations) { + if (!ts.isIdentifier(decl.name) || decl.name.text !== exportName) continue; + if (!decl.initializer || !ts.isObjectLiteralExpression(decl.initializer)) continue; + for (const prop of decl.initializer.properties) { + if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) continue; + if (prop.name.text !== provider) continue; + const init = prop.initializer; + if (!ts.isCallExpression(init) || !ts.isArrayLiteralExpression(init.arguments[1])) continue; + return init.arguments[1].elements + .filter(ts.isStringLiteral) + .map((el: ts.StringLiteral) => el.text); + } + } + } + throw new Error(`could not parse ${exportName}.${provider} out of the edited source`); +} + /** In-memory fake fs + gate for runDriftSyncCore — no real disk/git touched. */ function makeFakeDeps(overrides: Partial = {}): { deps: SyncCoreDeps; @@ -361,68 +406,61 @@ describe("unclassifiedFamiliesForSync (mirrors C4's unclassifiedFamilies)", () = // Mechanical registry edits — AST-located, single-line surgery. // --------------------------------------------------------------------------- -describe("removeFamilyLiteralInSource / addFamilyLiteralInSource", () => { - it("removes an existing family, comment-marking the line, touching nothing else", () => { +describe("addFamilyLiteralInSource", () => { + it("adds a new family literal, comment-marked", () => { const src = fixtureRegistrySource(); - const result = removeFamilyLiteralInSource( + const result = addFamilyLiteralInSource( src, "includeFamilies", "openai", - "gpt-4o", - "TEST-REMOVE", + "gpt-live", + "TEST-ADD", ); expect(result.changed).toBe(true); - expect(result.text).not.toContain('"gpt-4o"'); - expect(result.text).toContain("// TEST-REMOVE"); - // Every other seeded family is untouched. + expect(result.text).toContain('"gpt-live", // TEST-ADD'); + // Every seeded family is untouched. for (const f of includeFamilies.openai) { - if (f === "gpt-4o") continue; expect(result.text).toContain(`"${f}"`); } }); - it("no-ops when the family is not present (never mangles the file)", () => { + it("no-ops when the family is already present (never duplicates)", () => { const src = fixtureRegistrySource(); - const result = removeFamilyLiteralInSource( - src, - "includeFamilies", - "openai", - "zzz-not-real", - "x", - ); + const result = addFamilyLiteralInSource(src, "includeFamilies", "openai", "gpt-4o", "x"); expect(result.changed).toBe(false); - expect(result.text).toBe(src); }); - it("adds a new family literal, comment-marked", () => { + it("writes into an EMPTY, comment-only array — the ledger's day-one shape", () => { + // The empty array is the case with no sibling element to copy an indent + // from, and it is exactly the shape `deprecatedFamilies` ships in. An + // insert that lands one line early here would splice ABOVE the + // `provider: set(...)` line and produce a file that does not parse. const src = fixtureRegistrySource(); const result = addFamilyLiteralInSource( src, - "includeFamilies", - "openai", - "gpt-live", - "TEST-ADD", + "deprecatedFamilies", + "anthropic", + "claude-3-5-sonnet", + "DEPRECATED", ); expect(result.changed).toBe(true); - expect(result.text).toContain('"gpt-live", // TEST-ADD'); - }); - - it("no-ops when the family is already present (never duplicates)", () => { - const src = fixtureRegistrySource(); - const result = addFamilyLiteralInSource(src, "includeFamilies", "openai", "gpt-4o", "x"); - expect(result.changed).toBe(false); + expect(result.text).toContain(' "claude-3-5-sonnet", // DEPRECATED'); + // It landed INSIDE deprecatedFamilies.anthropic, not in includeFamilies and + // not in a sibling provider: the array it went into must now parse with the + // family as a member. + expect(parsedFamilyArray(result.text, "deprecatedFamilies", "anthropic")).toEqual([ + "claude-3-5-sonnet", + ]); + expect(parsedFamilyArray(result.text, "deprecatedFamilies", "openai")).toEqual([]); + expect(parsedFamilyArray(result.text, "deprecatedFamilies", "gemini")).toEqual([]); + expect(parsedFamilyArray(result.text, "includeFamilies", "anthropic")).toEqual([ + "claude-3-5-sonnet", + ]); }); it("the edited text is still syntactically valid TypeScript (parser round-trip)", () => { const src = fixtureRegistrySource(); - const removed = removeFamilyLiteralInSource(src, "includeFamilies", "openai", "gpt-4o", "r"); - const added = addFamilyLiteralInSource( - removed.text, - "includeFamilies", - "openai", - "gpt-live", - "a", - ); + const added = addFamilyLiteralInSource(src, "includeFamilies", "openai", "gpt-live", "a"); // A further edit against the already-edited text must still locate the // array correctly — proves the AST-based locator survives a prior edit. const secondAdd = addFamilyLiteralInSource( @@ -434,6 +472,15 @@ describe("removeFamilyLiteralInSource / addFamilyLiteralInSource", () => { ); expect(secondAdd.changed).toBe(true); expect(secondAdd.text).toContain('"gpt-live-2"'); + // Two successive appends into the SAME empty array must both land, and the + // second must not be fooled by the first's trailing comment. + const one = addFamilyLiteralInSource(src, "deprecatedFamilies", "openai", "gpt-4o", "d1"); + const two = addFamilyLiteralInSource(one.text, "deprecatedFamilies", "openai", "gpt-4", "d2"); + expect(two.changed).toBe(true); + expect(parsedFamilyArray(two.text, "deprecatedFamilies", "openai")).toEqual([ + "gpt-4o", + "gpt-4", + ]); }); }); @@ -461,11 +508,15 @@ describe("proposal notes", () => { expect(parseProposalDecision("Decision: include")).toBe("include"); }); - it("renderProposalNote never generates a Decision line for a deprecation note", () => { + it("renderProposalNote never generates a Decision line for a structural mismatch", () => { + // `Decision: include` is the new-family approval marker the NEXT run acts + // on. A structural mismatch has nothing to approve — the registry's shape + // moved and a human has to look — so offering the marker there would invite + // an "approval" that authorises nothing. const note = renderProposalNote( "openai", "gpt-4o", - "still-referenced-deprecation", + "registry-structural-mismatch", "detail", "2026-07-22", ); @@ -492,108 +543,131 @@ describe("runDriftSyncCore", () => { expect(runSyncCheck).not.toHaveBeenCalled(); }); - it("RED->GREEN (deprecation, zero-reference): removal PROPOSED to a human, registry never touched", () => { - const { deps, registry, notes, runSyncCheck, writeRegistrySource } = makeFakeDeps({ + /** A healthy openai listing that is missing exactly `absent`. */ + function listingMissing(absent: string[]): string[] { + const live = [...includeFamilies.openai].filter((f) => !absent.includes(f)); + return [...live, ...live.map((f) => `${f}-2025-01-01`)]; + } + + it("RED->GREEN (deprecation, zero-reference): RECORDED mechanically, no human paged", () => { + const { deps, registry, notes, runSyncCheck, writeProposalNote } = makeFakeDeps({ isReferenced: () => false, }); - const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); - const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; - const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: liveIds }]; + const inputs: ProviderChurnInput[] = [ + { provider: "openai", liveModelIds: listingMissing(["gpt-4o"]) }, + ]; const outcome = runDriftSyncCore(inputs, deps); - // RED (pre-fix): this applied the removal to `includeFamilies` and reported - // OK_APPLIED. Against the REAL gate that is a lie — `includeFamilies`'s - // membership is checksum-pinned in logic-pin.test.ts, gate-2 re-runs that - // file, and the pin reds on the very edit the gate is validating. The run - // then reverted the registry AND every needs-human note written alongside it - // and reported gate-failed: no PR, nothing delivered, a red cron every day. + // RED (observed, production run 31218975992 on 2026-08-07): a deprecation + // — of EITHER reference class — produced a `needs-human-*` outcome, so the + // run reported `reason=needs-human`, exited 1, opened a + // `drift-needs-human/*` PR and Slack-alerted the repo owner to "decide" a + // retirement the provider had already published. Ten of them in one + // morning, every morning. expect(outcome.outcomes).toContainEqual( expect.objectContaining({ provider: "openai", family: "gpt-4o", - action: "needs-human-zero-reference", + action: "deprecation-recorded", }), ); - expect(outcome.ok).toBe(false); - expect(outcome.reason).toBe(SyncCoreReason.NEEDS_HUMAN); - // The registry is not mutated at all, so there is no edit to gate. - expect(writeRegistrySource).not.toHaveBeenCalled(); - expect(registry.text).toContain('"gpt-4o",'); - expect(runSyncCheck).not.toHaveBeenCalled(); - expect(notes.has(`${DRIFT_PROPOSALS_DIR}/openai-gpt-4o-deprecated-unreferenced.md`)).toBe(true); - }); + expect(outcome.outcomes.some((o) => o.action.startsWith("needs-human-"))).toBe(false); + expect(outcome.ok).toBe(true); + expect(outcome.reason).toBe(SyncCoreReason.OK_APPLIED); + // Nobody was paged and nothing was proposed: no note file of any kind. + expect(writeProposalNote).not.toHaveBeenCalled(); + expect(notes.size).toBe(0); - it("the zero-reference note names BOTH edits a human must make — the literal AND its pin", () => { - // A note that says only "remove the family" sends the human into a red - // logic-pin.test.ts with no explanation. The re-pin is not incidental: it is - // the reviewed decision the pin exists to force, and the reason drift-sync - // proposes the removal rather than applying it. - const { deps, notes } = makeFakeDeps({ isReferenced: () => false }); - const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); - const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + // THE MOCK SURVIVES. `includeFamilies` is byte-identical — the family is + // still classified, still mocked, still served. Only the ledger grew. + expect(parsedFamilyArray(registry.text, "includeFamilies", "openai")).toEqual([ + ...includeFamilies.openai, + ]); + expect(parsedFamilyArray(registry.text, "deprecatedFamilies", "openai")).toEqual(["gpt-4o"]); + // Zero-reference is recorded in the COMMENT, where it tells a human the + // optional cleanup is safe — it no longer selects a different route. + expect(registry.text).toContain("no remaining aimock reference"); - runDriftSyncCore([{ provider: "openai", liveModelIds: liveIds }], deps); - - const note = notes.get(`${DRIFT_PROPOSALS_DIR}/openai-gpt-4o-deprecated-unreferenced.md`); - expect(note).toBeDefined(); - expect(note).toContain("includeFamilies.openai"); - expect(note).toContain(MODEL_REGISTRY_REL_PATH); - expect(note).toContain('DATA_FROZEN["includeFamilies.openai"]'); - expect(note).toContain("src/__tests__/drift/logic-pin.test.ts"); - // Never a Decision line: a deprecation note is not an approval gate the way - // a new-family note is — `Decision: include` here would read as authorising - // drift-sync to apply the removal on the next run, which it cannot do. - expect(note).not.toContain("## Decision"); + // A real registry edit was made, so the real gate DOES run — and with the + // live re-collect ON, because this run deferred nothing to a human. + expect(runSyncCheck).toHaveBeenCalledTimes(1); + expect(runSyncCheck).toHaveBeenCalledWith({ skipRecollect: false }); }); - it("a zero-reference deprecation re-fires without spamming a second note", () => { - const { deps, writeProposalNote } = makeFakeDeps({ isReferenced: () => false }); - const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); - const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; - const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: liveIds }]; + it("RED->GREEN (deprecation, STILL-REFERENCED): also recorded, and the mock is NOT removed", () => { + const { deps, registry, writeProposalNote } = makeFakeDeps({ isReferenced: () => true }); + const inputs: ProviderChurnInput[] = [ + { provider: "openai", liveModelIds: listingMissing(["gpt-4o"]) }, + ]; - runDriftSyncCore(inputs, deps); - expect(writeProposalNote).toHaveBeenCalledTimes(1); - writeProposalNote.mockClear(); + const outcome = runDriftSyncCore(inputs, deps); - // The daily cron re-detects the same drift for as long as the note is - // un-actioned. Family-keyed path => same file => no second write, no PR spam. - const again = runDriftSyncCore(inputs, deps); + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ + provider: "openai", + family: "gpt-4o", + action: "deprecation-recorded", + }), + ); + expect(outcome.ok).toBe(true); + expect(outcome.reason).toBe(SyncCoreReason.OK_APPLIED); expect(writeProposalNote).not.toHaveBeenCalled(); - expect(again.reason).toBe(SyncCoreReason.NEEDS_HUMAN); + // The whole point of the still-referenced class: users' suites still call + // this model. It stays in includeFamilies, so aimock keeps answering. + expect(parsedFamilyArray(registry.text, "includeFamilies", "openai")).toContain("gpt-4o"); + expect(parsedFamilyArray(registry.text, "deprecatedFamilies", "openai")).toEqual(["gpt-4o"]); + expect(registry.text).toContain("still referenced in aimock source — mock retained"); }); - it("RED->GREEN (deprecation, STILL-REFERENCED): routed to human, no auto-edit", () => { - const { deps, registry, writeProposalNote, runSyncCheck } = makeFakeDeps({ + it("a recorded deprecation goes QUIET on the next run — the daily cron stops re-deriving it", () => { + // The reason the ledger exists at all. Detection is a pure + // `includeFamilies − live` diff, so without a written record the same + // retirement is rediscovered every morning for ever. + const { deps, registry } = makeFakeDeps({ isReferenced: () => true, + // Model the ledger the FIRST run wrote: the core reads the registry text + // it edited, but `isRecordedDeprecation` reads the compiled module, so the + // second run's silence has to come from a source the core can see. + isRecorded: (family) => registry.text.includes(`"${family}", // DEPRECATED`), }); - const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); - const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; - const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: liveIds }]; + const inputs: ProviderChurnInput[] = [ + { provider: "openai", liveModelIds: listingMissing(["gpt-4o"]) }, + ]; - const outcome = runDriftSyncCore(inputs, deps); + const first = runDriftSyncCore(inputs, deps); + expect(first.reason).toBe(SyncCoreReason.OK_APPLIED); + expect(first.outcomes).toHaveLength(1); - expect(outcome.outcomes).toContainEqual( - expect.objectContaining({ - provider: "openai", - family: "gpt-4o", - action: "needs-human-still-referenced", - }), + const second = runDriftSyncCore(inputs, deps); + expect(second.reason).toBe(SyncCoreReason.OK_NO_CHURN); + expect(second.ok).toBe(true); + expect(second.outcomes).toEqual([]); + // And it was not recorded twice. + expect(parsedFamilyArray(registry.text, "deprecatedFamilies", "openai")).toEqual(["gpt-4o"]); + }); + + it("a whole morning's worth of deprecations records in ONE run, still zero escalations", () => { + // The production shape: Anthropic retired ten families at once. Every one + // must land in the same mechanical run — not nine plus one escalation. + const absent = [...includeFamilies.openai].slice(0, 10); + const { deps, registry } = makeFakeDeps({ isReferenced: (f) => f !== absent[0] }); + const outcome = runDriftSyncCore( + [{ provider: "openai", liveModelIds: listingMissing(absent) }], + deps, ); - expect(outcome.ok).toBe(false); - expect(outcome.reason).toBe(SyncCoreReason.NEEDS_HUMAN); - // Registry data itself was NEVER mechanically touched for a still-referenced family. - expect(registry.text).toContain('"gpt-4o"'); - expect(writeProposalNote).toHaveBeenCalledWith( - `${DRIFT_PROPOSALS_DIR}/openai-gpt-4o-deprecated-referenced.md`, - expect.stringContaining("still references it"), + + expect(outcome.reason).toBe(SyncCoreReason.OK_APPLIED); + expect(outcome.ok).toBe(true); + expect(outcome.outcomes.filter((o) => o.action === "deprecation-recorded")).toHaveLength(10); + expect(outcome.outcomes.some((o) => o.action.startsWith("needs-human-"))).toBe(false); + expect(parsedFamilyArray(registry.text, "deprecatedFamilies", "openai").sort()).toEqual( + [...absent].sort(), ); - // D-M1: a note-only run has NO registry edit to re-verify, so it is NEVER - // gated behind the (recollect-bearing) drift-sync-check — otherwise gate-3 - // would re-detect the un-actioned family it just routed to a human and - // revert the note. The gate is not consulted at all here. - expect(runSyncCheck).not.toHaveBeenCalled(); + // Not one of the ten left includeFamilies. + expect(parsedFamilyArray(registry.text, "includeFamilies", "openai")).toEqual([ + ...includeFamilies.openai, + ]); }); it("RED->GREEN (genuinely new family, no prior decision): RED alert + single deduped note, no auto-classify", () => { @@ -687,29 +761,28 @@ describe("runDriftSyncCore", () => { expect(revertFiles).toHaveBeenCalledWith([MODEL_REGISTRY_REL_PATH]); }); - it("RED->GREEN: a zero-reference deprecation never puts the run's OTHER notes at the gate's mercy", () => { - // THE DEFECT, as an invariant. `includeFamilies` membership is checksum-pinned - // in logic-pin.test.ts, and gate-2 re-runs that file over the edited tree — so - // a PERSISTED mechanical removal is guaranteed to fail its own gate. This gate - // models that faithfully (it refuses any registry edit exactly as the real pin - // does), while the run ALSO has genuine still-referenced deprecations to - // deliver. + it("recording a deprecation leaves the includeFamilies checksum pin GREEN", () => { + // THE CONSTRAINT THE LEDGER EXISTS TO SATISFY. `includeFamilies`'s + // membership is checksum-pinned in logic-pin.test.ts and gate-2 re-runs that + // file over the edited tree, so any edit that touches that set fails the + // sync's own gate, reverts, and delivers nothing. This gate models the pin + // faithfully: it refuses precisely when includeFamilies moved. // - // RED (pre-fix, OBSERVED against the real gate and the repo's own frozen - // healthy 16-id anthropic wave): removal applied -> pin-check-failed -> - // revertFiles([...touchedFiles]) wiped the registry edit AND all three notes - // -> reason=gate-failed -> no PR of any class, so the eight deprecations the - // anthropic floor fix exists to surface were detected and then thrown away, - // every morning, behind a gate-failure alert. - const runSyncCheck = vi.fn( - (): SyncCheckResultLike => ({ - ok: false, - reason: "pin-check-failed", - detail: 'Frozen data set "includeFamilies.openai" membership changed', - }), - ); - // gpt-4o is zero-reference (the removal candidate); every other absent family - // is still referenced (a human note). + // RED (the shape this replaced): the sync's answer to a zero-reference + // deprecation was to remove the family from includeFamilies, which reddened + // exactly this gate — `revertFiles` wiped the edit and every note written + // alongside it, `reason=gate-failed`, no PR of any class. Which is why the + // pre-fix code did not apply the removal at all and paged a human instead. + const runSyncCheck = vi.fn((): SyncCheckResultLike => { + const included = parsedFamilyArray(registry.text, "includeFamilies", "openai"); + return included.length === includeFamilies.openai.size + ? { ok: true, reason: "ok", detail: "allowlist + pin ok" } + : { + ok: false, + reason: "pin-check-failed", + detail: 'Frozen data set "includeFamilies.openai" membership changed', + }; + }); const { deps, notes, registry, revertFiles } = makeFakeDeps({ isReferenced: (family) => family !== "gpt-4o", runSyncCheck, @@ -720,18 +793,15 @@ describe("runDriftSyncCore", () => { const outcome = runDriftSyncCore([{ provider: "openai", liveModelIds: liveIds }], deps); - // GREEN: nothing is reverted, because nothing was risked — no registry edit - // means the gate is never consulted, and every note survives to become the - // one needs-human PR this run is supposed to produce. - expect(outcome.reason).toBe(SyncCoreReason.NEEDS_HUMAN); - expect(runSyncCheck).not.toHaveBeenCalled(); + // GREEN: the pin-modelling gate PASSES, so the edit is kept and delivered. + expect(outcome.reason).toBe(SyncCoreReason.OK_APPLIED); + expect(outcome.ok).toBe(true); + expect(runSyncCheck).toHaveBeenCalledTimes(1); expect(revertFiles).not.toHaveBeenCalled(); - expect(registry.text).toContain('"gpt-4o",'); - expect(notes.size).toBe(dropped.length); - for (const family of dropped) { - const kind = family === "gpt-4o" ? "deprecated-unreferenced" : "deprecated-referenced"; - expect(notes.has(`${DRIFT_PROPOSALS_DIR}/openai-${family}-${kind}.md`)).toBe(true); - } + expect(notes.size).toBe(0); + expect(parsedFamilyArray(registry.text, "deprecatedFamilies", "openai").sort()).toEqual( + [...dropped].sort(), + ); }); it("a provider whose live listing was skipped (no key / infra error) is recorded, not treated as churn", () => { @@ -837,8 +907,11 @@ describe("D-M1: recollect gate vs route-to-human invariant", () => { describe("G#1: locator miss routes to human", () => { it("RED->GREEN (deprecation locator miss): writes a note + NEEDS_HUMAN, never a silent no-op", () => { - // A registry source the AST locator cannot parse into includeFamilies — - // models the real file's structure changing out from under the editor. + // A registry source the AST locator cannot parse into deprecatedFamilies — + // models the real file's structure changing out from under the editor. A + // deprecation that cannot be RECORDED would otherwise be re-derived and + // re-dropped every morning in silence, which is the one deprecation shape a + // human genuinely has to see. const brokenSource = "export const somethingElse = { openai: [] };\n"; const brokenRegistry = { text: brokenSource }; const { deps, notes } = makeFakeDeps({ diff --git a/src/__tests__/drift-sync-mirror-equivalence.test.ts b/src/__tests__/drift-sync-mirror-equivalence.test.ts index 2bbb7fde..27b1ceb1 100644 --- a/src/__tests__/drift-sync-mirror-equivalence.test.ts +++ b/src/__tests__/drift-sync-mirror-equivalence.test.ts @@ -295,4 +295,68 @@ describe("drift-sync mirror ≡ text-drift.ts source (classification equivalence expect(syncFamilies).toContain(genuinelyRetired); expect(canonicalFamilies).toContain(genuinelyRetired); }); + + it("detectDeprecatedFamilies: the sync mirror's ALREADY-RECORDED exclusion is a second INTENTIONAL, bounded divergence", () => { + // The sibling of the forward-looking divergence above, and the same + // layering: DETECTION vs POLICY. The canonical detector reports every + // classified family missing from the live listing, every time. The sync + // mirror additionally drops the ones whose retirement is already WRITTEN + // DOWN in `deprecatedFamilies` (model-registry.ts) — because acting on a + // deprecation is a one-time mechanical record, and re-deriving it on the + // next daily cron is noise, not news. + // + // `deprecatedFamilies` is EMPTY in the committed registry (drift-sync fills + // it in production), so the divergence is driven through the mirror's + // injectable `isRecorded` seam rather than by mutating shared module state. + // Without the injection the two agree — which is the bound: the mirror + // drops recorded families and NOTHING else. + // + // Do NOT "fix" a future failure here by deleting the `isRecorded` filter + // from drift-sync.ts to make the two identical again — that reinstates the + // daily re-derivation of every already-recorded retirement, which is the + // whole defect this filter removes. + const anthropicProvider: Provider = "anthropic"; + const allAnthropic = [...includeFamilies.anthropic]; + const alreadyRecorded = "claude-3-5-sonnet"; + const freshlyRetired = "claude-3-opus"; + const liveFamiliesList = allAnthropic.filter( + (f) => f !== alreadyRecorded && f !== freshlyRetired && f !== "claude-fable-5", + ); + const liveIds = [...liveFamiliesList, ...liveFamiliesList.map((f) => `${f}-2025-01-01`)]; + + const canonicalResult = detectDeprecatedFamilies(liveIds, anthropicProvider); + const withoutLedger = detectDeprecatedFamiliesForSync(liveIds, anthropicProvider); + const withLedger = detectDeprecatedFamiliesForSync(liveIds, anthropicProvider, { + isRecorded: (family) => family === alreadyRecorded, + }); + + expect(canonicalResult.status).toBe("checked"); + expect(withoutLedger.status).toBe("checked"); + expect(withLedger.status).toBe("checked"); + if ( + canonicalResult.status !== "checked" || + withoutLedger.status !== "checked" || + withLedger.status !== "checked" + ) { + throw new Error("expected all three results to be 'checked' for this fixture"); + } + + const canonicalFamilies = canonicalResult.candidates.map((c) => c.family); + const withoutLedgerFamilies = withoutLedger.candidates.map((c) => c.family); + const withLedgerFamilies = withLedger.candidates.map((c) => c.family); + + // Un-recorded, the mirror agrees with the canonical detector on BOTH + // families — so the divergence below is the ledger's doing and nothing else. + expect(withoutLedgerFamilies).toContain(alreadyRecorded); + expect(canonicalFamilies).toContain(alreadyRecorded); + + // The one intended divergence: recorded => dropped by the mirror, still + // reported by the canonical detector. + expect(withLedgerFamilies).not.toContain(alreadyRecorded); + expect(canonicalFamilies).toContain(alreadyRecorded); + + // Bounded: a retirement that is NOT recorded is still reported by both. + expect(withLedgerFamilies).toContain(freshlyRetired); + expect(canonicalFamilies).toContain(freshlyRetired); + }); }); From 70c6afce7b8f50d91a34dff045b01209764a8cae Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 7 Aug 2026 15:30:15 -0700 Subject: [PATCH 3/4] test(drift-sync): derive retired-family fixtures so the growing ledger cannot void them Recording anthropic's ten 2026-08-07 retirements broke four fixtures that hard-coded claude-3-opus as a genuinely-retired stand-in: the mirror now correctly drops a recorded family, so those fixtures asserted on an empty candidate list. The stand-ins are derived from families that are neither forward-looking nor already recorded, and throw loudly if none remain. --- src/__tests__/drift-sync-core.test.ts | 40 +++++++++++-- .../drift-sync-mirror-equivalence.test.ts | 58 ++++++++++++++++--- 2 files changed, 83 insertions(+), 15 deletions(-) diff --git a/src/__tests__/drift-sync-core.test.ts b/src/__tests__/drift-sync-core.test.ts index 14e57240..d7ba329e 100644 --- a/src/__tests__/drift-sync-core.test.ts +++ b/src/__tests__/drift-sync-core.test.ts @@ -24,7 +24,7 @@ import { join } from "node:path"; import ts from "typescript"; import { includeFamilies, deprecatedFamilies } from "./drift/model-registry.js"; -import { MIN_LISTING_SIZE } from "./drift/deprecation-detector.js"; +import { MIN_LISTING_SIZE, FORWARD_LOOKING_FAMILIES } from "./drift/deprecation-detector.js"; import { detectDeprecatedFamiliesForSync, unclassifiedFamiliesForSync, @@ -85,6 +85,35 @@ function fixtureRegistrySource(): string { return lines.join("\n"); } +/** + * `count` anthropic families that stand in for GENUINELY RETIRED ones: each is + * classified INCLUDE, not forward-looking, and not already in the recorded + * deprecation ledger — so the sync mirror's `missing` set really does report it. + * + * DERIVED, NEVER HARD-CODED, and that is load-bearing. `deprecatedFamilies` + * GROWS on its own: drift-sync appends to it unattended the morning a provider + * retires a family. A fixture pinned to a literal `claude-3-opus` silently + * stops being a valid stand-in the moment the sync records that family — the + * mirror then (correctly) drops it, and the fixture asserts on an empty + * candidate list. Observed: recording anthropic's ten 2026-08-07 retirements + * broke four tests across this file and the mirror-equivalence guard at once. + * Throws loudly rather than letting a fixture go quietly vacuous. + */ +function unrecordedAnthropicFamilies(count: number): string[] { + const usable = [...includeFamilies.anthropic].filter( + (f) => !deprecatedFamilies.anthropic.has(f) && !FORWARD_LOOKING_FAMILIES.anthropic.has(f), + ); + if (usable.length < count) { + throw new Error( + `need ${count} anthropic families that are neither forward-looking nor already recorded ` + + `as deprecated, but only ${usable.length} remain (${usable.join(", ")}). These fixtures ` + + `need a family the deprecation detector will actually report; pick a different provider ` + + `rather than deleting the assertion.`, + ); + } + return usable.slice(0, count); +} + /** * Re-parse an edited registry source and read back `exportName[provider]`'s * array members, using the TypeScript parser DIRECTLY rather than the sync's own @@ -205,11 +234,10 @@ describe("detectDeprecatedFamiliesForSync (mirrors C4's detectDeprecatedFamilies // missing from the same listing must still be proposed normally. it("a forward-looking family (claude-fable-5) absent from live is NEVER proposed for removal, while a genuinely-retired sibling still is", () => { const allAnthropic = [...includeFamilies.anthropic]; + const [retired] = unrecordedAnthropicFamilies(1); // Live listing omits BOTH claude-fable-5 (forward-looking, not launched) - // AND claude-3-opus (stand-in for a genuinely retired family). - const liveFamilies = allAnthropic.filter( - (f) => f !== "claude-fable-5" && f !== "claude-3-opus", - ); + // AND a genuinely-retired stand-in. + const liveFamilies = allAnthropic.filter((f) => f !== "claude-fable-5" && f !== retired); const liveIds = [...liveFamilies, ...liveFamilies.map((f) => `${f}-20250101`)]; const result = detectDeprecatedFamiliesForSync(liveIds, "anthropic", { isReferenced: () => false, @@ -217,7 +245,7 @@ describe("detectDeprecatedFamiliesForSync (mirrors C4's detectDeprecatedFamilies expect(result.status).toBe("checked"); if (result.status !== "checked") return; const families = result.candidates.map((c) => c.family).sort(); - expect(families).toEqual(["claude-3-opus"]); + expect(families).toEqual([retired]); }); }); diff --git a/src/__tests__/drift-sync-mirror-equivalence.test.ts b/src/__tests__/drift-sync-mirror-equivalence.test.ts index 27b1ceb1..51471f33 100644 --- a/src/__tests__/drift-sync-mirror-equivalence.test.ts +++ b/src/__tests__/drift-sync-mirror-equivalence.test.ts @@ -54,7 +54,36 @@ import { type Provider, } from "../../scripts/drift-sync.js"; import { detectDeprecatedFamilies, unclassifiedFamilies } from "./drift/text-drift.js"; -import { includeFamilies } from "./drift/model-registry.js"; +import { includeFamilies, deprecatedFamilies } from "./drift/model-registry.js"; +import { FORWARD_LOOKING_FAMILIES } from "./drift/deprecation-detector.js"; + +/** + * `count` anthropic families that stand in for GENUINELY RETIRED ones in these + * fixtures: classified INCLUDE, not forward-looking, and not already in the + * recorded-deprecation ledger — so BOTH detectors really do report them and the + * comparison is not between two empty lists. + * + * DERIVED, NEVER HARD-CODED. `deprecatedFamilies` grows unattended (drift-sync + * appends to it the morning a provider retires a family), so a literal + * `claude-3-opus` stops being a valid stand-in the moment the sync records it: + * the mirror then correctly drops it and the fixture's premise silently + * evaporates. Observed — recording anthropic's ten 2026-08-07 retirements broke + * three fixtures in this file at once. Throws rather than going vacuous. + */ +function unrecordedAnthropicFamilies(count: number): string[] { + const usable = [...includeFamilies.anthropic].filter( + (f) => !deprecatedFamilies.anthropic.has(f) && !FORWARD_LOOKING_FAMILIES.anthropic.has(f), + ); + if (usable.length < count) { + throw new Error( + `need ${count} anthropic families that are neither forward-looking nor already recorded ` + + `as deprecated, but only ${usable.length} remain (${usable.join(", ")}). These fixtures ` + + `need families both detectors will actually report; pick a different provider rather ` + + `than deleting the assertion.`, + ); + } + return usable.slice(0, count); +} /** * The two families to drop from a live listing so the resulting `missing` set @@ -234,13 +263,20 @@ describe("drift-sync mirror ≡ text-drift.ts source (classification equivalence // faithful for an ordinary retirement. const anthropicProvider: Provider = "anthropic"; const allAnthropic = [...includeFamilies.anthropic]; - const genuinelyRetired = "claude-3-opus"; + const [genuinelyRetired] = unrecordedAnthropicFamilies(1); const liveFamiliesList = allAnthropic.filter((f) => f !== genuinelyRetired); const liveIds = [...liveFamiliesList, ...liveFamiliesList.map((f) => `${f}-2025-01-01`)]; - expect(detectDeprecatedFamiliesForSync(liveIds, anthropicProvider)).toEqual( - detectDeprecatedFamilies(liveIds, anthropicProvider), - ); + const syncResult = detectDeprecatedFamiliesForSync(liveIds, anthropicProvider); + expect(syncResult).toEqual(detectDeprecatedFamilies(liveIds, anthropicProvider)); + // Pin what "identical" means here, so the equivalence cannot be satisfied + // by both copies going equally empty — which is exactly what a ledgered or + // forward-looking stand-in would produce. + expect(syncResult.status).toBe("checked"); + if (syncResult.status !== "checked") { + throw new Error("expected 'checked' for a listing well above the fail-closed floor"); + } + expect(syncResult.candidates.map((c) => c.family)).toContain(genuinelyRetired); }); it("detectDeprecatedFamilies: the sync mirror's forward-looking-family exclusion is an INTENTIONAL, bounded divergence from the canonical detector", () => { @@ -266,7 +302,7 @@ describe("drift-sync mirror ≡ text-drift.ts source (classification equivalence // check whether that filter was accidentally removed. const anthropicProvider: Provider = "anthropic"; const allAnthropic = [...includeFamilies.anthropic]; - const genuinelyRetired = "claude-3-opus"; + const [genuinelyRetired] = unrecordedAnthropicFamilies(1); const forwardLooking = "claude-fable-5"; const liveFamiliesList = allAnthropic.filter( (f) => f !== genuinelyRetired && f !== forwardLooking, @@ -317,15 +353,19 @@ describe("drift-sync mirror ≡ text-drift.ts source (classification equivalence // whole defect this filter removes. const anthropicProvider: Provider = "anthropic"; const allAnthropic = [...includeFamilies.anthropic]; - const alreadyRecorded = "claude-3-5-sonnet"; - const freshlyRetired = "claude-3-opus"; + const [alreadyRecorded, freshlyRetired] = unrecordedAnthropicFamilies(2); const liveFamiliesList = allAnthropic.filter( (f) => f !== alreadyRecorded && f !== freshlyRetired && f !== "claude-fable-5", ); const liveIds = [...liveFamiliesList, ...liveFamiliesList.map((f) => `${f}-2025-01-01`)]; const canonicalResult = detectDeprecatedFamilies(liveIds, anthropicProvider); - const withoutLedger = detectDeprecatedFamiliesForSync(liveIds, anthropicProvider); + // Both mirror calls drive `isRecorded` explicitly, so the REAL ledger (which + // grows on its own) cannot decide the outcome either way: the only thing + // separating these two results is the predicate. + const withoutLedger = detectDeprecatedFamiliesForSync(liveIds, anthropicProvider, { + isRecorded: () => false, + }); const withLedger = detectDeprecatedFamiliesForSync(liveIds, anthropicProvider, { isRecorded: (family) => family === alreadyRecorded, }); From 0e6c0ca6196e16ae7421404643d3bedbfaa4fcfc Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 7 Aug 2026 15:35:03 -0700 Subject: [PATCH 4/4] docs(drift-sync): correct the gate's description of the edits it gates --- scripts/drift-sync-check.ts | 5 +++-- scripts/drift-sync.ts | 11 ++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/drift-sync-check.ts b/scripts/drift-sync-check.ts index 80768d61..344f2455 100644 --- a/scripts/drift-sync-check.ts +++ b/scripts/drift-sync-check.ts @@ -5,8 +5,9 @@ * anti-cheat predicate (`drift-success-predicate.ts`, spec §3/§6). * * `drift-sync.ts` (C2) never freewrites a fix — it only ever performs one of - * two deterministic, data-only edits: (a) a zero-reference deprecation - * removal in `model-registry.ts`, or (b) drop a needs-human dedup note file + * two deterministic, data-only edits: (a) append one comment-marked family + * literal to a `model-registry.ts` set (a recorded deprecation, or a + * human-approved new family), or (b) drop a needs-human dedup note file * under `drift-proposals/`. Because the SYNC path can no longer produce an * arbitrary diff, verifying it is "real" no longer needs adversarial-intent * modeling or TS-diff parsing (the predicate's whole reason for being 916 diff --git a/scripts/drift-sync.ts b/scripts/drift-sync.ts index 83bd744d..7ad82f33 100644 --- a/scripts/drift-sync.ts +++ b/scripts/drift-sync.ts @@ -1008,8 +1008,8 @@ export function runDriftSyncCore( // stayed on the data-only surface and left the frozen classification logic // intact. Gate-3 (the live re-collect) only makes sense when this run CLAIMS // to have fully resolved the drift — i.e. no family was simultaneously - // deferred to a human. In a mixed run (a valid removal PLUS a family routed - // to a human), the re-collect would (correctly) still see that deferred + // deferred to a human. In a mixed run (a valid registry edit PLUS a family + // routed to a human), the re-collect would (correctly) still see that deferred // family as residual drift and would wrongly revert the valid edit (D-M1, // mixed-run leg), so skip gate-3 and report NEEDS_HUMAN with the edit kept. deps.writeRegistrySource(registrySource); @@ -1266,14 +1266,15 @@ const REAL_SYNC_CORE_DEPS: SyncCoreDeps = { * the branch name. * * WHY not key on the committed note-file paths alone (the workflow's older - * approach): the D-M1 "mixed run" (a mechanical registry removal of family X + * approach): the D-M1 "mixed run" (a mechanical registry edit for family X * committed the SAME run a *different* family Y is deferred to a human, Y's * note already sitting on `main` from a prior run) commits ONLY the registry * edit — no `drift-proposals/*` file — so a note-path key is EMPTY and the * dedup is bypassed, re-opening a near-identical PR every daily cron run * (unbounded PR-spam). The outcome-derived key is non-empty here (it carries - * both `removed:openai/X` and `needs-human-…:gemini/Y`) and identical on every - * re-fire, so the workflow can find the already-open PR and skip. + * both `deprecation-recorded:anthropic/X` and `needs-human-…:gemini/Y`) and + * identical on every re-fire, so the workflow can find the already-open PR and + * skip. * * A 16-hex-char SHA-256 prefix is used as the marker token: fixed-length, so * two distinct changesets can never be substring-confused in the PR-body