From 45aad42aa89936c2e15db0971c285452af45a255 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 4 Aug 2026 08:47:50 -0700 Subject: [PATCH 1/3] test(drift): make DATA_FROZEN members lazy, and pin FORWARD_LOOKING_FAMILIES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `FROZEN.source` was made a thunk precisely so that one renamed or reshaped surface fails its OWN case instead of throwing during collection and taking the file down. `DATA_FROZEN.members` twelve lines below kept the module-scope evaluation: vitest resolves a renamed export to `undefined` rather than failing the import, so a single renamed registry export made `[...undefined]` throw at collection and reported the WHOLE file as zero tests — all twelve source pins, every behavioural anchor, and all nine data pins. Verified: renaming `gaRealtimeModels` took the file from 29 passing to `Tests no tests`; with `members` lazy the same rename fails only its own pin (plus the `detectVoiceModelDrift` source pin, whose body genuinely references it) and the other 30 still run and pass. Also pins `FORWARD_LOOKING_FAMILIES` (deprecation-detector.ts) per provider, including the two empty sets. A family listed there is dropped from the sync mirror's deprecation candidates entirely — no removal proposal, no needs-human note — so adding an id was a one-line way to stop reporting a genuinely retired family, and no guard watched it. Mutation-checked: seeding the openai set reddens its pin and nothing else. --- src/__tests__/drift/logic-pin.test.ts | 56 ++++++++++++++++++++------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/src/__tests__/drift/logic-pin.test.ts b/src/__tests__/drift/logic-pin.test.ts index 01b889e1..982f81d4 100644 --- a/src/__tests__/drift/logic-pin.test.ts +++ b/src/__tests__/drift/logic-pin.test.ts @@ -28,8 +28,9 @@ * detectVoiceModelDrift * * Frozen surfaces — DATA, membership-pinned in `DATA_FROZEN`: - * model-registry.ts — includeFamilies and excludeFamilies, per provider - * voice-models.ts — knownVoiceModelFamilies, gaRealtimeModels + * model-registry.ts — includeFamilies and excludeFamilies, per provider + * voice-models.ts — knownVoiceModelFamilies, gaRealtimeModels + * deprecation-detector.ts — FORWARD_LOOKING_FAMILIES, per provider * * Keep both lists exact. A guard whose own inventory is wrong is a false * record: it invites the reader to assume a surface is covered when it is not. @@ -48,6 +49,7 @@ import { excludeFamilies, } from "./model-registry.js"; import { normalizeModelFamily } from "./model-family.js"; +import { FORWARD_LOOKING_FAMILIES } from "./deprecation-detector.js"; import { isVoiceModelId, knownVoiceModelFamilies, @@ -392,30 +394,38 @@ describe("classification-logic checksum freeze (Phase-0 anti-silence guard)", () * DO NOT "fix" a red pin by blindly pasting the new hash. A red pin means the * classified-family DATA moved — confirm the move is intended and reviewed * BEFORE updating the pin. + * + * `members` is a THUNK, for exactly the reason `FROZEN.source` is one: every + * entry here reads an imported binding, and vitest resolves a renamed export to + * `undefined` rather than failing the import, so a module-scope `[...binding]` + * threw during COLLECTION and took the whole FILE down — all twelve source pins, + * every behavioural anchor, and every other data pin reported as zero tests. + * Reading the set inside the `it` keeps the blast radius to the one entry whose + * data actually moved. */ -const DATA_FROZEN: Record = { +const DATA_FROZEN: Record string[]; pin: string }> = { "includeFamilies.openai": { - members: [...includeFamilies.openai].sort(), + members: () => [...includeFamilies.openai].sort(), pin: "802989cfefe27838cf7303ac905dbb5fb6641e9fb859924834422b86cce8fb9c", }, "includeFamilies.anthropic": { - members: [...includeFamilies.anthropic].sort(), + members: () => [...includeFamilies.anthropic].sort(), pin: "ab79ff332fadeff93c2678ebe3e0af7a6280ce6f0deb4694228e316944dfeb74", }, "includeFamilies.gemini": { - members: [...includeFamilies.gemini].sort(), + members: () => [...includeFamilies.gemini].sort(), pin: "c2e2c56b8f8d5fc56152b4633e7d3782e95b7eeb9bc123da71f00e884a54a743", }, "excludeFamilies.openai": { - members: [...excludeFamilies.openai].sort(), + members: () => [...excludeFamilies.openai].sort(), pin: "e4484f780a6a64928a54004a52420969c28d92c861272b10fffbbc7f96625f76", }, "excludeFamilies.anthropic": { - members: [...excludeFamilies.anthropic].sort(), + members: () => [...excludeFamilies.anthropic].sort(), pin: "03ccd17333fe45b1fc01d2dc79c4337930204e178205b896aef7000d4378d79f", }, "excludeFamilies.gemini": { - members: [...excludeFamilies.gemini].sort(), + members: () => [...excludeFamilies.gemini].sort(), pin: "c95dedab7588212bbba0bf9ab6434bfd43a449adbe7b35f81f48271ee849a9c2", }, // The realtime canary's seed sets, previously pinned NOWHERE. An edit to @@ -426,21 +436,41 @@ const DATA_FROZEN: Record = { // Both are one-line silencing edits, which is exactly what this file exists // to make impossible. knownVoiceModelFamilies: { - members: [...knownVoiceModelFamilies].sort(), + members: () => [...knownVoiceModelFamilies].sort(), pin: "3897b6bd6fc370ef14f080f4717dde653f2ae6029501d55c4cbda89523f9f3c6", }, gaRealtimeModels: { - members: [...gaRealtimeModels].sort(), + members: () => [...gaRealtimeModels].sort(), pin: "deea068b1a4498d811f0b5399fd18db0f4d20561a63325f4daaddccf1e4e9410", }, + // The forward-looking allowlist. A family listed here is dropped from the sync + // mirror's deprecation candidates ENTIRELY — no removal proposal, no + // needs-human note (see `isForwardLookingFamily` in deprecation-detector.ts). + // So adding an id here is a one-line way to make a genuinely retired family + // stop being reported, and nothing pinned it. Pinned per provider (including + // the two EMPTY sets) so quietly opening a lane for openai or gemini is a + // reviewed re-pin too. + "FORWARD_LOOKING_FAMILIES.openai": { + members: () => [...FORWARD_LOOKING_FAMILIES.openai].sort(), + pin: "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + }, + "FORWARD_LOOKING_FAMILIES.anthropic": { + members: () => [...FORWARD_LOOKING_FAMILIES.anthropic].sort(), + pin: "cd52fbaa5591c6e37db66ff915493fbda50450b254666a47b2c6f9102bdc169b", + }, + "FORWARD_LOOKING_FAMILIES.gemini": { + members: () => [...FORWARD_LOOKING_FAMILIES.gemini].sort(), + pin: "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + }, }; describe("classification-data membership freeze (include/exclude families + voice seed sets)", () => { for (const [name, { members, pin }] of Object.entries(DATA_FROZEN)) { it(`freezes ${name} membership`, () => { + const frozen = members(); expect( - sha256(JSON.stringify(members)), - `Frozen data set "${name}" membership changed (now: ${JSON.stringify(members)}). If ` + + sha256(JSON.stringify(frozen)), + `Frozen data set "${name}" membership changed (now: ${JSON.stringify(frozen)}). If ` + `this is a deliberate, reviewed addition/removal of a classified family, update its ` + `pin here; if not, it is a silent canary-silencing edit and must be reverted.`, ).toBe(pin); From 37c808ea0691253668f904a246aad00efbfffd4a Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 4 Aug 2026 08:52:00 -0700 Subject: [PATCH 2/3] test(drift): actually catch the deprecation mirror's `.sort()`, and correct the coverage claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are TWO `.sort()`s in the drift-sync mirror, one per mirrored function, and the guard only covered one of them. Every deprecation fixture produced at most ONE missing family, and the sole two-candidate case asserted with order-INSENSITIVE `toContain`, so dropping `detectDeprecatedFamiliesForSync`'s `.sort()` left this file 6/6 green and `drift-sync-core.test.ts` 34/34 green. The docstring's claim to catch "all three named divergences" was therefore true of the unclassified-families sort only. Adds a fixture that removes two families whose registry INSERTION order is not alphabetical, so sorted and unsorted `missing` differ, and asserts the whole result object with order-sensitive `toEqual` plus an absolute order pin (so dropping the sort from BOTH copies at once, which keeps them equivalent, still reddens). The pair is derived from the registry rather than hard-coded, and `firstUnsortedPair` throws if the registry ever becomes fully sorted rather than letting the fixture go quietly vacuous. Re-verified the other two named divergences DO red: dropping the `NON_MODEL_TOKENS` check and weakening the fail-closed floor each fail. Narrowed the token-fixture comment to what it actually covers — the line as a whole, not its `|| NON_MODEL_TOKENS.has(id)` half, which is unreachable while the sole token normalizes to itself. Also: exercise `gemini` (the two exclude-by-rule lanes had no coverage on either copy), name the fixture in the loop's assertion message, drop the re-declared local `Provider` in favour of the mirror's own exported type, and make the "healthy listing" case honest — its `missing` set is empty, so it never reaches `isFamilyStillReferenced`; the new two-missing-families fixture is what exercises the real reference scan. --- .../drift-sync-mirror-equivalence.test.ts | 130 ++++++++++++++++-- 1 file changed, 121 insertions(+), 9 deletions(-) diff --git a/src/__tests__/drift-sync-mirror-equivalence.test.ts b/src/__tests__/drift-sync-mirror-equivalence.test.ts index 694866c0..2bbb7fde 100644 --- a/src/__tests__/drift-sync-mirror-equivalence.test.ts +++ b/src/__tests__/drift-sync-mirror-equivalence.test.ts @@ -32,21 +32,57 @@ * the two behaviours, and originally only the fail-closed floor did: no fixture * contained a `NON_MODEL_TOKENS` member, and every fixture's output happened to * already be in insertion order, so dropping the mirror's token check or its - * `.sort()` left all of these tests green. Both are now covered by dedicated - * fixtures (see the comments inline). Do not "simplify" the fixture list by + * `.sort()` left all of these tests green. Do not "simplify" the fixture list by * removing cases that look like near-duplicates of each other — verify first * that the mutation each one exists to catch still reddens without it. + * + * NOTE THAT THERE ARE TWO `.sort()`s, one per mirrored function, and each needs + * its OWN order-distinguishing fixture. Covering `unclassifiedFamiliesForSync`'s + * sort did not cover `detectDeprecatedFamiliesForSync`'s: every deprecation + * fixture here produced at most ONE candidate, and the sole two-candidate case + * asserted with order-INSENSITIVE `toContain`, so dropping the deprecation + * copy's `.sort()` left this whole guard (and `drift-sync-core.test.ts`) green. + * Each mutation below was re-run against this file after the fixture was added: + * dropping either `.sort()`, the `NON_MODEL_TOKENS` check, or the fail-closed + * floor now reddens. */ import { describe, it, expect } from "vitest"; import { detectDeprecatedFamiliesForSync, unclassifiedFamiliesForSync, + type Provider, } from "../../scripts/drift-sync.js"; import { detectDeprecatedFamilies, unclassifiedFamilies } from "./drift/text-drift.js"; import { includeFamilies } from "./drift/model-registry.js"; -type Provider = "openai" | "anthropic" | "gemini"; +/** + * The two families to drop from a live listing so the resulting `missing` set + * distinguishes a SORTED copy from an unsorted one: the first pair, in + * `includeFamilies[provider]`'s insertion order, that is NOT already in + * alphabetical order. Derived rather than hard-coded so a future reordering of + * the registry cannot quietly turn the fixture that uses it into a no-op. + * + * Throws if the registry happens to be fully sorted, because then no + * two-family removal can tell the two orders apart and the fixture would be + * vacuous — a loud failure is the point (`String.sort()`'s default comparator + * and `>` on strings both order by UTF-16 code unit, so this pair really does + * invert under `.sort()`). + */ +function firstUnsortedPair(provider: Provider): [string, string] { + const insertionOrder = [...includeFamilies[provider]]; + for (let i = 0; i < insertionOrder.length; i++) { + for (let j = i + 1; j < insertionOrder.length; j++) { + if (insertionOrder[i] > insertionOrder[j]) return [insertionOrder[i], insertionOrder[j]]; + } + } + throw new Error( + `includeFamilies.${provider} is in strict alphabetical order, so removing two of its ` + + `families can no longer distinguish a sorted candidate list from an unsorted one. ` + + `This fixture would silently stop guarding the mirror's \`.sort()\` — add an ` + + `explicitly out-of-order case instead of deleting it.`, + ); +} describe("drift-sync mirror ≡ text-drift.ts source (classification equivalence)", () => { const provider: Provider = "openai"; @@ -67,6 +103,14 @@ describe("drift-sync mirror ≡ text-drift.ts source (classification equivalence // its docstring claims to protect that check. `gemini-interactions` is the // sole token and is NOT otherwise classified for openai, so with the check // this listing yields [] and without it yields ["gemini-interactions"]. + // + // Scope, precisely: this covers dropping the token check as a WHOLE. It + // does NOT distinguish the line's two clauses — `gemini-interactions` + // normalizes to itself, so `has(family)` alone suppresses it and the + // `|| NON_MODEL_TOKENS.has(id)` half is unexercised (verified: reducing the + // mirror's line to `has(family)` keeps this file green). That half only + // becomes reachable if a future token normalizes to something OTHER than + // itself, at which point a fixture carrying that raw id belongs here too. ["gpt-4o", "gemini-interactions"], // Exercises the `.sort()` specifically. Every fixture above happens to // produce output that is ALREADY in insertion order, so dropping the @@ -78,9 +122,32 @@ describe("drift-sync mirror ≡ text-drift.ts source (classification equivalence ["gpt-omega", "gpt-alpha"], ]; for (const modelIds of fixtures) { - expect(unclassifiedFamiliesForSync(modelIds, provider)).toEqual( - unclassifiedFamilies(modelIds, provider), - ); + // Name the fixture in the message: seven listings share this one `it`, and + // a bare array-mismatch diff does not say WHICH one diverged. + expect( + unclassifiedFamiliesForSync(modelIds, provider), + `mirror/source disagreed on fixture ${JSON.stringify(modelIds)}`, + ).toEqual(unclassifiedFamilies(modelIds, provider)); + } + }); + + it("unclassifiedFamilies (gemini): the exclude-by-RULE lanes classify identically", () => { + // The `provider` const above is `openai`, whose classification never reaches + // the PREVIEW_FAMILY / GEMMA_FAMILY exclude-by-rule patterns in practice — + // so nothing here exercised the two rule lanes, on either copy. Gemini is + // the provider those rules exist for. + const geminiProvider: Provider = "gemini"; + const fixtures: string[][] = [ + ["gemini-9-pro-preview"], // PREVIEW_FAMILY exclude-by-rule + ["gemma-4-31b-it"], // GEMMA_FAMILY exclude-by-rule + ["gemini-interactions"], // the sole NON_MODEL_TOKEN, under its own provider + [...includeFamilies.gemini, "gemini-omega", "gemini-alpha"], // reverse-alphabetical unknowns + ]; + for (const modelIds of fixtures) { + expect( + unclassifiedFamiliesForSync(modelIds, geminiProvider), + `mirror/source disagreed on gemini fixture ${JSON.stringify(modelIds)}`, + ).toEqual(unclassifiedFamilies(modelIds, geminiProvider)); } }); @@ -101,12 +168,57 @@ describe("drift-sync mirror ≡ text-drift.ts source (classification equivalence ).toEqual(detectDeprecatedFamilies(liveIds, provider, { isReferenced: () => true })); }); - it("detectDeprecatedFamilies: a healthy listing missing nothing classifies identically (real isFamilyStillReferenced)", () => { + it("detectDeprecatedFamilies: a healthy listing missing nothing classifies identically, with zero candidates", () => { const allFamilies = [...includeFamilies.openai]; const liveIds = [...allFamilies, ...allFamilies.map((f) => `${f}-2025-01-01`)]; - expect(detectDeprecatedFamiliesForSync(liveIds, provider)).toEqual( - detectDeprecatedFamilies(liveIds, provider), + const syncResult = detectDeprecatedFamiliesForSync(liveIds, provider); + expect(syncResult).toEqual(detectDeprecatedFamilies(liveIds, provider)); + // Pin what "healthy" means here, so the equivalence above cannot be + // satisfied by both copies going equally wrong (e.g. both skipping). This + // fixture's `missing` set is EMPTY, so it does not reach + // `isFamilyStillReferenced` at all — the real reference scan is exercised by + // the two-missing-families fixture below, which passes no `isReferenced`. + expect(syncResult).toEqual({ status: "checked", candidates: [] }); + }); + + it("detectDeprecatedFamilies: two families missing in a non-alphabetical registry order classify identically (pins the mirror's `.sort()`)", () => { + // The deprecation lane's own `.sort()` guard. Every other fixture in this + // file yields at most ONE missing family — and one candidate is trivially + // "in order" — while the forward-looking divergence test below yields two + // but asserts with order-INSENSITIVE `toContain`. So dropping + // `detectDeprecatedFamiliesForSync`'s `.sort()` left every test here green, + // even though this file's docstring claimed the sort was covered (the + // fixture that covers it guards `unclassifiedFamiliesForSync`'s sort, a + // DIFFERENT `.sort()` in a different function). + // + // Both copies build `missing` from `[...includeFamilies[provider]]`, i.e. in + // registry INSERTION order, then sort. Removing a pair whose insertion order + // is not alphabetical therefore makes sorted and unsorted output differ, and + // `toEqual` on the result object is order-sensitive. + const [insertedFirst, alphabeticallyFirst] = firstUnsortedPair(provider); + const liveFamilies = [...includeFamilies[provider]].filter( + (f) => f !== insertedFirst && f !== alphabeticallyFirst, ); + const liveIds = [...liveFamilies, ...liveFamilies.map((f) => `${f}-2025-01-01`)]; + + const syncResult = detectDeprecatedFamiliesForSync(liveIds, provider); + const canonicalResult = detectDeprecatedFamilies(liveIds, provider); + + // Order-sensitive: a mismatch in candidate ORDER alone fails here. + expect(syncResult).toEqual(canonicalResult); + + // …and pin the order absolutely, not just relatively, so dropping the + // `.sort()` from BOTH copies at once (which keeps them equivalent) still + // reddens. Sorted order is the reverse of the insertion order of this pair, + // by construction of `firstUnsortedPair`. + expect(syncResult.status).toBe("checked"); + if (syncResult.status !== "checked") { + throw new Error("expected 'checked' for a listing well above the fail-closed floor"); + } + expect(syncResult.candidates.map((c) => c.family)).toEqual([ + alphabeticallyFirst, + insertedFirst, + ]); }); it("detectDeprecatedFamilies (anthropic): a non-forward-looking retirement classifies identically to the canonical mirror", () => { From 4f845ae54453da5bb8213b9e5323d11cd72a1f70 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 4 Aug 2026 08:53:12 -0700 Subject: [PATCH 3/3] test(drift): make the exclude-family normalization test falsifiable, and cover the real ada-002 id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems in the enumerate-every-exclude-family case. It fed in the BARE registry keys alongside the dated ones, and those are `excludeFamilies.openai` by definition, so that half of the payload could never contribute a result. And `toEqual([])` is precisely what a neutered `unclassifiedFamilies` (`return []`) produces, so the assertion passed under the mutation it looks like it guards. Now only the dated ids go in, plus a negative control of the same shape that MUST report. Verified: with `unclassifiedFamilies` neutered to `return []` this test reddens, where before it stayed green. Also adds the id form OpenAI actually lists. The loop derives ids from the registry keys, which are seeded through `normalizeModelFamily`, so `text-embedding-ada-002` is stored as `text-embedding-ada` and the id the loop generated for it was `text-embedding-ada-2026-07-28` — a string that does not exist. The real bare and dated forms, which need the `-002` build tag and the date to normalize in composition, were exercised nowhere. --- src/__tests__/drift/text-drift.test.ts | 36 +++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/__tests__/drift/text-drift.test.ts b/src/__tests__/drift/text-drift.test.ts index 41925c30..7bb39e51 100644 --- a/src/__tests__/drift/text-drift.test.ts +++ b/src/__tests__/drift/text-drift.test.ts @@ -12,6 +12,7 @@ import { describe, it, expect } from "vitest"; import { isClassifiedFamily, excludeFamilies } from "./model-registry.js"; +import { normalizeModelFamily } from "./model-family.js"; import { unclassifiedFamilies } from "./text-drift.js"; // --------------------------------------------------------------------------- @@ -88,8 +89,41 @@ describe("openai transcription line is classified as EXCLUDED (PR #343)", () => // nothing proved a DATED snapshot of them (the form the live listing // actually carries) collapses back onto the excluded key instead of // false-positiving as a new family. + // + // Only the DATED ids are fed in. Including the bare keys added nothing: they + // ARE `excludeFamilies.openai`, so `isClassifiedFamily` returns true for them + // by definition and that half of the payload could never contribute a + // result. const bare = [...excludeFamilies.openai]; const dated = bare.map((family) => `${family}-2026-07-28`); - expect(unclassifiedFamilies([...bare, ...dated], "openai")).toEqual([]); + expect(unclassifiedFamilies(dated, "openai")).toEqual([]); + + // NEGATIVE CONTROL, in the same test. `toEqual([])` on its own is exactly + // what a neutered `unclassifiedFamilies` (`return []`) also produces, so the + // assertion above is only meaningful alongside a payload of the same shape + // that MUST report. A dated id on an unclassified family reports its family. + expect(unclassifiedFamilies(["gpt-nonexistent-family-2026-07-28"], "openai")).toEqual([ + "gpt-nonexistent-family", + ]); + }); + + it("the real text-embedding-ada-002 id form is excluded, bare and dated", () => { + // The loop above derives its ids from the REGISTRY KEYS, which are seeded + // through `normalizeModelFamily` — so the `text-embedding-ada-002` entry is + // stored as `text-embedding-ada` (the `-002` reads as a build tag) and the id + // the loop generates for it is `text-embedding-ada-2026-07-28`, a string + // OpenAI never lists. The real id, and its dated form, were exercised + // nowhere: the two normalization steps (`-002` build tag, then the date) have + // to compose for the live listing's actual shape to classify. + expect(normalizeModelFamily("text-embedding-ada-002", "openai")).toBe("text-embedding-ada"); + expect(normalizeModelFamily("text-embedding-ada-002-2026-07-28", "openai")).toBe( + "text-embedding-ada", + ); + expect( + unclassifiedFamilies( + ["text-embedding-ada-002", "text-embedding-ada-002-2026-07-28"], + "openai", + ), + ).toEqual([]); }); });