From 1c4ab4eb5dffee4f582341e9817c139ebdf7aff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AE=B7=E4=BA=AE=E8=BE=89?= Date: Sat, 8 Aug 2026 11:37:46 +0000 Subject: [PATCH] test(scripts): assert every peer-line restatement against its own manifest (#3717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #3710 ruling B was "keep and pin": narrow the prose, then ADD A TEST asserting each README's peer restatement equals its own `peerDependencies` verbatim. Only the first half landed — the `why` strings kept saying "re-verify with a one-line read of the manifest", i.e. a human. This replaces that sentence with a run. For every `restatement` entry the assertion resolves the literal back to its line via the same scan the ratchet judges, parses the dep and the FULL range off that line, reads the manifest beside the README, and demands the ranges match. 12 comparisons today: 11 bullet-spelled READMEs plus react-runtime's prose sentence. The hole this closes is structural, not laziness. The inventory key is the matched literal and a match stops at the FIRST version token, so a key recorded for `^18.0.0 || ^19.0.0` is only `^18.0.0`. Widen a manifest to a third arm and the literal, the entry and both directions of the ratchet are untouched and green over a README that now misstates the range. Reverse-verified in that exact direction: perturbing a README's SECOND arm fails this assertion alone (1 failed, 9 passed, both ratchet tests green), and perturbing the manifest instead fails it identically — #3690's shape, which the ratchet could only record as `stale` after a human noticed. Two boundaries, each pinned by its own test: - Peer lines only. plugin-chatbot restates the MAJOR of its own dependencies["@ai-sdk/react"], not a peer range, so it carries `notAPeerRestatement` and is skipped BY NAME — one rule judges one kind of fact. The opt-out is not an escape hatch: deleting it turns the assertion red ("NOTHING was compared"), and attaching it to a live peer entry turns the closure test red as a stale opt-out suppressing a real check. Both verified. - Equality is verbatim up to whitespace INSIDE the range, because react-runtime's README writes `react >= 18` where its manifest writes `>=18` — the same single comparator, so byte-strict equality would demand a cosmetic edit or lost coverage in exchange for nothing. Every drift shape survives the normalisation and is pinned negatively. No changeset: scripts-test-only, same as PR #3711 which added this file. Out-of-scope finding filed as #3750 — the peer BLOCK is wider than the ledger, and 9 of its other 16 lines disagree with their manifests, including a live drift (`components` README says tailwindcss `^3.0.0`, manifest says `^4.2.1`) that neither the ratchet nor this assertion can see. No README or manifest touched here. Fixes #3717 Co-Authored-By: Claude --- scripts/__tests__/doc-version-claims.test.ts | 383 ++++++++++++++++++- 1 file changed, 373 insertions(+), 10 deletions(-) diff --git a/scripts/__tests__/doc-version-claims.test.ts b/scripts/__tests__/doc-version-claims.test.ts index 2ce0fa38f..9d6982159 100644 --- a/scripts/__tests__/doc-version-claims.test.ts +++ b/scripts/__tests__/doc-version-claims.test.ts @@ -17,10 +17,11 @@ import { fileURLToPath } from 'node:url'; * * ## What this gate actually promises, and what it does not * - * It is a RATCHET, not a correctness check. It cannot know whether "Tailwind CSS v3.3+" - * is true — that would need a per-claim anchor, and most claims here have none. What it - * can decide is the EVENT: a version literal appeared in a doc surface and nobody wrote - * down why. Every literal on the two scanned surfaces is either + * It is a RATCHET first, and a correctness check for exactly ONE class (the section after + * next). It cannot know whether "Tailwind CSS v3.3+" is true — that would need a per-claim + * anchor, and most claims here have none. What it can decide is the EVENT: a version + * literal appeared in a doc surface and nobody wrote down why. Every literal on the two + * scanned surfaces is either * * - structurally exempt, because it sits under a version heading (release notes are * history: `## v3.3.0` sections describe what was true at that release and must stay @@ -36,6 +37,43 @@ import { fileURLToPath } from 'node:url'; * never rot into a permanent hole (same stance as `DOCUMENTATION_EXEMPT` in * `ci-cd-pipeline-doc.test.ts`). * + * ## The one class this file ASSERTS instead of recording (objectui#3717) + * + * The paragraph above holds for four of the five classes below. `restatement` is the + * exception, and it can be one for a structural reason the others lack: a README restating + * its OWN peer range has a machine-readable anchor lying next to it. Every peer-line + * restatement entry is therefore resolved to the line it was matched on, the dep name and + * the FULL range are parsed off that line, the manifest beside that README is read, and + * the two ranges must be equal. That is `describe('the peer-line assertion')` below, and + * it is the whole of objectui#3710's ruling B — which landed only its first half, the + * prose narrowing; the `why` strings kept saying "re-verify with a one-line read of the + * manifest", i.e. a human, until this test replaced that sentence with a run. + * + * The hole it closes is one the ratchet is STRUCTURALLY blind to, not merely lazy about. + * The inventory key is the matched literal, and a match stops at the FIRST version token: + * the key for a line reading `^18.0.0 || ^19.0.0` records only `^18.0.0`. So widen a + * manifest to a third arm, or narrow it to one, and the literal is untouched, the entry is + * untouched, and both directions of the ratchet report green over a README that now + * misstates the range. objectui#3690 was exactly that shape, found by a human census + * (there the README was right and the MANIFEST lagged it — the assertion is symmetric, it + * names the disagreement, not the guilty side). objectui#3741, still open, proposes + * narrowing react-runtime's manifest range; the day someone does, this test goes red until + * that README follows. + * + * Two boundaries, both deliberate, both pinned below: + * + * - PEER lines only. plugin-chatbot's entry restates the MAJOR of its own + * dependencies["@ai-sdk/react"], which is not a peer range and not verbatim, so it + * carries `notAPeerRestatement` and is skipped BY NAME. One rule judges one kind of + * fact: a rule mixing verbatim equality with major-compatibility would leave the next + * reader unable to say what its red means. + * - Whitespace inside a range is insignificant; nothing else is. See `sameRange`. + * + * Still not promised, stated so it is not mistaken for covered: a literal with no + * same-package anchor (`unanchored`, `sample`) is recorded and not checked, and the peer + * BLOCK is judged only where the ledger has an entry — narrower than the block itself, + * whose other 16 lines include a live drift. Measured and filed as objectui#3750. + * * ## The census that set the design (measured on d46b40324, the merge of PR #3698) * * The dispatch expected the bare-claim count to be zero, since #3688 and #3698 had just @@ -337,11 +375,11 @@ const keyOf = (c: Pick): string => `${c.file} :: ${c.cl * and the one that had already drifted: the two drifted members were * repaired by objectui#3710 and re-filed here as restatements, and a * third joined from `stale` when objectui#3690 widened the manifest its - * README already described. Note what this class does and does not buy - * — the equality it names is re-verified by a HUMAN reading the - * manifest, not by this gate, which only asks whether a literal was - * recorded at all. Pinning README to manifest mechanically is still the - * payoff this class is pointing at. + * README already described. Since objectui#3717 this is the one class + * whose named equality is MACHINE-checked: the peer-line assertion below + * reads the manifest and demands the ranges match. An entry that does + * not restate a peer range cannot join quietly — it must say so in + * `notAPeerRestatement`. * `sample` - illustrative or template content. NOT an assertion about this * repository's versions: a changelog a plugin renders as demo data, or * a package.json skeleton the reader owns after copying it. @@ -364,6 +402,17 @@ interface KnownClaim { claim: string; kind: ClaimKind; why: string; + /** + * Only legal on a `restatement`, and it is how such an entry opts OUT of the verbatim + * peer assertion — carrying the reason, not a boolean, so the opt-out is as documented + * as the entry itself. + * + * The assertion refuses to let this field and real coverage coexist: an entry either + * resolves to a peer statement that is checked, or names why it cannot, never both and + * never neither. So a future `restatement` with no peer anchor cannot drift out of + * coverage by simply not parsing — it goes red until someone writes the sentence. + */ + notAPeerRestatement?: string; } /** @@ -376,7 +425,7 @@ interface KnownClaim { */ const PEER_18_19 = 'react' + TICK + ' ^18.0.0'; const PEER_RESTATEMENT_OK = - 'Restates this package peerDependencies.react verbatim; re-verify with a one-line read of the manifest.'; + 'Restates this package peerDependencies.react verbatim; asserted against the manifest by the peer-line assertion below (objectui#3717), no longer by a human reading it.'; const KNOWN_CLAIMS: KnownClaim[] = [ // --- content/docs ------------------------------------------------------------ @@ -452,9 +501,164 @@ const KNOWN_CLAIMS: KnownClaim[] = [ claim: '@ai-sdk/react' + TICK + ' v4', kind: 'restatement', why: 'Names the major of this package own dependencies["@ai-sdk/react"], which is ^4.0.47. Kept rather than deleted because the reader follows it to the streaming protocol docs; re-verify with a one-line read of the manifest.', + notAPeerRestatement: + 'Restates a dependencies entry, not a peerDependencies range, and restates its MAJOR ("v4" for ^4.0.47) rather than the range verbatim. Both facts put it outside the peer-line assertion, which judges verbatim equality of a peer range and nothing else. Covering it would need a second, weaker rule ("same major"), and one rule that returns two kinds of red is a rule whose failures nobody can read.', }, ]; +/** + * The one file shape with a manifest lying next to it. A `restatement` entry naming any + * other file has nothing to be checked against, and the assertion reports that rather + * than skipping it — a restatement of a manifest that cannot be located is not a + * restatement. + */ +const PACKAGE_README = /^packages\/([^/]+)\/README\.md$/; + +/** Wraps text in the backticks markdown quotes it with, without writing one raw. */ +const ticked = (text: string): string => TICK + text + TICK; + +interface PeerStatement { + dep: string; + range: string; +} + +/** A range must OPEN with an operator or a digit. See `parsePeerStatement`. */ +const RANGE_OPENS = /^[\^~><=\d]/; + +/** + * The dep and the FULL range one README line states as a peer requirement, or null when + * the line states none. Two spellings, both measured on the corpus rather than assumed: + * + * - a bullet in a peer-dependency list — 11 of the 12 entries, one spelling between them: + * - `react` ^18.0.0 || ^19.0.0 + * - one code span read as prose, `packages/react-runtime/README.md` alone: + * `react >= 18` is a peer dependency. + * + * Note what the range is here and is not in the inventory key: the key stops at the first + * version token (`^18.0.0`), this reads the range to end of line (`^18.0.0 || ^19.0.0`). + * That difference is the entire added reach of the assertion. + * + * `RANGE_OPENS` is load-bearing, not tidiness. Without it the bullet shape swallows every + * list item that opens with a backticked identifier — an API bullet parses as dep + * "useChat()" with range "returns a stream", and the manifest lookup then fails for a + * reason that has nothing to do with drift. A gate that goes red for the wrong reason + * teaches people to stop reading it. + */ +function parsePeerStatement(line: string): PeerStatement | null { + const bullet = new RegExp( + '^\\s*[-*]\\s+' + TICK + '([^' + TICK + ']+)' + TICK + '\\s+(\\S.*?)\\s*$', + ).exec(line); + if (bullet !== null && RANGE_OPENS.test(bullet[2])) { + return { dep: bullet[1].trim(), range: bullet[2] }; + } + + const prose = new RegExp( + TICK + '([^' + TICK + '\\s]+)\\s+([^' + TICK + ']+)' + TICK + '\\s+is a peer dependency', + ).exec(line); + if (prose !== null && RANGE_OPENS.test(prose[2].trim())) { + return { dep: prose[1], range: prose[2].trim() }; + } + + return null; +} + +/** + * Range equality: VERBATIM, up to whitespace INSIDE the range. + * + * That one-space normalisation is not a softening, and it exists for a measured reason. + * `packages/react-runtime/README.md` writes `react >= 18` while its manifest writes + * `>=18`. Nothing is drifted there — the two spell the same single comparator and npm + * parses them identically — so byte-strict equality would paint that entry red on a tree + * where nothing is wrong. The only two answers to such a red are to rewrite one side for + * the gate's benefit, or to declare the entry uncovered: a cosmetic edit, or lost + * coverage, in exchange for nothing. + * + * Everything this assertion exists to catch survives, because whitespace is the only + * thing dropped: a bumped major, a `||` arm added or removed, `^` turning into `~`, a + * vanished upper bound all still compare unequal. Pinned by its own test below — a + * normaliser that quietly grew to strip operators would make the whole assertion vacuous + * while every other test in this file stayed green. + */ +const sameRange = (a: string, b: string): boolean => + a.replace(/\s+/g, '') === b.replace(/\s+/g, ''); + +function manifestPeers(pkgDir: string): Record | null { + const abs = path.join(repoRoot, 'packages', pkgDir, 'package.json'); + let raw: string; + try { + raw = fs.readFileSync(abs, 'utf8'); + } catch { + return null; + } + const json: unknown = JSON.parse(raw); + const peers = (json as { peerDependencies?: Record }).peerDependencies; + return peers ?? {}; +} + +const readmeLines = new Map(); +function linesOf(rel: string): string[] { + let cached = readmeLines.get(rel); + if (cached === undefined) { + cached = fs.readFileSync(path.join(repoRoot, rel), 'utf8').split('\n'); + readmeLines.set(rel, cached); + } + return cached; +} + +interface ComparedPeer { + line: number; + dep: string; + readme: string; + /** `undefined` when the manifest does not declare this dep as a peer at all. */ + manifest: string | undefined; +} + +interface PeerCheck { + entry: KnownClaim; + /** Every line carrying this entry's literal that reads as a peer statement. */ + compared: ComparedPeer[]; + /** The entry's file has no manifest beside it. */ + noManifest: boolean; + /** The literal is not in the tree — the downward ratchet above is its single reporter. */ + absent: boolean; +} + +/** + * Resolution goes through `flaggedClaims`, the same scan the ratchet judges, rather than + * re-grepping the READMEs. Two reasons, both about composing with what is already here: + * the scan already turned the whitespace-normalised inventory key back into concrete line + * numbers, and keying off it means "the literal is not in the tree" is the SAME fact in + * both tests, so the two can agree on who reports it. + */ +const claimsByKey = new Map(); +for (const claim of flaggedClaims) { + const bucket = claimsByKey.get(keyOf(claim)); + if (bucket === undefined) claimsByKey.set(keyOf(claim), [claim]); + else bucket.push(claim); +} + +const peerChecks: PeerCheck[] = KNOWN_CLAIMS.filter((e) => e.kind === 'restatement').map((entry) => { + const occurrences = claimsByKey.get(keyOf(entry)) ?? []; + const pkgDir = PACKAGE_README.exec(entry.file)?.[1]; + const peers = pkgDir === undefined ? null : manifestPeers(pkgDir); + const compared: ComparedPeer[] = []; + + if (peers !== null) { + for (const occurrence of occurrences) { + const stated = parsePeerStatement(linesOf(entry.file)[occurrence.line - 1] ?? ''); + if (stated === null) continue; + compared.push({ + line: occurrence.line, + dep: stated.dep, + readme: stated.range, + manifest: peers[stated.dep], + }); + } + } + + return { entry, compared, noManifest: peers === null, absent: occurrences.length === 0 }; +}); + describe('doc version claims - the scan itself', () => { it('reads a plausible corpus, so a broken scan cannot report green', () => { // Every count below is a floor, not the measured value: docs are added and @@ -551,3 +755,162 @@ describe('doc version claims - the ratchet', () => { expect(new Set(keys).size, 'KNOWN_CLAIMS has duplicate entries').toBe(keys.length); }); }); + +describe('doc version claims - the peer-line assertion', () => { + it('pins every peer-line restatement to the range its own manifest declares', () => { + const failures: string[] = []; + + for (const check of peerChecks) { + // Skipped by name, with its reason on the entry. The closure test below is what + // stops this branch from becoming a way out. + if (check.entry.notAPeerRestatement !== undefined) continue; + + // A literal that is no longer in the tree is an inventory defect, and the downward + // ratchet above already names it in its own red. Reporting it here too would read + // as two problems where there is one; the coverage floor at the end of this test is + // what keeps the skip from becoming a hole. + if (check.absent) continue; + + if (check.noManifest) { + failures.push( + `${check.entry.file} :: is not a package README, so there is no manifest beside ` + + `it to restate - reclassify the entry or point it at the package`, + ); + continue; + } + + if (check.compared.length === 0) { + failures.push( + `${check.entry.file} :: the literal ${JSON.stringify(check.entry.claim)} is present, ` + + `but the line carrying it does not read as a peer statement, so NOTHING was ` + + `compared - teach parsePeerStatement the spelling, or set notAPeerRestatement`, + ); + continue; + } + + for (const stated of check.compared) { + if (stated.manifest === undefined) { + failures.push( + `${check.entry.file}:${stated.line} ${stated.dep}: the README states a peer ` + + `range, but this package's peerDependencies does not declare ${stated.dep} at all`, + ); + } else if (!sameRange(stated.readme, stated.manifest)) { + failures.push( + `${check.entry.file}:${stated.line} ${stated.dep}: README says ` + + `${JSON.stringify(stated.readme)}, manifest says ${JSON.stringify(stated.manifest)}`, + ); + } + } + } + + expect( + failures, + `A README restates its own package's peer range and the two no longer agree:\n` + + failures.map((f) => ` - ${f}`).join('\n') + + `\n\nThis assertion does not decide which side is wrong. objectui#3710 narrowed the ` + + `PROSE to the manifest; objectui#3690 widened the MANIFEST to the prose, because ` + + `there the README was right. Read the package and fix the side that is actually ` + + `stale, then run this again.\n\n` + + `Note that the ratchet above cannot see this class of drift: the inventory key stops ` + + `at the first version token, so changing the second arm of a range leaves the key, ` + + `the literal and the entry all untouched and both directions of the ratchet green.`, + ).toEqual([]); + + // Vacuity floor. Everything above is a loop over entries: delete the entries, or break + // the line parser, and the loop reports success over nothing. Measured at 12 comparisons + // on 8ad6070fb (11 bullet-spelled READMEs plus react-runtime's prose sentence); a floor + // rather than a pin, because this list legitimately shrinks when a README stops + // restating its manifest. + const compared = peerChecks.flatMap((c) => c.compared); + expect( + compared.length, + 'the peer-line assertion compared implausibly few lines - the parser or the ' + + 'inventory collapsed, and the assertion is now green over nothing', + ).toBeGreaterThanOrEqual(10); + }); + + it('lets no restatement entry sit outside the assertion without saying so', () => { + // The facts this test owns, none of which the assertion above reports: that an opt-out + // carries a real reason, that it has not gone stale, and that it cannot appear on a + // kind it does not apply to. "Covered by neither" is deliberately left to the + // assertion above, so each defect has exactly one reporter. + for (const check of peerChecks) { + const excluded = check.entry.notAPeerRestatement; + if (excluded === undefined) continue; + + expect( + excluded.length, + `${keyOf(check.entry)} opts out of the peer assertion and must say why, at length`, + ).toBeGreaterThan(25); + + expect( + check.compared, + `${keyOf(check.entry)} carries notAPeerRestatement, but its line DOES read as a peer ` + + `statement now - the opt-out is stale and is suppressing a real check`, + ).toEqual([]); + } + + for (const entry of KNOWN_CLAIMS) { + if (entry.kind === 'restatement') continue; + expect( + entry.notAPeerRestatement, + `${keyOf(entry)} is ${entry.kind}, not a restatement - notAPeerRestatement opts out ` + + `of a check that never applied to it, so it reads as coverage that was never there`, + ).toBeUndefined(); + } + }); + + it('reads a peer statement in the two spellings the corpus uses, and refuses the rest', () => { + expect(parsePeerStatement('- ' + ticked('react') + ' ^18.0.0 || ^19.0.0')).toEqual({ + dep: 'react', + range: '^18.0.0 || ^19.0.0', + }); + expect(parsePeerStatement('- ' + ticked('react-router-dom') + ' ^6.0.0 || ^7.0.0')).toEqual({ + dep: 'react-router-dom', + range: '^6.0.0 || ^7.0.0', + }); + expect(parsePeerStatement(ticked('react >= 18') + ' is a peer dependency.')).toEqual({ + dep: 'react', + range: '>= 18', + }); + + for (const notAPeerLine of [ + // A listed peer with no range: there is nothing to compare, and pretending + // otherwise would compare "" against the manifest and always be red. + '- ' + ticked('@object-ui/core'), + // An API bullet. The shape RANGE_OPENS exists to reject. + '- ' + ticked('useChat()') + ' returns a stream of message parts', + '**Peer Dependencies:**', + // plugin-chatbot's own claim line, which is why that entry is opted out by name. + 'using ' + ticked('@ai-sdk/react') + ' v4 (Vercel UI Message Stream protocol) for streaming,', + ]) { + expect( + parsePeerStatement(notAPeerLine), + `${JSON.stringify(notAPeerLine)} must not read as a peer statement`, + ).toBeNull(); + } + }); + + it('treats whitespace inside a range as insignificant, and nothing else', () => { + // The one pair the normalisation exists for, and the only place in the corpus where + // README and manifest differ by anything at all. + expect(sameRange('>= 18', '>=18')).toBe(true); + expect(sameRange('^18.0.0 || ^19.0.0', '^18.0.0||^19.0.0')).toBe(true); + + for (const [readme, manifest] of [ + // The manifest gained an arm and the README did not follow. This is the objectui#3690 + // shape, and the one the ratchet above is structurally blind to. + ['^18.0.0 || ^19.0.0', '^18.0.0 || ^19.0.0 || ^20.0.0'], + ['^18.0.0 || ^19.0.0', '^19.0.0'], + ['^18.0.0', '~18.0.0'], + ['^18.0.0', '^18.0.1'], + ['>= 18', '>= 19'], + ['>=18', '>18'], + ]) { + expect( + sameRange(readme, manifest), + `${readme} and ${manifest} are different ranges and must not compare equal`, + ).toBe(false); + } + }); +});