From 8d3e42f0ab13d09e271d78f7ee9e2489ef023938 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Tue, 26 May 2026 19:30:57 -0300 Subject: [PATCH 1/2] fix(scripts): flip public-method-coverage to strict-zero gate (SD-673) Removes the ratchet / debt-snapshot machinery from check-public-method- coverage.mjs. The snapshot was already empty (knownUnmet: []) on main; this PR makes re-introducing grandfathering impossible. Scanner changes (check-public-method-coverage.mjs): - Removed SNAPSHOT_PATH, loadSnapshot, writeSnapshot, writeMode, newUnmet, stale, and all snapshot messaging. - main flow now fails directly when unmetNow.length > 0 instead of comparing against a snapshot. - --write rejected with exit 2 and a pointer at the allowlist, so contributors don't accidentally re-introduce grandfathering. - Status line drops 'Tracked as known debt:' and 'Snapshot at:' fields; shows 'Unmet obligations: N' instead. - 'OK' line drops the 'ratchet snapshot in sync' suffix. - Removed writeFileSync from the fs import (no longer used). - Header comment rewritten: 'Strict-zero obligation gate' replaces 'Obligation-based ratchet'; removed the SNAPSHOT DRIFT failure mode and the --write refresh instructions. Kept the allowlist contract description; allowlist is the only escape hatch. Deleted tests/consumer-typecheck/public-method-coverage-debt-snapshot.json. The file was empty ('knownUnmet': []) but kept the ratchet alive as a concept. Removing it makes the strict-zero stance explicit on disk. Kept public-method-coverage-allowlist.cjs as-is. That's not debt; it's the explicit contract for intentionally non-consumer-callable members (8 entries currently). The allowlist contract enforcement (key matches a real member, value is non-empty reason) is unchanged. Docs updated to remove ratchet / debt-snapshot language: - scripts/check-public-contract.mjs: stage header comment and blurb now say 'strict-zero obligation gate' and mention the allowlist as the only escape hatch. - packages/superdoc/scripts/README.md: scanner row now says 'Strict-zero obligation gate' and removes the snapshot / --write references. Verified: - node check-public-method-coverage.mjs -> OK, 72 obligations across 36 members, zero unmet. - node check-public-method-coverage.mjs --write -> rejected, exit 2. - pnpm check:public:superdoc --skip-build -> PASS (11 ran, 1 skipped, 136.2s). --- packages/superdoc/scripts/README.md | 2 +- scripts/check-public-contract.mjs | 30 ++--- .../check-public-method-coverage.mjs | 103 +++++------------- .../public-method-coverage-debt-snapshot.json | 4 - 4 files changed, 45 insertions(+), 94 deletions(-) delete mode 100644 tests/consumer-typecheck/public-method-coverage-debt-snapshot.json diff --git a/packages/superdoc/scripts/README.md b/packages/superdoc/scripts/README.md index f27f6f0332..12a25891a4 100644 --- a/packages/superdoc/scripts/README.md +++ b/packages/superdoc/scripts/README.md @@ -145,7 +145,7 @@ what an actual consumer would see — not the workspace source. | `check-all-public-types-fixture.mjs` | Asserts every type-only root export has an `AssertNotAny` line in `src/all-public-types.ts`. | Derives the expected set from `superdoc-root-classification.json`. | | `package-shape-gate.mjs` | External package-shape linters (publint + attw) against the packed tarball. | Catches condition ordering, masquerading exports, missing field declarations. | | `check-root-classification-closure.mjs` | Asserts no `supported-root` or `legacy-root` export references an `internal-candidate` symbol in its public declared type. | Closure rule from SD-3212. | -| `check-public-method-coverage.mjs` | Obligation-based ratchet over public `SuperDoc` methods + getters. For each member the AST computes which obligations are meaningful (`parameters`, `returns`, or `call`); the gate fails when any required obligation is unsatisfied by a fixture under `src/` AND not on the debt snapshot. Catches the `search(text: string)` regression class — call sites do NOT satisfy `parameters`/`returns` on their own. | Snapshot at `public-method-coverage-debt-snapshot.json`; allowlist at `public-method-coverage-allowlist.cjs` (each entry validated: key must match a real member, value must be a non-empty reason). Refresh with `--write`. | +| `check-public-method-coverage.mjs` | Strict-zero obligation gate over public `SuperDoc` methods + getters. For each member the AST computes which obligations are meaningful (`parameters`, `returns`, or `call`); the gate fails on any unmet obligation. No grandfathered debt snapshot, no `--write`. Catches the `search(text: string)` regression class — call sites do NOT satisfy `parameters`/`returns` on their own. | Allowlist at `public-method-coverage-allowlist.cjs` is the only escape hatch (intentionally non-consumer-callable members; each entry validated: key must match a real member, value must be a non-empty reason). | Six of these run as wrapper stages of `check:public:superdoc`. `public-method-coverage` runs alongside the cheap policy gates diff --git a/scripts/check-public-contract.mjs b/scripts/check-public-contract.mjs index e3515b6242..0580e2a82e 100755 --- a/scripts/check-public-contract.mjs +++ b/scripts/check-public-contract.mjs @@ -47,19 +47,21 @@ * grandfathered baseline); fails * on any type-bearing JSDoc. See * packages/superdoc/scripts/type-hygiene.md. - * 6. public-method-coverage - obligation-based ratchet over + * 6. public-method-coverage - strict-zero obligation gate over * public SuperDoc methods + * getters. For each member the * AST computes which obligations * are meaningful (parameters / - * returns / call); each unmet - * obligation must be on the debt - * snapshot or the gate fails. - * Call sites do NOT satisfy - * parameters/returns obligations - * on their own — that's why - * `search(text: string)` shipped - * under v1 of this gate. + * returns / call); the gate fails + * on any unmet obligation. The + * only escape hatch is the + * public-method-coverage-allowlist + * (intentionally non-consumer- + * callable members). Call sites + * do NOT satisfy parameters/ + * returns obligations on their own + * — that's why `search(text: string)` + * shipped under v1 of this gate. * 7. build - vite build + the postbuild * validator chain * (check-tsconfig-type-surface, @@ -186,11 +188,13 @@ const stages = [ cmd: 'node', args: ['tests/consumer-typecheck/check-public-method-coverage.mjs'], blurb: - 'Obligation-based ratchet over public SuperDoc methods + getters. ' + + 'Strict-zero obligation gate over public SuperDoc methods + getters. ' + 'Each member has computed obligations (parameters / returns / call) ' + - 'that must be satisfied by a typed assertion in a consumer fixture, ' + - 'or be on the debt snapshot. Call sites do NOT satisfy parameters/' + - 'returns on their own (this is why search(text: string) shipped).', + 'that must be satisfied by a typed assertion in a consumer fixture; ' + + 'the gate fails on any unmet obligation. Only escape hatch is the ' + + 'public-method-coverage-allowlist for intentionally non-consumer-callable ' + + 'members. Call sites do NOT satisfy parameters/returns on their own ' + + '(this is why search(text: string) shipped).', }, { name: 'build', diff --git a/tests/consumer-typecheck/check-public-method-coverage.mjs b/tests/consumer-typecheck/check-public-method-coverage.mjs index dd155f47ef..47d9968603 100644 --- a/tests/consumer-typecheck/check-public-method-coverage.mjs +++ b/tests/consumer-typecheck/check-public-method-coverage.mjs @@ -2,11 +2,11 @@ /** * Public-method fixture coverage gate. * - * Obligation-based ratchet over public SuperDoc methods + getters. + * Strict-zero obligation gate over public SuperDoc methods + getters. * For each public member, the gate computes what fixture coverage is - * meaningful (`parameters`, `returns`, or `call`) and fails when any - * required obligation is unmet AND the member is not on the debt - * snapshot. + * meaningful (`parameters`, `returns`, or `call`) and fails on any + * unmet obligation. There is no debt snapshot, no `--write`, and no + * grandfathering. * * Obligations (per member, computed from the AST): * @@ -29,34 +29,23 @@ * Call sites do NOT satisfy parameter or return obligations on their * own (TypeScript would accept a wrong-typed argument if the consumer * matched the signature). This is the central distinction from a - * "mentioned somewhere" ratchet: the gate must catch the + * "mentioned somewhere" gate: the gate must catch the * `search(text: string)` regression class, where a call site * `sd.search('hello')` shipped while `Parameters` * was never asserted. * - * Two failure modes: - * - * 1. RATCHET — A NEW unmet obligation lands (member added, fixture - * removed, or migration narrows a signature) and the obligation - * is not on the debt snapshot. - * 2. SNAPSHOT DRIFT — A snapshot entry is stale (the obligation it - * records is now satisfied). The contributor must run `--write` - * to lock the win. - * - * Refresh the snapshot after intentional changes: - * node tests/consumer-typecheck/check-public-method-coverage.mjs --write - * * Allowlist: `tests/consumer-typecheck/public-method-coverage-allowlist.cjs`. * Use only for members that are intentionally not consumer-callable * (e.g. internal lifecycle relays that escaped `private` for runtime * reasons). Each entry requires (a) a key that matches an actual public * member of `SuperDoc`, and (b) a non-empty string reason. The gate - * validates both. + * validates both. The allowlist is the only escape hatch — there is + * no grandfathered debt snapshot. * * Wrapper stage: `public-method-coverage` in `scripts/check-public-contract.mjs`. */ -import { readFileSync, readdirSync, existsSync, writeFileSync } from 'node:fs'; +import { readFileSync, readdirSync, existsSync } from 'node:fs'; import { dirname, resolve, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createRequire } from 'node:module'; @@ -66,13 +55,22 @@ const REPO_ROOT = resolve(HERE, '..', '..'); const SUPERDOC_TS = resolve(REPO_ROOT, 'packages/superdoc/src/core/SuperDoc.ts'); const FIXTURE_DIR = resolve(REPO_ROOT, 'tests/consumer-typecheck/src'); const ALLOWLIST_PATH = resolve(HERE, 'public-method-coverage-allowlist.cjs'); -const SNAPSHOT_PATH = resolve(HERE, 'public-method-coverage-debt-snapshot.json'); const require = createRequire(import.meta.url); const ts = require('typescript'); -const flags = new Set(process.argv.slice(2)); -const writeMode = flags.has('--write'); +// --write was used during the ratchet phase to refresh a grandfathered +// debt snapshot. Strict-zero mode rejects it loudly so contributors +// don't accidentally re-introduce grandfathering. +if (process.argv.includes('--write')) { + console.error( + '[public-method-coverage] --write is no longer supported. The gate is\n' + + 'strict zero — every unmet obligation must be satisfied by a consumer\n' + + 'fixture or moved to public-method-coverage-allowlist.cjs (with a\n' + + 'one-line reason). No grandfathered snapshot.', + ); + process.exit(2); +} const EVENT_EMITTER_MEMBERS = new Set([ 'on', 'off', 'once', 'emit', @@ -88,27 +86,6 @@ function loadAllowlist() { return mod; } -function loadSnapshot() { - if (!existsSync(SNAPSHOT_PATH)) return []; - const raw = JSON.parse(readFileSync(SNAPSHOT_PATH, 'utf8')); - if (!Array.isArray(raw.knownUnmet)) { - console.error(`[public-method-coverage] invalid snapshot at ${SNAPSHOT_PATH} (missing "knownUnmet" array)`); - process.exit(1); - } - return raw.knownUnmet.slice().sort(); -} - -function writeSnapshot(entries) { - const payload = { - $comment: - 'Auto-managed by tests/consumer-typecheck/check-public-method-coverage.mjs. ' + - 'Each entry is "memberName:obligation" where obligation is one of ' + - 'parameters | returns | call. Refresh with --write after adding fixtures.', - knownUnmet: entries.slice().sort(), - }; - writeFileSync(SNAPSHOT_PATH, JSON.stringify(payload, null, 2) + '\n'); -} - /** Enumerate public members and compute their obligations. */ function enumerateObligations() { const src = readFileSync(SUPERDOC_TS, 'utf8'); @@ -245,33 +222,17 @@ for (const m of members) { } unmetNow.sort(); -if (writeMode) { - writeSnapshot(unmetNow); - console.log( - `[public-method-coverage] wrote ${SNAPSHOT_PATH.replace(REPO_ROOT + '/', '')} (${unmetNow.length} entries).`, - ); - process.exit(0); -} - -const snapshot = loadSnapshot(); -const snapshotSet = new Set(snapshot); -const unmetSet = new Set(unmetNow); - -const newUnmet = unmetNow.filter((e) => !snapshotSet.has(e)); -const stale = snapshot.filter((e) => !unmetSet.has(e)); - const totalObligations = members.reduce((n, m) => n + m.obligations.length, 0); const HR = '='.repeat(72); -console.log('[public-method-coverage] SuperDoc public-surface fixture coverage'); +console.log('[public-method-coverage] SuperDoc public-surface fixture coverage (strict zero)'); console.log(HR); console.log(`Members inspected: ${members.length}`); console.log(` Methods (non-EventEmitter): ${members.filter((m) => m.kind === 'method').length}`); console.log(` Getters: ${members.filter((m) => m.kind === 'getter').length}`); console.log(`Total obligations: ${totalObligations}`); console.log(`Allowlisted members: ${allowlistKeys.size}`); -console.log(`Tracked as known debt: ${unmetNow.length - newUnmet.length}`); -console.log(`Snapshot at: ${SNAPSHOT_PATH.replace(REPO_ROOT + '/', '')}`); +console.log(`Unmet obligations: ${unmetNow.length}`); console.log(''); const failures = []; @@ -279,10 +240,10 @@ if (allowlistFailures.length > 0) { failures.push('public-method-coverage-allowlist contract violations:'); for (const f of allowlistFailures) failures.push(f); } -if (newUnmet.length > 0) { +if (unmetNow.length > 0) { if (failures.length > 0) failures.push(''); - failures.push(`${newUnmet.length} NEW unmet obligation(s):`); - for (const e of newUnmet) failures.push(` + ${e}`); + failures.push(`${unmetNow.length} unmet obligation(s):`); + for (const e of unmetNow) failures.push(` + ${e}`); failures.push(''); failures.push(`Add a consumer fixture under tests/consumer-typecheck/src/ that asserts the`); failures.push(`required shape for each entry above. Obligation key is "memberName:obligation":`); @@ -294,24 +255,14 @@ if (newUnmet.length > 0) { failures.push(`If the member is intentionally not consumer-callable, add an entry with a`); failures.push(`one-line reason to public-method-coverage-allowlist.cjs.`); } -if (stale.length > 0) { - if (failures.length > 0) failures.push(''); - failures.push(`${stale.length} stale entry/entries in the debt snapshot (obligation now satisfied):`); - for (const e of stale) failures.push(` - ${e}`); - failures.push(''); - failures.push( - `Run \`node tests/consumer-typecheck/check-public-method-coverage.mjs --write\``, - ); - failures.push(`to refresh the snapshot and lock in the win.`); -} if (failures.length > 0) { - console.log('FAIL fixture coverage drift:'); + console.log('FAIL fixture coverage gap:'); for (const line of failures) console.log(line); process.exit(1); } console.log( - `OK ${totalObligations} obligation(s) across ${members.length - allowlistKeys.size} members; ${unmetNow.length} tracked as known debt; ratchet snapshot in sync.`, + `OK ${totalObligations} obligation(s) across ${members.length - allowlistKeys.size} members; zero unmet.`, ); process.exit(0); diff --git a/tests/consumer-typecheck/public-method-coverage-debt-snapshot.json b/tests/consumer-typecheck/public-method-coverage-debt-snapshot.json deleted file mode 100644 index 1ce3e4e768..0000000000 --- a/tests/consumer-typecheck/public-method-coverage-debt-snapshot.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "$comment": "Auto-managed by tests/consumer-typecheck/check-public-method-coverage.mjs. Each entry is \"memberName:obligation\" where obligation is one of parameters | returns | call. Refresh with --write after adding fixtures.", - "knownUnmet": [] -} From 7d145c1b282975e15e144f9fbd6ead01262294d9 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Tue, 26 May 2026 20:12:07 -0300 Subject: [PATCH 2/2] fix(scripts): distinguish enforced vs allowlisted obligation counts The status block conflated total obligations across all inspected members with the count actually enforced by the gate. With 8 allowlisted members carrying 13 obligations, the old output showed 'Total obligations: 72' and 'OK 72 obligation(s) across 36 members', which is internally inconsistent (the 72 is across 44 members; only 36 are enforced). Now distinguishes the two: - Enforced members: 36 - Enforced obligations: 59 (the number the gate runs on) - Total obligations (pre-allowlist): 72 (raw count from AST) OK line now reads '59 enforced obligation(s) across 36 members; zero unmet.' so success output matches what was actually checked. Verified: - node check-public-method-coverage.mjs -> OK, 59 enforced across 36 - node check-public-method-coverage.mjs --write -> rejected, exit 2 - pnpm check:public:superdoc --skip-build -> PASS (11/12, 136.4s) --- .../check-public-method-coverage.mjs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/consumer-typecheck/check-public-method-coverage.mjs b/tests/consumer-typecheck/check-public-method-coverage.mjs index 47d9968603..6fa204e303 100644 --- a/tests/consumer-typecheck/check-public-method-coverage.mjs +++ b/tests/consumer-typecheck/check-public-method-coverage.mjs @@ -223,16 +223,20 @@ for (const m of members) { unmetNow.sort(); const totalObligations = members.reduce((n, m) => n + m.obligations.length, 0); +const enforcedMembers = members.filter((m) => !allowlistKeys.has(m.name)); +const enforcedObligations = enforcedMembers.reduce((n, m) => n + m.obligations.length, 0); const HR = '='.repeat(72); console.log('[public-method-coverage] SuperDoc public-surface fixture coverage (strict zero)'); console.log(HR); -console.log(`Members inspected: ${members.length}`); -console.log(` Methods (non-EventEmitter): ${members.filter((m) => m.kind === 'method').length}`); -console.log(` Getters: ${members.filter((m) => m.kind === 'getter').length}`); -console.log(`Total obligations: ${totalObligations}`); -console.log(`Allowlisted members: ${allowlistKeys.size}`); -console.log(`Unmet obligations: ${unmetNow.length}`); +console.log(`Members inspected: ${members.length}`); +console.log(` Methods (non-EventEmitter): ${members.filter((m) => m.kind === 'method').length}`); +console.log(` Getters: ${members.filter((m) => m.kind === 'getter').length}`); +console.log(`Allowlisted members: ${allowlistKeys.size}`); +console.log(`Enforced members: ${enforcedMembers.length}`); +console.log(`Enforced obligations: ${enforcedObligations}`); +console.log(`Total obligations (pre-allowlist): ${totalObligations}`); +console.log(`Unmet obligations: ${unmetNow.length}`); console.log(''); const failures = []; @@ -263,6 +267,6 @@ if (failures.length > 0) { } console.log( - `OK ${totalObligations} obligation(s) across ${members.length - allowlistKeys.size} members; zero unmet.`, + `OK ${enforcedObligations} enforced obligation(s) across ${enforcedMembers.length} members; zero unmet.`, ); process.exit(0);