diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4eaafd0..905a304 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,15 @@ jobs: node test/vue-interp-expr.ts node test/yaml-issue12-regressions.ts node test/yaml-depth-witnesses.ts + node test/generative.ts + # The gap ledger is the deterministic, oracle-classified record of the divergences the + # generative check DISCOVERS. Its self-test asserts the ddmin keep-path + the oracle + # drop-path + determinism; --check fails if the committed KNOWN-GAPS.md is stale (the + # ledger is a pure function of the grammar, so it must be regenerated when a grammar + # changes — `npm run gap-ledger`). This is the deterministic source of truth; a later + # layer can turn rows into issues, but the committed artifact is gated here first. + node test/gap-ledger-selftest.ts + node test/gap-ledger.ts --check # The derived tree-sitter highlighter is the strongest thesis proof (a real GLR # parser from the same grammar, beating the official hand-written one). Build its diff --git a/KNOWN-GAPS.md b/KNOWN-GAPS.md new file mode 100644 index 0000000..8cd7f44 --- /dev/null +++ b/KNOWN-GAPS.md @@ -0,0 +1,62 @@ +# KNOWN-GAPS — Monogram flat-highlighter divergences (auto-generated) + + + +A **gap** is a position where, on **valid input** (accepted by the language’s external +authority — typescript / yaml / parse5), the **flat TextMate highlighter** paints a token a +different visual role than the **Monogram parser** assigns it by construction. These are the +floor-blind divergences the generative scope≡role check (`test/generative.ts`) DISCOVERS over +grammar-derived inputs — the monogram#23/#24 class — which the corpus-bound scope-gap metric is +blind to (a small/clean corpus may never contain the shape, and the role-graded metric ignores +punctuation-floor mis-paints). Each gap’s input is **minimized** (delta-debugged to a minimal +repro that still parses and still diverges) and **fingerprinted** (a content hash, stable across +commits) so the ledger is deterministic and commit-trackable. + +Regenerate: `node test/gap-ledger.ts --write` · verify up-to-date: `node test/gap-ledger.ts --check`. + +**2 gaps** across 7 grammars · 0 dropped. + +## `525e867dc205` — html: #24 structural-literal→content + +- **Language:** html +- **Minimal repro:** `` +- **Divergent token:** `/` (parser token `$punct`) +- **Role vs scope:** want **punct**, got **string** (highlighter scope `string.unquoted.html`) +- **Fingerprint:** `525e867dc205` + +```json +{ + "id": "525e867dc205", + "language": "html", + "kind": "#24 structural-literal→content", + "repro": "", + "tokenType": "$punct", + "tokenText": "/", + "want": "punct", + "got": "string", + "gotScope": "string.unquoted.html" +} +``` + +## `85c793d02a86` — vue: #24 structural-literal→content + +- **Language:** vue +- **Minimal repro:** `` +- **Divergent token:** `/` (parser token `$punct`) +- **Role vs scope:** want **punct**, got **string** (highlighter scope `string.unquoted.vue`) +- **Fingerprint:** `85c793d02a86` + +```json +{ + "id": "85c793d02a86", + "language": "vue", + "kind": "#24 structural-literal→content", + "repro": "", + "tokenType": "$punct", + "tokenText": "/", + "want": "punct", + "got": "string", + "gotScope": "string.unquoted.vue" +} +``` + diff --git a/package.json b/package.json index 3491dd9..ec8f9c8 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,10 @@ "scripts": { "gen": "node src/cli.ts typescript.ts && node src/cli.ts javascript.ts && node src/cli.ts typescriptreact.ts && node src/cli.ts javascriptreact.ts && node src/cli.ts html.ts && node src/cli.ts vue.ts && node src/cli.ts yaml.ts", "test": "node test/sanity-check.ts", + "generative": "node test/generative.ts", + "gap-ledger": "node test/gap-ledger.ts --write", + "gap-ledger:check": "node test/gap-ledger.ts --check", + "gap-ledger:selftest": "node test/gap-ledger-selftest.ts", "conformance": "node test/run-conformance.ts", "conformance:js": "node test/js-conformance.ts", "conformance:tsx": "node test/tsx-conformance.ts", @@ -29,19 +33,19 @@ "bench:perf": "node test/perf-bench.ts", "coverage": "node test/scope-coverage.ts", "compat": "node test/repo-compat.ts", - "src-coverage:ts": "node test/src-coverage-ts.ts", - "src-coverage:js": "node test/src-coverage-js.ts", - "src-coverage:jsx": "node test/src-coverage-jsx.ts", - "src-coverage:tsx": "node test/src-coverage-tsx.ts", - "src-coverage:html": "node test/src-coverage-html.ts", - "src-coverage:yaml": "node test/src-coverage-yaml.ts", - "scope-gap:ts": "node test/scope-gap-ts.ts", - "scope-gap:js": "node test/scope-gap-js.ts", - "scope-gap:jsx": "node test/scope-gap-jsx.ts", - "scope-gap:tsx": "node test/scope-gap-tsx.ts", - "scope-gap:html": "node test/scope-gap-html.ts", - "scope-gap:yaml": "node test/scope-gap-yaml.ts", - "scope-gap:vue": "node test/scope-gap-vue.ts", + "src-coverage:ts": "node test/src-coverage-run.ts ts", + "src-coverage:js": "node test/src-coverage-run.ts js", + "src-coverage:jsx": "node test/src-coverage-run.ts jsx", + "src-coverage:tsx": "node test/src-coverage-run.ts tsx", + "src-coverage:html": "node test/src-coverage-run.ts html", + "src-coverage:yaml": "node test/src-coverage-run.ts yaml", + "scope-gap:ts": "node test/scope-gap-run.ts ts", + "scope-gap:js": "node test/scope-gap-run.ts js", + "scope-gap:jsx": "node test/scope-gap-run.ts jsx", + "scope-gap:tsx": "node test/scope-gap-run.ts tsx", + "scope-gap:html": "node test/scope-gap-run.ts html", + "scope-gap:yaml": "node test/scope-gap-run.ts yaml", + "scope-gap:vue": "node test/scope-gap-run.ts vue", "coverage:table": "node test/coverage-table.ts --write" }, "devDependencies": { diff --git a/test/bench-vs-ts-agg.ts b/test/bench-vs-ts-agg.ts deleted file mode 100644 index a9a42ad..0000000 --- a/test/bench-vs-ts-agg.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { createParser } from '../src/gen-parser.ts'; -import { readdir } from 'fs/promises'; -import { readFileSync } from 'fs'; -import { join } from 'path'; -import ts from 'typescript'; -const grammar = (await import('../typescript.ts')).default; -const { parse } = createParser(grammar); -const base = '/tmp/ts-repo/tests/cases/conformance'; -async function all(d: string): Promise { const o:string[]=[]; for(const e of await readdir(d,{withFileTypes:true})){const f=join(d,e.name); if(e.isDirectory())o.push(...await all(f)); else if(e.name.endsWith('.ts')&&!e.name.endsWith('.d.ts'))o.push(f);} return o; } -const files = (await all(base)).map(f => readFileSync(f,'utf-8')); -const totalKB = files.reduce((s,c)=>s+c.length,0)/1024; -// warm up -for(const c of files.slice(0,200)){ try{parse(c);}catch{} ts.createSourceFile('t.ts',c,ts.ScriptTarget.Latest,false,ts.ScriptKind.TS); } -let t0=process.hrtime.bigint(); for(const c of files){ try{parse(c);}catch{} } const ours=Number(process.hrtime.bigint()-t0)/1e6; -t0=process.hrtime.bigint(); for(const c of files){ ts.createSourceFile('t.ts',c,ts.ScriptTarget.Latest,false,ts.ScriptKind.TS); } const tsms=Number(process.hrtime.bigint()-t0)/1e6; -console.log(`${files.length} files, ${totalKB.toFixed(0)} KB total`); -console.log(` ours: ${ours.toFixed(0)} ms (${(totalKB/1024/(ours/1000)).toFixed(1)} MB/s)`); -console.log(` ts: ${tsms.toFixed(0)} ms (${(totalKB/1024/(tsms/1000)).toFixed(1)} MB/s)`); -console.log(` ours/ts: ×${(ours/tsms).toFixed(1)}`); diff --git a/test/bench-vs-ts.ts b/test/bench-vs-ts.ts deleted file mode 100644 index c15d4ea..0000000 --- a/test/bench-vs-ts.ts +++ /dev/null @@ -1,38 +0,0 @@ -// Compare our grammar-driven parser against TypeScript's own parser (ts.createSourceFile) -// on the same inputs. Both do a full from-scratch parse (no incremental reuse). -import { createParser } from '../src/gen-parser.ts'; -import { readFileSync } from 'fs'; -import ts from 'typescript'; - -const grammar = (await import('../typescript.ts')).default; -const { parse } = createParser(grammar); - -const tsParse = (code: string) => - ts.createSourceFile('t.ts', code, ts.ScriptTarget.Latest, /*setParentNodes*/ false, ts.ScriptKind.TS); - -function timeIt(fn: () => void, iters: number): number { - for (let i = 0; i < 3; i++) fn(); // warm up - const start = process.hrtime.bigint(); - for (let i = 0; i < iters; i++) fn(); - return Number(process.hrtime.bigint() - start) / 1e6 / iters; // ms/parse -} - -const files = [ - ['parserharness.ts', '/tmp/ts-repo/tests/cases/conformance/parser/ecmascript5/RealWorld/parserharness.ts'], - ['fixSignatureCaching.ts', '/tmp/ts-repo/tests/cases/conformance/fixSignatureCaching.ts'], - ['parserRealSource7.ts', '/tmp/ts-repo/tests/cases/conformance/parser/ecmascript5/parserRealSource7.ts'], - ['parserindenter.ts', '/tmp/ts-repo/tests/cases/conformance/parser/ecmascript5/RealWorld/parserindenter.ts'], -]; - -console.log('file KB ours(ms) ts(ms) ours/ts'); -for (const [name, path] of files) { - const code = readFileSync(path, 'utf-8'); - const kb = (code.length / 1024).toFixed(0); - const ours = timeIt(() => { try { parse(code); } catch {} }, 30); - const tsms = timeIt(() => { tsParse(code); }, 30); - console.log( - name.padEnd(28) + kb.padStart(4) + - ours.toFixed(1).padStart(11) + tsms.toFixed(2).padStart(9) + - ('×' + (ours / tsms).toFixed(1)).padStart(10), - ); -} diff --git a/test/classify-ts.ts b/test/classify-ts.ts deleted file mode 100644 index 348a8de..0000000 --- a/test/classify-ts.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { createParser } from '../src/gen-parser.ts'; -import { readdir, writeFile } from 'fs/promises'; -import { readFileSync } from 'fs'; -import { join } from 'path'; -import ts from 'typescript'; - -const grammar = (await import('../typescript.ts')).default; -const { parse } = createParser(grammar); -const baseDir = '/tmp/ts-repo/tests/cases/conformance'; - -async function getAllTsFiles(dir: string): Promise { - const files: string[] = []; - for (const entry of await readdir(dir, { withFileTypes: true })) { - const full = join(dir, entry.name); - if (entry.isDirectory()) files.push(...await getAllTsFiles(full)); - else if (entry.name.endsWith('.ts') && !entry.name.endsWith('.d.ts')) files.push(full); - } - return files; -} - -// Count syntactic parse diagnostics for a chunk of TS source. -function syntaxErrors(text: string, name = 't.ts'): number { - const sf = ts.createSourceFile(name, text, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS); - return (sf as any).parseDiagnostics?.length ?? 0; -} - -// Split TS conformance file by `// @filename:` directives. -function splitMultiFile(text: string): string[] { - if (!/^\s*\/\/\s*@filename:/im.test(text)) return [text]; - const parts: string[] = []; - const re = /^\s*\/\/\s*@filename:.*$/gim; - let last = 0, m: RegExpExecArray | null, started = false; - const idxs: number[] = []; - while ((m = re.exec(text))) idxs.push(m.index); - if (idxs.length === 0) return [text]; - // preamble before first @filename (global directives) — ignore as its own chunk - for (let i = 0; i < idxs.length; i++) { - const start = idxs[i]; - const end = i + 1 < idxs.length ? idxs[i + 1] : text.length; - parts.push(text.slice(start, end)); - } - return parts; -} - -const files = await getAllTsFiles(baseDir); -files.sort(); - -interface Row { file: string; ourMsg: string; tsWhole: number; tsParts: number; multi: boolean; } -const rows: Row[] = []; - -for (const file of files) { - const code = readFileSync(file, 'utf-8'); - let ourFail = false, ourMsg = ''; - try { parse(code); } catch (e: any) { ourFail = true; ourMsg = e.message.replace(/\s*\[farthest.*/, ''); } - if (!ourFail) continue; - - const path = file.replace(baseDir + '/', ''); - const tsWhole = syntaxErrors(code); - const parts = splitMultiFile(code); - const multi = parts.length > 1; - const tsParts = multi ? parts.reduce((a, p) => a + syntaxErrors(p), 0) : tsWhole; - rows.push({ file: path, ourMsg, tsWhole, tsParts, multi }); -} - -// Categories: -// REAL: TS reports 0 syntax errors (on parts if multi, else whole) -> we should parse -// MULTI: multi-file, parts clean but whole dirty (concatenation issue, structural) -// ERRORTEST: TS reports syntax errors -> intentional -const real = rows.filter(r => !r.multi && r.tsWhole === 0); -const multiClean = rows.filter(r => r.multi && r.tsParts === 0); -const multiDirty = rows.filter(r => r.multi && r.tsParts > 0); -const errorTest = rows.filter(r => !r.multi && r.tsWhole > 0); - -const out: string[] = []; -out.push(`Total our failures: ${rows.length}`); -out.push(`REAL (TS clean, single-file) : ${real.length}`); -out.push(`MULTI-CLEAN (parts clean, concat fails): ${multiClean.length}`); -out.push(`MULTI-DIRTY (multi-file w/ syntax err) : ${multiDirty.length}`); -out.push(`ERROR-TEST (TS reports syntax error) : ${errorTest.length}`); -out.push(''); -out.push('===== REAL (should fix) ====='); -for (const r of real) out.push(` ${r.file}\n ${r.ourMsg}`); -out.push(''); -out.push('===== MULTI-CLEAN (structural, @filename concat) ====='); -for (const r of multiClean) out.push(` ${r.file}\n ${r.ourMsg}`); -out.push(''); -out.push('===== MULTI-DIRTY (has intentional errors in some part) ====='); -for (const r of multiDirty) out.push(` ${r.file} (tsParts=${r.tsParts})`); - -const text = out.join('\n'); -await writeFile('/tmp/classify.txt', text); -console.log(text.split('\n').slice(0, 6).join('\n')); -console.log('\nFull report: /tmp/classify.txt'); diff --git a/test/coverage-table.ts b/test/coverage-table.ts index 5484881..eb61dbb 100644 --- a/test/coverage-table.ts +++ b/test/coverage-table.ts @@ -18,27 +18,29 @@ function runAdapter(script: string, args: string[], marker: string, env?: NodeJS } catch { return null; } } -// TS/JS use deterministic stride subsets for speed; the rest run their full corpus. +// Both metrics now run through ONE data-driven driver each, parameterised by the `` code +// (test/scope-gap-run.ts, test/src-coverage-run.ts). TS/JS use deterministic stride subsets for +// speed; the rest run their full corpus. const COV = [ - { lang: 'TypeScript', script: 'test/src-coverage-ts.ts', args: ['1500'] }, - { lang: 'JavaScript', script: 'test/src-coverage-js.ts', args: ['800'] }, - { lang: 'JSX', script: 'test/src-coverage-jsx.ts', args: [] }, - { lang: 'TSX', script: 'test/src-coverage-tsx.ts', args: [] }, - { lang: 'HTML', script: 'test/src-coverage-html.ts', args: [] }, - { lang: 'YAML', script: 'test/src-coverage-yaml.ts', args: [] }, + { lang: 'TypeScript', script: 'test/src-coverage-run.ts', args: ['ts', '1500'] }, + { lang: 'JavaScript', script: 'test/src-coverage-run.ts', args: ['js', '800'] }, + { lang: 'JSX', script: 'test/src-coverage-run.ts', args: ['jsx'] }, + { lang: 'TSX', script: 'test/src-coverage-run.ts', args: ['tsx'] }, + { lang: 'HTML', script: 'test/src-coverage-run.ts', args: ['html'] }, + { lang: 'YAML', script: 'test/src-coverage-run.ts', args: ['yaml'] }, ]; -// The 4 TS-family scope-gap adapters all read ONE shared env var (MONOGRAM_OFFICIAL_TM) for -// the official grammar, so each needs its OWN grammar mapped in (CI sets MONOGRAM_OFFICIAL_TS/ -// TSX/JS/JSX). html/yaml read their own var (MONOGRAM_OFFICIAL_HTML/_YAML), inherited as-is; -// vue is vendored. Absent (local, no env) → each adapter's VS Code-install fallback path. +// The 4 TS-family scope-gap entries all read ONE shared env var (MONOGRAM_OFFICIAL_TM) for the +// official grammar, so each needs its OWN grammar mapped in (CI sets MONOGRAM_OFFICIAL_TS/TSX/JS/JSX). +// html/yaml read their own var (MONOGRAM_OFFICIAL_HTML/_YAML), inherited as-is; vue is vendored. +// Absent (local, no env) → the driver's VS Code-install fallback path. const GAP = [ - { lang: 'TypeScript', script: 'test/scope-gap-ts.ts', args: ['800'], officialEnv: 'MONOGRAM_OFFICIAL_TS' }, - { lang: 'JavaScript', script: 'test/scope-gap-js.ts', args: ['800'], officialEnv: 'MONOGRAM_OFFICIAL_JS' }, - { lang: 'JSX', script: 'test/scope-gap-jsx.ts', args: [], officialEnv: 'MONOGRAM_OFFICIAL_JSX' }, - { lang: 'TSX', script: 'test/scope-gap-tsx.ts', args: [], officialEnv: 'MONOGRAM_OFFICIAL_TSX' }, - { lang: 'HTML', script: 'test/scope-gap-html.ts', args: [] }, - { lang: 'YAML', script: 'test/scope-gap-yaml.ts', args: [] }, - { lang: 'Vue', script: 'test/scope-gap-vue.ts', args: [] }, + { lang: 'TypeScript', script: 'test/scope-gap-run.ts', args: ['ts', '800'], officialEnv: 'MONOGRAM_OFFICIAL_TS' }, + { lang: 'JavaScript', script: 'test/scope-gap-run.ts', args: ['js', '800'], officialEnv: 'MONOGRAM_OFFICIAL_JS' }, + { lang: 'JSX', script: 'test/scope-gap-run.ts', args: ['jsx'], officialEnv: 'MONOGRAM_OFFICIAL_JSX' }, + { lang: 'TSX', script: 'test/scope-gap-run.ts', args: ['tsx'], officialEnv: 'MONOGRAM_OFFICIAL_TSX' }, + { lang: 'HTML', script: 'test/scope-gap-run.ts', args: ['html'] }, + { lang: 'YAML', script: 'test/scope-gap-run.ts', args: ['yaml'] }, + { lang: 'Vue', script: 'test/scope-gap-run.ts', args: ['vue'] }, ] as { lang: string; script: string; args: string[]; officialEnv?: string }[]; const pct = (v: number | null | undefined) => (v == null ? '—' : v.toFixed(1) + '%'); diff --git a/test/scope-gap-jsx.ts b/test/curated-corpora.ts similarity index 55% rename from test/scope-gap-jsx.ts rename to test/curated-corpora.ts index 9a0a8bf..136c3e8 100644 --- a/test/scope-gap-jsx.ts +++ b/test/curated-corpora.ts @@ -1,18 +1,7 @@ -// scope-gap-jsx.ts — JSX (.jsx) adapter for the unified scope-gap harness. Grades VS Code's -// OFFICIAL JavaScriptReact.tmLanguage.json AND Monogram's javascriptreact.tmLanguage.json against -// the parser oracle (oracle.ts with ScriptKind.JSX). Both grammars declare scopeName `source.js.jsx`. -// Neither the TS suite nor Test262 ships a .jsx corpus, so this uses a CURATED set exercising both -// halves (plain JS + JSX), copied verbatim from src-coverage-jsx.ts. It is small, so token counts -// are low; a real .jsx corpus is a follow-up. Run (bare node): node test/scope-gap-jsx.ts -import ts from 'typescript'; -import { run } from './scope-gap.ts'; -import { oracle } from './oracle.ts'; +// curated-corpora.ts — small hand-written corpora shared by the folded scope-gap / src-coverage drivers. +// JSX (plain-JS + JSX halves) and realistic HTML — the languages with no public single-file corpus. -const OFFICIAL = process.env.MONOGRAM_OFFICIAL_TM - ?? '/Applications/Visual Studio Code.app/Contents/Resources/app/extensions/javascript/syntaxes/JavaScriptReact.tmLanguage.json'; - -// No TS types — these are .jsx (JavaScript + JSX) only. Copied verbatim from src-coverage-jsx.ts. -const JSX_CASES: string[] = [ +export const JSX_CASES: string[] = [ // --- plain JS half --- 'const x = 1, y = 2;', 'function f(a, b = 1, ...rest) { return a + b + rest.length; }', @@ -52,15 +41,25 @@ const JSX_CASES: string[] = [ 'const boolAttr = ;', ]; -await run({ - name: 'JavaScriptReact (.jsx)', - scopeName: 'source.js.jsx', - officialPath: OFFICIAL, - monogramPath: 'javascriptreact.tmLanguage.json', - loadCorpus: () => JSX_CASES.map((code, i) => ({ name: ``, text: code })), - roleOracle: (text) => oracle(text, ts.ScriptKind.JSX), - isGradable: (text) => { - const sf = ts.createSourceFile('c.jsx', text, ts.ScriptTarget.Latest, true, ts.ScriptKind.JSX); - return (((sf as any).parseDiagnostics?.length ?? 0) === 0); - }, -}); +export const HTML_GENERAL: string[] = [ + '

Hello world.

', + '', + 'a picture', + '', + '', + '

Title

Body with bold and italic.

', + '', + '
', + '
AB
12
', + '
x
', + '', + '
', + '', + '', + '', + 'text', + 'body', + 'link', + '', + '
photo
cap
', +]; diff --git a/test/diag.ts b/test/diag.ts deleted file mode 100644 index 9fcd116..0000000 --- a/test/diag.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { createParser } from '../src/gen-parser.ts'; -import { readFileSync } from 'fs'; -const grammar = (await import('../typescript.ts')).default; -const { parse } = createParser(grammar); -for (const f of process.argv.slice(2)) { - const code = readFileSync(f, 'utf-8'); - try { parse(code); console.log(f.split('/').pop(), 'OK'); } - catch (e: any) { - console.log(f.split('/').pop(), '\n ', e.message); - const m = e.message.match(/farthest: offset (\d+)/); - if (m) { const o = +m[1]; console.log(' CTX:', JSON.stringify(code.slice(Math.max(0, o - 70), o + 30))); } - } -} diff --git a/test/gap-ledger-selftest.ts b/test/gap-ledger-selftest.ts new file mode 100644 index 0000000..4c2125f --- /dev/null +++ b/test/gap-ledger-selftest.ts @@ -0,0 +1,89 @@ +// ───────────────────────────────────────────────────────────────────────────── +// gap-ledger-selftest.ts — asserts the gap ledger's two load-bearing behaviours +// on the REAL HTML probe, independent of how many gaps happen to surface: +// +// (A) DETERMINISM — `generateInputs` + ddmin + fingerprint are a pure function of +// the grammar, so two full ledger builds are byte-identical. Asserted here over +// the rendered KNOWN-GAPS.md (the committed artifact) by building it twice. +// +// (B) the oracle CLASSIFY DROP-PATH — a divergence whose minimized repro the external +// oracle REJECTS (a parser over-accept, not a real highlighter gap) is DROPPED, +// not filed. We assert the ledger's keep/drop predicate (`oracleAccepts(repro)`) +// routes a parser OVER-ACCEPT (a markup the Monogram parser accepts but parse5 +// REJECTS — `< a/>`, `<:a/>`) to DROP, and the oracle-VALID ``-shape to +// KEEP. (Note: the self-close `/` divergence itself only arises on WELL-FORMED tag +// shapes — which parse5 also accepts — so a single input that BOTH diverges AND is +// oracle-rejected does not exist for this gap; the drop-path is exercised by the +// classify predicate over real over-accept markup, which is what would gate it.) +// +// Run (bare node): node test/gap-ledger-selftest.ts +// ───────────────────────────────────────────────────────────────────────────── +import { execFileSync } from 'node:child_process'; +import { createParser } from '../src/gen-parser.ts'; +import type { CstGrammar } from '../src/types.ts'; +import { buildRoleMap, anchoredScopes, leafRoles, collectViolations, isGated } from './generative-detect.ts'; +import { loadTm, tmTokenize, reproStillDiverges, sig, minimize, LANGS, type Probe } from './gap-ledger.ts'; + +let failures = 0; +const ok = (cond: boolean, msg: string) => { console.log(`${cond ? ' ✓' : ' ✗ FAIL:'} ${msg}`); if (!cond) failures++; }; + +// ── build the HTML probe (the cheapest grammar with a known divergence) ── +const htmlCfg = LANGS.find((l) => l.name === 'html')!; +const grammar = (await import(htmlCfg.module)).default as CstGrammar; +const { parse } = createParser(grammar); +const tm = await loadTm(htmlCfg.scopeName, { [htmlCfg.scopeName]: htmlCfg.tmPath, ...(htmlCfg.tmExtra ?? {}) }); +if (!tm) throw new Error('failed to load html grammar'); +const probe: Probe = { parse, tm, grammar, roleOf: buildRoleMap(grammar), anchored: anchoredScopes(grammar) }; + +// the ledger's CLASSIFY predicate, verbatim: keep iff the oracle accepts the minimal repro as VALID. +const classifyKeeps = (text: string) => htmlCfg.oracleAccepts(text); + +console.log('gap-ledger self-test\n'); + +// ── (B1) the canonical KEPT case: ``-shape, oracle-valid, still diverges ── +const keptInput = ''; // the generator's tight-markup shape +{ + // detect the divergence on the real input, minimize, classify + const v0 = probeDivergence(keptInput); + ok(!!v0, `kept case: a self-close \`/\` divergence is detected on ${JSON.stringify(keptInput)}`); + if (v0) { + const repro = minimize(probe, keptInput, v0.target); + ok(!!reproStillDiverges(probe, repro, v0.target), `kept case: minimized repro ${JSON.stringify(repro)} still diverges`); + ok(classifyKeeps(repro), `kept case: parse5 ACCEPTS the minimized repro → KEEP (a real highlighter gap)`); + } +} + +// ── (B2) the DROP case: real parser OVER-ACCEPTS (parser accepts, parse5 REJECTS) ── +// markup the Monogram markup parser accepts but parse5 does NOT recover as an element — exactly the +// "Monogram parses but the oracle rejects" class the ledger must DROP (a parser concern, not a +// highlighter gap). We assert each is parser-accepted AND classify-DROPPED (oracleAccepts == false). +const overAccepts = ['< a/>', '<:a/>']; +let dropProven = false; +for (const cand of overAccepts) { + let parserOk = false; try { parse(cand); parserOk = true; } catch { /* */ } + if (!parserOk) continue; + ok(!classifyKeeps(cand), `drop case: ${JSON.stringify(cand)} is parser-accepted but parse5-REJECTS → classify DROPS it`); + dropProven = true; +} +ok(dropProven, 'drop case: at least one real parser-over-accept is parser-accepted and confirmed dropped'); +// and the dual: the oracle-VALID minimal repro is KEPT (not dropped) — the keep/drop split is real. +ok(classifyKeeps(''), 'keep/drop split: the oracle-VALID ``-shape repro is KEPT (not dropped)'); + +// ── (A) determinism of the rendered artifact: two builds byte-identical ── +console.log('\n determinism (two full ledger builds)…'); +const run = () => execFileSync('node', ['test/gap-ledger.ts'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 64 * 1024 * 1024 }); +const a = run(), b = run(); +ok(a === b, `two \`node test/gap-ledger.ts\` runs produce byte-identical output (${a.length} bytes)`); + +console.log(failures ? `\n${failures} self-test failure(s).` : '\nAll gap-ledger self-tests passed.'); +process.exit(failures ? 1 : 0); + +// ── helper: detect the self-close `/` divergence on `text`, returning its signature ── +function probeDivergence(text: string): { target: string } | null { + let cst; try { cst = parse(text); } catch { return null; } + let toks; try { toks = tmTokenize(probe.tm, text); } catch { return null; } + const leaves = leafRoles(grammar, cst, probe.roleOf); + const vs = collectViolations({ input: text, strategy: 'fuzz', cst, toks, leaves, anchored: probe.anchored }); + const v = vs.find((x) => !isGated(x)); + return v ? { target: sig(v) } : null; +} diff --git a/test/gap-ledger.ts b/test/gap-ledger.ts new file mode 100644 index 0000000..7b3769d --- /dev/null +++ b/test/gap-ledger.ts @@ -0,0 +1,380 @@ +// ───────────────────────────────────────────────────────────────────────────── +// gap-ledger.ts — a DETERMINISTIC, auto-maintained GAP LEDGER for Monogram. +// +// The generative by-construction check (test/generative.ts) DISCOVERS divergences +// where the flat TextMate highlighter and the Monogram parser disagree on the +// visual role of a token in a grammar-DERIVED input — the floor-blind class the +// corpus-bound scope-gap metric is blind to (monogram#23/#24). That check REPORTS +// them; this ledger OPERATIONALIZES them into a stable, commit-trackable artifact: +// +// 1. DISCOVER — for each of the 7 grammars, generate inputs deterministically +// (grammar-gen.ts), tokenize with the flat grammar + parse with the parser, +// and collect the divergences using the SAME detector generative.ts uses +// (generative-detect.ts) — not a reimplementation. +// 2. MINIMIZE — delta-debug (ddmin) each divergence's input down to a minimal +// repro that still parses AND still exhibits the SAME divergence (same parser +// role-bucket vs same highlighter bucket, identified by a position-independent +// signature). The generator + ddmin are deterministic, so the minimal repro is +// stable across runs and commits. +// 3. CLASSIFY — parse the minimal repro with the language's EXTERNAL authority +// (typescript / yaml / parse5). File ONLY divergences the oracle accepts as +// VALID input (a real highlighter gap on valid input). A repro the parser +// accepts but the oracle rejects is a parser OVER-ACCEPT — a different concern; +// it is DROPPED from the gap list (its count is reported, not listed). +// 4. FINGERPRINT — a stable id = hash(language, normalized repro, role, bucket), +// so the same gap keeps the same id across commits. +// 5. EMIT — a sorted KNOWN-GAPS.md (committed artifact): per gap, the language, +// escaped minimal repro, role-vs-scope (want vs got), fingerprint, and a +// machine-readable JSON block. +// +// DETERMINISM is the whole point (a commit-trackable ledger): two runs produce a +// BYTE-IDENTICAL KNOWN-GAPS.md. The generator is a pure function of the grammar +// (no seed), ddmin is deterministic, the oracle is deterministic, and the hash is +// content-only — so nothing varies run-to-run. +// +// Run (bare node): +// node test/gap-ledger.ts # print the ledger to stdout (don't write) +// node test/gap-ledger.ts --write # (re)write KNOWN-GAPS.md +// node test/gap-ledger.ts --check # fail if KNOWN-GAPS.md is stale (CI guard) +// node test/gap-ledger.ts yaml # one language +// ───────────────────────────────────────────────────────────────────────────── +import { readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import vsctm from 'vscode-textmate'; +import onig from 'vscode-oniguruma'; +import ts from 'typescript'; +import { parseAllDocuments } from 'yaml'; +import { parseFragment } from 'parse5'; +import sfcCompiler from '@vue/compiler-sfc'; +import { createParser, type CstNode } from '../src/gen-parser.ts'; +import type { CstGrammar } from '../src/types.ts'; +import { generateInputs } from './grammar-gen.ts'; +import { + type TmTok, type Violation, + buildRoleMap, leafRoles, anchoredScopes, collectViolations, isGated, +} from './generative-detect.ts'; + +// ── language registry — the SAME per-language DATA shape as generative.ts's LANGS, plus an +// `oracleAccepts(text)`: the external authority's verdict on whether the minimal repro is VALID +// input. THAT is the only per-language wiring (a config table, like generative.ts's LANGS); the +// ddmin / fingerprint / emit ENGINE below is language-agnostic. ── +interface LangCfg { + name: string; + module: string; // grammar module (default export = CstGrammar) + scopeName: string; // TextMate scope, e.g. source.yaml + tmPath: string; // the derived flat .tmLanguage.json + tmExtra?: Record; // extra scopeName → file for multi-file grammars + oracleAccepts: (text: string) => boolean; // the neutral oracle's "is this VALID input?" verdict +} + +// ── oracle validity verdicts (DATA) ────────────────────────────────────────────────────────────── +// TS-family: tsc's own parser — zero parseDiagnostics means it accepts the text as valid source. +const tsAccepts = (kind: ts.ScriptKind) => (text: string): boolean => { + try { + const sf = ts.createSourceFile('gap.ts', text, ts.ScriptTarget.Latest, /*setParentNodes*/ false, kind); + return ((sf as any).parseDiagnostics?.length ?? 0) === 0; + } catch { return false; } +}; +// YAML: the `yaml` package — a document with zero `.errors` is valid (the same independent authority +// the scope-gap YAML oracle uses). A throw or any error ⇒ not valid. +const yamlAccepts = (text: string): boolean => { + try { const docs = parseAllDocuments(text); return docs.length > 0 && docs.every((d: any) => (d.errors?.length ?? 0) === 0); } + catch { return false; } +}; +// HTML: parse5 is error-TOLERANT (never throws), so "valid" = it recovered a real element structure — +// at least one element/tag node (not pure text / a dropped ``). This matches html-oracle.ts's own +// emission gate (it only emits tag/attr roles when parse5 reports a tagName + location). +const htmlAccepts = (text: string): boolean => { + try { + const frag: any = parseFragment(text, { sourceCodeLocationInfo: true }); + const hasEl = (nodes: any[]): boolean => nodes.some((n) => (n.tagName && n.sourceCodeLocation) || (n.childNodes && hasEl(n.childNodes))); + return hasEl(frag.childNodes ?? []); + } catch { return false; } +}; +// Vue SFC: the template markup sub-language IS HTML — vue-oracle.ts composes parse5 over the template +// content as its markup authority. @vue/compiler-sfc only does SFC BLOCK splitting; a bare template- +// level markup fragment (what the generator emits for the vue grammar — ``, not a full +// `` SFC) is NOT a top-level SFC block, so the SFC parser reports no template. +// The right neutral verdict for such markup is therefore parse5's (the template arbiter): if the SFC +// parser DID isolate a