fix: decode llms.txt entities once, after every tag strip

website/AGENTS.md

@@ -121,7 +121,22 @@ website/
                        path `webjs ui add` writes them to in a real app
     links.ts           cross-app URLs + in-app paths for the header and footer
     samples.ts         the code samples shown on the marketing pages
-    docs-llms.server.ts  enumerates the doc pages on disk (sitemap, llms.txt)
+    docs-llms.server.ts  enumerates the doc pages on disk (sitemap, llms.txt).
+                       Strips tags at every stage and decodes entities exactly
+                       ONCE, at the end. Decoding earlier puts a bare `<` in
+                       front of a later tag strip, which then matches to the
+                       next `>` anywhere in the document and deletes
+                       everything between the two. That once cost
+                       `/docs/metadata-routes` 5 of its 9 code samples and
+                       deleted an escaped tag from 253 prose lines across the
+                       corpus. It also keeps the template holes a reader
+                       actually sees: `\${x}` is ESCAPED (literal text, not an
+                       interpolation) and `${"lit"}` interpolates a known
+                       literal, so both are preserved, while only a bare
+                       `${x}` is render-time and gets dropped. Treating all
+                       three alike printed `<form action=\>` on 12 corpus
+                       lines, teaching an LLM the one shape invariant 12
+                       exists to rule out.
   modules/
     ui/components/     GITIGNORED mirror of the @webjsdev/ui registry sources,
                        written by scripts/copy-registry.mjs. NEVER hand-write

website/lib/docs-llms.server.ts

@@ -117,21 +117,36 @@ function metadataBlock(raw: string): string {
   return raw.slice(startIdx, end);
 }
 
-/** Collapse a fragment to a single trimmed line of plain text. */
+/**
+ * Collapse a fragment to a single trimmed line of plain text.
+ *
+ * Deliberately does NOT decode entities. Entities are decoded exactly once,
+ * at the END of the pipeline (`decodeEntities(body)`), because a decoded `<`
+ * re-entering a later tag strip matches from there to the next `>` anywhere
+ * in the document and deletes everything between the two. That is what cost
+ * /docs/metadata-routes 5 of its 9 samples: one 935-character match that ran
+ * from a decoded `&lt;` in one paragraph to a decoded `&lt;title&gt;` sixty
+ * lines further down. Decoding belongs after every strip, never before one.
+ */
 function oneLine(s: string): string {
   return s
     .replace(/<[^>]+>/g, ' ')
-    .replace(/&amp;/g, '&')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&#123;/g, '{')
-    .replace(/&#125;/g, '}')
-    .replace(/&#39;|&apos;/g, "'")
-    .replace(/&quot;/g, '"')
     .replace(/\s+/g, ' ')
     .trim();
 }
 
+/**
+ * `oneLine` plus the decode, in the one order that is safe: strip, THEN
+ * decode. Exported for the same reason `bodyToMarkdown` is, so a unit test
+ * can drive it on a fixture instead of planting scaffolding in a real docs
+ * page. The whitespace collapse is re-run after decoding because `&nbsp;`
+ * decodes to a literal space, so a run of them is only collapsible once the
+ * decode has happened.
+ */
+export function plainText(s: string): string {
+  return decodeEntities(oneLine(s)).replace(/\s+/g, ' ').trim();
+}
+
 /** Truncate at a word boundary, appending an ellipsis when cut. */
 function truncate(s: string, max: number): string {
   if (s.length <= max) return s;
@@ -190,22 +205,24 @@ export function bodyToMarkdown(raw: string): string {
   // had the decoded tags deleted out of it, silently, in the one pipeline
   // whose silent losses this function exists to avoid.
   //
-  // A related loss is still live and is NOT fixed here: oneLine() below
-  // decodes `&lt;` to a bare `<` while rewriting a <p>, and the generic tag
-  // strip further down then matches from that stray `<` to the next `>` and
-  // swallows what lies between, including these sentinels. On
-  // /docs/metadata-routes that costs 5 of its 9 samples and the paragraphs
-  // among them. Not fixed here: the repair reorders this pipeline for every
-  // page, and the decode is what makes prose about markup readable, so it
-  // needs its own before-and-after across all 43. test/ssr/docs-llms.test.ts
-  // pins that page at its exact counts, so it cannot decay further and a
-  // repair fails the test rather than passing unnoticed.
+  // The pipeline invariant, and the reason the stages are ordered this way:
+  // tags are stripped at EVERY stage, entities are decoded exactly ONCE, at
+  // the end. A captured sample decodes on its own path below; prose decodes
+  // at `decodeEntities(body)` after the generic strip. Decoding earlier puts
+  // a bare `<` in front of a strip that then eats to the next `>`.
   const codeBlocks: string[] = [];
   body = body.replace(/<(?:pre|code-block)(?=[\s>])[^>]*>([\s\S]*?)<\/(?:pre|code-block)>/g, (_m, code) => {
     codeBlocks.push(decodeEntities(String(code)).replace(/\n+$/, ''));
     return `\uE000CODE${codeBlocks.length - 1}\uE000`;
   });
 
+  // Prose template holes that survive rather than being dropped, parked
+  // behind a sentinel while the dynamic-hole strip runs. Same U+E001
+  // escape-not-literal rule as the code-block sentinel above, so the
+  // file stays diffable text.
+  const heldHoles: string[] = [];
+  const keepHole = (text: string) => `\uE001HOLE${heldHoles.push(text) - 1}\uE001`;
+
   body = body
     // Headings -> markdown
     .replace(/<h1[^>]*>([\s\S]*?)<\/h1>/g, (_m, t) => `\n# ${oneLine(t)}\n`)
@@ -221,8 +238,41 @@ export function bodyToMarkdown(raw: string): string {
     .replace(/<\/?(ul|ol)[^>]*>/g, '\n')
     // Strip every remaining tag
     .replace(/<[^>]+>/g, ' ')
-    // Drop template-interpolation holes
-    .replace(/\$\{[^}]*\}/g, '');
+    // Template holes in prose. Three shapes, and only one is dynamic:
+    //
+    //   \${x}      an ESCAPED hole. The `\$` means the page is not
+    //              interpolating at all, so the reader sees the literal text
+    //              `${x}`. Keep it, minus the escape.
+    //   ${"lit"}   a hole whose value is a string literal, so the reader sees
+    //              that literal. Keep the literal.
+    //   ${x}       a real hole. What it renders is known only at render time,
+    //              so there is nothing to put in the corpus. Drop it.
+    //
+    // All three used to be dropped alike, which deleted the binding out of
+    // every sentence teaching `<form action=${action}>` and stranded the
+    // escape backslash. On main that damage was hidden, because the runaway
+    // strip had already eaten those fragments whole; restoring them exposed
+    // 12 corpus lines reading `<form action=\>`, in the one surface whose
+    // reader is an LLM and about the exact shape invariant 12 governs.
+    //
+    // The kept text is parked behind a sentinel so the dynamic-hole strip
+    // below cannot eat what these two just preserved, and it is restored
+    // before `decodeEntities` so entities inside it decode like any prose.
+    .replace(/\\\$\{((?:[^{}]|\{[^}]*\})*)\}/g, (_m, inner) => keepHole('${' + unescapeJs(inner) + '}'))
+    .replace(/\$\{"((?:[^"\\]|\\.)*)"\}/g, (_m, lit) => keepHole(unescapeJs(lit)))
+    // Brace-aware: `[^}]*` would stop at the FIRST `}`, leaving `"}` debris
+    // behind a nested hole.
+    .replace(/\$\{(?:[^{}]|\{[^}]*\})*\}/g, '');
+
+  // Restore kept holes. A kept hole can itself contain a parked sentinel (an
+  // escaped hole nested inside a string-literal one), so this repeats until
+  // none is left. Replacing once emitted the inner sentinel verbatim, which
+  // would ship a private-use codepoint in a text/plain response and into the
+  // search index. Bounded: a hole can only contain sentinels parked before
+  // it, so each pass resolves at least one.
+  for (let pass = 0; pass <= heldHoles.length && /\uE001HOLE\d+\uE001/.test(body); pass++) {
+    body = body.replace(/\uE001HOLE(\d+)\uE001/g, (_m, i) => heldHoles[Number(i)]);
+  }
 
   body = decodeEntities(body);
 
@@ -287,11 +337,11 @@ async function extractPage(file: string): Promise<DocPage> {
   let description = '';
   const descMatch = meta.match(/description:\s*(?:'((?:\\.|[^'\\])*)'|"((?:\\.|[^"\\])*)"|`((?:\\.|[^`\\])*)`)/);
   if (descMatch) {
-    description = oneLine(decodeEntities(unescapeJs(descMatch[1] ?? descMatch[2] ?? descMatch[3] ?? '')));
+    description = plainText(unescapeJs(descMatch[1] ?? descMatch[2] ?? descMatch[3] ?? ''));
   }
   if (!description) {
     const pMatch = raw.match(/<p[^>]*>([\s\S]*?)<\/p>/);
-    if (pMatch) description = oneLine(decodeEntities(pMatch[1]));
+    if (pMatch) description = plainText(pMatch[1]);
   }
   description = truncate(description, 200);
 

website/test/ssr/docs-llms.test.ts

@@ -20,7 +20,7 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import { readFile } from 'node:fs/promises';
-import { bodyToMarkdown, getDocPage, getDocPages } from '#lib/docs-llms.server.ts';
+import { bodyToMarkdown, getDocPage, getDocPages, plainText } from '#lib/docs-llms.server.ts';
 
 const fenceCount = (md: string) => (md.match(/^```/gm) ?? []).length / 2;
 
@@ -55,53 +55,20 @@ test('a fenced sample keeps the interpolation holes and indentation the prose pi
   assert.match(layout, /\n {2,}\S/, 'indentation survives inside the fence');
 });
 
-/**
- * The one page whose samples do NOT all reach the corpus, pinned at its exact
- * counts rather than merely named. A bare name would exempt the page from
- * every check, which is the failure this set exists to avoid: skipped by name
- * is still skipped, and the page could then lose its remaining samples
- * unnoticed. The counts are asserted below in both directions, so further loss
- * fails and so does a repair, the latter forcing this entry to be removed
- * instead of quietly outliving the bug.
- *
- * Cause, verified rather than assumed: `oneLine()` decodes `<` to a bare
- * `<` while rewriting a `<p>`, and the generic tag strip that runs afterwards
- * (`lib/docs-llms.server.ts`, `.replace(/<[^>]+>/g, ' ')`) then matches from
- * that stray `<` to the next `>`, swallowing whatever lies between. On this
- * page that is 5 of its 9 code-block sentinels plus the paragraphs among
- * them. Removing the angle-bracket decode from `oneLine()` restores all 9,
- * which is how the cause was pinned down; that is not the fix, because the
- * decode is what makes prose about markup readable in the corpus, and the
- * real repair reorders the pipeline for all 43 pages. Pre-existing: the
- * generated corpus is byte-identical on main, so this PR neither causes it
- * nor fixes it.
- */
-const KNOWN_TRUNCATED = new Map([['/docs/metadata-routes', { authored: 9, fenced: 4 }]]);
-
-test('the truncation exemption still describes reality', async () => {
-  // An exemption nobody rechecks is a blind spot. This is what keeps the
-  // entry above honest: it fails if the page loses more samples, and it fails
-  // if the pipeline is repaired, which is the signal to delete the entry.
-  for (const [path, expected] of KNOWN_TRUNCATED) {
-    const page = (await getDocPages()).find((p) => p.path === path);
-    assert.ok(page, `${path} is exempted but no longer exists, so remove the entry`);
-    const src = await readFile(new URL(`../../app${path}/page.ts`, import.meta.url), 'utf8');
-    assert.equal((src.match(/<code-block(?=[\s>])/g) ?? []).length, expected.authored, `${path} authors a different number of samples now`);
-    assert.equal(fenceCount(page.markdown), expected.fenced, `${path} reaches the corpus with a different number of samples now: if it is ${expected.authored}, the pipeline was repaired, so delete its KNOWN_TRUNCATED entry`);
-  }
-});
-
 test('every sample a page authors reaches the corpus', async () => {
   // Stronger than the fence test above, which only asks for one fence per
   // page: this asks for ALL of them. A regression that drops or merges blocks
-  // changes this count, and the named exemption is what stops such a
-  // regression from being waved through as "already known".
+  // changes this count. There is no exemption: /docs/metadata-routes used to
+  // carry one, pinned at 9 authored and 4 fenced, because a decoded `<` in its
+  // prose let the generic tag strip eat 5 of its sentinels. The extractor now
+  // decodes exactly once, at the end, so every page reaches parity and an
+  // exemption would only hide the next such loss.
   const off: string[] = [];
   let checked = 0;
   for (const page of await getDocPages()) {
     const src = await readFile(new URL(`../../app${page.path}/page.ts`, import.meta.url), 'utf8');
     const authored = (src.match(/<code-block(?=[\s>])/g) ?? []).length;
-    if (!authored || KNOWN_TRUNCATED.has(page.path)) continue;
+    if (!authored) continue;
     checked++;
     const fenced = fenceCount(page.markdown);
     if (fenced !== authored) off.push(`${page.path}: ${authored} authored, ${fenced} fenced`);
@@ -174,6 +141,106 @@ test('a sample teaching escaped markup keeps it', () => {
   assert.match(md, /```\nuse <code>x<\/code> inline\n```/);
 });
 
+test('a paragraph teaching a lone escaped angle bracket does not eat the rest of the page', () => {
+  // The exact shape of /docs/metadata-routes: a lone `&lt;` in one paragraph,
+  // a sample, then a later paragraph whose own escaped tag supplies the `>`
+  // that closed the runaway match. A PAIRED `&lt;code&gt;` does not reproduce
+  // it, because that decodes into a complete tag the strip removes locally.
+  const md = bodyToMarkdown(
+    'html`<p>a value with <code>&lt;</code> cannot break the document</p>' +
+      '<code-block>const x = 1;</code-block>' +
+      '<p>injects <code>&lt;title&gt;</code> tags</p>`'
+  );
+  assert.match(md, /a value with < cannot break the document/);
+  assert.match(md, /```\nconst x = 1;\n```/);
+  assert.match(md, /injects <title> tags/);
+});
+
+test('an escaped tag in prose survives to the corpus', () => {
+  // oneLine used to decode &lt;code&gt; to a real <code> tag mid-pipeline and
+  // the generic strip deleted it, so the sentence lost the very thing it was
+  // written to show. This is the site-wide half of the same bug.
+  const md = bodyToMarkdown('html`<p>a value with &lt;code&gt; here</p>`');
+  assert.equal(md, 'a value with <code> here');
+});
+
+test('a hole whose value is a string literal keeps the value', () => {
+  // /docs/architecture authors this shape. The outer hole interpolates a
+  // string literal, so a reader of the rendered page sees the inner text, and
+  // the corpus has to show the same thing. Dropping it printed
+  // `<form action=>`, a form-binding sentence with the binding deleted, in
+  // the one surface whose reader is an LLM.
+  const md = bodyToMarkdown('html`<p>a <code>&lt;form action=${"${createPost}"}&gt;</code> posts</p>`');
+  assert.equal(md, 'a <form action=${createPost}> posts');
+});
+
+test('an escaped hole is literal text, not an interpolation to drop', () => {
+  // `\${x}` in the source is NOT a hole: the escape means the page renders the
+  // literal `${x}`. Dropping it as if it were dynamic deleted the binding and
+  // stranded the escape backslash, which is what put `@submit=\` and
+  // `<form action=\>` in the corpus.
+  const md = bodyToMarkdown('html`<p>handlers (<code>@submit=\\${e =&gt; { e.preventDefault(); }}</code>) are untouched</p>`');
+  assert.equal(md, 'handlers ( @submit=${e => { e.preventDefault(); }} ) are untouched');
+});
+
+test('a genuinely dynamic hole is still dropped', () => {
+  // The counterpart to the two above: a hole referencing a variable renders
+  // something known only at render time, so there is nothing to put in the
+  // corpus and it must still go.
+  assert.equal(bodyToMarkdown('html`<p>text ${children} here</p>`'), 'text here');
+});
+
+test('a dynamic hole containing braces is dropped whole', () => {
+  // A naive `[^}]*` stops at the FIRST `}`, so the nested object literal
+  // leaves `)}` behind as debris. The kept-hole shapes above no longer reach
+  // this strip, so without this fixture nothing pins it at all.
+  //
+  // ONE level of nesting is all the regex handles, which is what the docs
+  // actually author: `${fn({a:{b:1}})}` still leaves `)}`, and no page writes
+  // that (checked across all 44). Arbitrary depth is not a regex's job, so the
+  // limit is stated rather than papered over.
+  assert.equal(bodyToMarkdown('html`<p>text ${fn({a: 1})} here</p>`'), 'text here');
+  assert.equal(bodyToMarkdown('html`<p>a ${fn({a:{b:1}})} b</p>`'), 'a )} b');
+});
+
+test('a kept hole is unescaped, since the source is a template literal', () => {
+  // The hole text is copied out of page SOURCE, where a backtick has to be
+  // written `\\``. Keeping it verbatim moved the escape debris from in front
+  // of the hole to inside it: /docs/components rendered `.fallback=${html\`…\`}`
+  // where the page shows `.fallback=${html`…`}`.
+  const md = bodyToMarkdown('html`<p>x <code>.fallback=\\${html\\`hi\\`}</code> y</p>`');
+  assert.equal(md, 'x .fallback=${html`hi`} y');
+
+  // The string-literal pass copies from source too, so it needs the same fold.
+  // No docs page carries an escape inside one today, so only a fixture can
+  // hold that half of the rule.
+  assert.equal(bodyToMarkdown('html`<p>x ${"a\\`b"} y</p>`'), 'x a`b y');
+});
+
+test('a kept hole nested inside another leaves no sentinel in the output', () => {
+  // The two keep passes run in sequence, so a string-literal hole can park
+  // text that already contains an escaped hole's sentinel. Restoring once
+  // emitted the inner sentinel verbatim, shipping a private-use codepoint into
+  // a text/plain response and the search index.
+  const md = bodyToMarkdown('html`<p>a ${"\\${x}"} b</p>`');
+  assert.equal(md, 'a ${x} b');
+  assert.ok(!/[\uE000-\uF8FF]/.test(md), 'no private-use sentinel survives into the output');
+});
+
+test('plainText strips tags before it decodes entities', () => {
+  assert.equal(plainText('a value with &lt;code&gt; here'), 'a value with <code> here');
+  assert.match(plainText('intercepts same-origin &lt;a&gt; clicks'), /same-origin <a> clicks/);
+});
+
+test('a page description keeps the escaped tags it teaches', async () => {
+  // extractPage used to run oneLine(decodeEntities(...)), decoding first and
+  // stripping second, so a description teaching a tag lost it. This is the
+  // counterfactual for the plainText fixture above, which passes on its own.
+  const page = await getDocPage('client-router');
+  assert.ok(page, 'the client-router page is in the corpus');
+  assert.match(page.description, /same-origin <a> clicks and <form> submissions/);
+});
+
 test('a sample is fenced whether it is authored as code-block or pre', () => {
   for (const tag of ['code-block', 'pre']) {
     const md = bodyToMarkdown(`html\`<${tag}>const x = 1;</${tag}>\``);