fix(plugin-markdown): keep code-span content out of extractToc's HTML strip (objectui#7658)

.changeset/7658-toc-code-span-anchor-parity.md

@@ -0,0 +1,29 @@
+---
+'@object-ui/plugin-markdown': patch
+---
+
+Fix `extractToc` deleting tag-shaped text that lives INSIDE an inline code span,
+so its `#id` links resolve to the heading they name again (objectui#7658).
+
+`stripInline()` applied its rules in sequence: the inline-code rule unwrapped
+`` `objectui add <component>` `` to `objectui add <component>`, and the raw-HTML
+rule that ran next over that same text deleted `<component>` as if it were
+markup. The slug became `objectui-add` while `rehype-slug` — which slugs the
+RENDERED heading, where a code span's content is a literal text value — put
+`objectui-add-component` on the anchor. The TOC entry rendered, was clickable,
+and silently went nowhere. The same sequencing let the emphasis rules eat the
+underscores out of `` `a_b_c` `` and the link rule rewrite `` `[x](y)` ``.
+
+Code spans are now lifted out before any other inline rule runs and restored
+verbatim at the end, so nothing reaches inside one. The raw-HTML rule is
+unchanged and still strips genuine markup — `remark-rehype` runs without
+`allowDangerousHtml`, so the renderer likewise drops raw html nodes and keeps the
+text they wrapped. Link labels that are code spans still collapse (`` [`getData`](/api) ``
+→ `getData`), because the placeholder stays ordinary text to the link rule.
+
+Seven live headings in this repo's own docs were affected
+(`content/docs/utilities/cli.mdx`, `content/docs/utilities/runner.mdx`,
+`packages/cli/README.md`). Pinned against the real render pipeline rather than a
+second derivation of the slug rules: the new test renders each heading through
+`MarkdownImpl` and compares `extractToc`'s id to the `id` attribute
+`rehype-slug` actually emitted.

packages/plugin-markdown/src/toc-anchor-parity.test.tsx

@@ -0,0 +1,113 @@
+/**
+ * ObjectUI
+ * Copyright (c) 2024-present ObjectStack Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * objectui#7658 — `extractToc`'s ids must be the ids `rehype-slug` actually
+ * puts on the RENDERED headings, because that is the only thing a `#id` link
+ * can resolve to.
+ *
+ * The truth source here is the real render pipeline (`MarkdownImpl` — the same
+ * remark/rehype chain this plugin ships), NOT a second derivation of the slug
+ * rules. Re-deriving them would only prove the two derivations agree; reading
+ * the rendered `id` attribute proves the anchor exists.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { renderToStaticMarkup } from 'react-dom/server';
+import * as React from 'react';
+import MarkdownImpl from './MarkdownImpl';
+import { extractToc } from './toc';
+
+/** The ids `rehype-slug` put on the rendered headings, in document order. */
+function renderedHeadingIds(markdown: string): string[] {
+  const html = renderToStaticMarkup(React.createElement(MarkdownImpl, { content: markdown }));
+  return [...html.matchAll(/<h[1-6]\b[^>]*\bid="([^"]*)"/g)].map((m) => m[1]);
+}
+
+/** Every heading `extractToc` sees, in document order — h1–h6, not just the default h2–h3. */
+function tocIds(markdown: string): string[] {
+  return extractToc(markdown, { minDepth: 1, maxDepth: 6 }).map((item) => item.id);
+}
+
+/**
+ * The three heading shapes this repository's own docs carry that hit the
+ * defect: tag-shaped text inside a code span. In the DOM it is literal text
+ * (a code span's content is a text value), so it is part of the anchor.
+ */
+const LIVE_SHAPES: ReadonlyArray<{ md: string; id: string; where: string }> = [
+  {
+    md: '### `objectui generate <type> <name>` (alias `g`)',
+    id: 'objectui-generate-type-name-alias-g',
+    where: 'content/docs/utilities/cli.mdx:136, packages/cli/README.md:108',
+  },
+  {
+    md: '### `objectui add <component>`',
+    id: 'objectui-add-component',
+    where: 'content/docs/utilities/cli.mdx:220, packages/cli/README.md:112',
+  },
+  {
+    md: '#### Serving metadata over HTTP (`?api=<base>`)',
+    id: 'serving-metadata-over-http-apibase',
+    where: 'content/docs/utilities/runner.mdx:106',
+  },
+];
+
+describe('extractToc ↔ rendered-anchor parity (objectui#7658)', () => {
+  for (const { md, id, where } of LIVE_SHAPES) {
+    it(`resolves the anchor for ${md} (${where})`, () => {
+      const source = `${md}\n`;
+      // Reading the truth source is itself the lit control: an empty list here
+      // means the harness rendered nothing, and every comparison below it would
+      // be a vacuous pass.
+      expect(renderedHeadingIds(source)).toEqual([id]);
+      expect(tocIds(source)).toEqual([id]);
+    });
+  }
+
+  it('control: agrees on the inline shapes that never had the defect', () => {
+    // Lit control — these ids are non-empty and already matched before the fix,
+    // so a run in which they read `[]` (or drifted) is a broken instrument
+    // rather than evidence about the defect.
+    const source = '# Title\n\n## First Section\n\n## The **overlay** `rule` and a [link](/x)\n';
+    const rendered = renderedHeadingIds(source);
+    expect(rendered).toEqual(['title', 'first-section', 'the-overlay-rule-and-a-link']);
+    expect(tocIds(source)).toEqual(rendered);
+  });
+
+  it('still drops genuine raw HTML, exactly as the renderer does', () => {
+    // The raw-HTML rule is not being removed, only kept off code spans:
+    // `remark-rehype` drops raw html nodes (no `allowDangerousHtml`), so the
+    // rendered heading keeps the wrapped text and not the tags.
+    const source = '## A <b>bold</b> tag\n';
+    expect(tocIds(source)).toEqual(renderedHeadingIds(source));
+  });
+
+  it('keeps duplicate-suffix alignment across affected headings', () => {
+    // The `-1/-2` suffixes only line up if EVERY heading slugs the same text
+    // the renderer slugs — one wrong id upstream shifts every later anchor.
+    const source =
+      '# `objectui add <component>`\n\n' +
+      '## `objectui add <component>`\n\n' +
+      '## `objectui add <component>`\n';
+    const rendered = renderedHeadingIds(source);
+    expect(rendered).toEqual(['objectui-add-component', 'objectui-add-component-1', 'objectui-add-component-2']);
+    expect(tocIds(source)).toEqual(rendered);
+  });
+
+  it('keeps a code span literal against every other inline rule', () => {
+    // Markdown inside a code span is not markdown — the renderer emits the
+    // bytes verbatim, so no inline rule may reach inside one.
+    const source = '## `a_b_c` and `**not bold**` and `[x](y)`\n';
+    expect(tocIds(source)).toEqual(renderedHeadingIds(source));
+  });
+
+  it('still collapses a link whose label is a code span', () => {
+    // The masking must stay VISIBLE to the link rule as ordinary text,
+    // otherwise `[`code`](url)` stops collapsing to its label.
+    const source = '## Read [`getData`](/api) now\n';
+    expect(tocIds(source)).toEqual(renderedHeadingIds(source));
+  });
+});

packages/plugin-markdown/src/toc.ts

@@ -17,15 +17,51 @@ export interface TocItem {
   id: string
 }
 
-/** Strip the inline-markdown wrappers so the text matches `rehype-slug`'s. */
+/**
+ * A code span's slot while the other inline rules run: private-use sentinels
+ * around its index in `codeSpans`.
+ *
+ * Inert to the emphasis and raw-HTML rules (it carries no `*`, `_` or angle
+ * bracket), but still ORDINARY TEXT to the image and link rules — which is
+ * what keeps ``[`getData`](/api)`` collapsing to its label, exactly as the
+ * renderer does.
+ */
+const SLOT_OPEN = "\uE000"
+const SLOT_CLOSE = "\uE001"
+const SLOT_RE = /\uE000(\d+)\uE001/g
+const SENTINEL_RE = /[\uE000\uE001]/g
+
+/**
+ * Strip the inline-markdown wrappers so the text matches `rehype-slug`'s.
+ *
+ * Code spans are lifted out BEFORE any other rule and put back verbatim at the
+ * end, because markdown inside a code span is not markdown: `rehype-slug` slugs
+ * the rendered heading's flattened text, and a code span contributes its content
+ * as a literal text value inside `<code>`. Rules run over already-unwrapped
+ * code-span text therefore delete characters the anchor is built from — the
+ * raw-HTML rule ate `<type>` out of `` `objectui generate <type> <name>` `` and
+ * the emphasis rules ate the underscores out of `` `a_b_c` `` — so the TOC's
+ * `#id` named a heading anchor that does not exist (objectui#7658).
+ *
+ * The raw-HTML rule itself stays: `remark-rehype` runs without
+ * `allowDangerousHtml`, so it drops raw html nodes and keeps the text they
+ * wrapped, which is what removing the tags reproduces. Sentinels present in the
+ * source are dropped first, so no input can forge a slot.
+ */
 function stripInline(s: string): string {
+  const codeSpans: string[] = []
   return s
+    .replace(SENTINEL_RE, "") // no input can forge a slot
+    .replace(/`([^`]+)`/g, (_match, content: string) => {
+      codeSpans.push(content)
+      return `${SLOT_OPEN}${codeSpans.length - 1}${SLOT_CLOSE}`
+    }) // inline code → an opaque slot
     .replace(/!\[[^\]]*\]\([^)]*\)/g, "") // images
     .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") // links → text
-    .replace(/`([^`]+)`/g, "$1") // inline code
     .replace(/(\*\*|__)(.*?)\1/g, "$2") // bold
     .replace(/(\*|_)(.*?)\1/g, "$2") // italic
     .replace(/<[^>]+>/g, "") // raw html
+    .replace(SLOT_RE, (_match, index: string) => codeSpans[Number(index)]) // code spans, verbatim
     .trim()
 }