From f3f62a5a20f4881bb04f65f192df8e1ccf7fb601 Mon Sep 17 00:00:00 2001 From: Bjorn Lu Date: Thu, 7 Sep 2023 22:28:02 +0800 Subject: [PATCH] Refactor mdx remark plugins (#8430) --- .changeset/cuddly-baboons-begin.md | 5 + .changeset/stupid-olives-push.md | 5 + packages/integrations/mdx/package.json | 3 - packages/integrations/mdx/src/plugins.ts | 11 ++- packages/integrations/mdx/src/remark-prism.ts | 18 ---- packages/integrations/mdx/src/remark-shiki.ts | 94 ------------------- packages/markdown/remark/src/index.ts | 18 ++-- packages/markdown/remark/src/remark-prism.ts | 20 +--- .../remark/src/remark-scoped-styles.ts | 18 ---- packages/markdown/remark/src/remark-shiki.ts | 46 +++++---- packages/markdown/remark/src/types.ts | 4 - pnpm-lock.yaml | 51 +--------- 12 files changed, 52 insertions(+), 241 deletions(-) create mode 100644 .changeset/cuddly-baboons-begin.md create mode 100644 .changeset/stupid-olives-push.md delete mode 100644 packages/integrations/mdx/src/remark-prism.ts delete mode 100644 packages/integrations/mdx/src/remark-shiki.ts delete mode 100644 packages/markdown/remark/src/remark-scoped-styles.ts diff --git a/.changeset/cuddly-baboons-begin.md b/.changeset/cuddly-baboons-begin.md new file mode 100644 index 000000000000..bfb45dae0a47 --- /dev/null +++ b/.changeset/cuddly-baboons-begin.md @@ -0,0 +1,5 @@ +--- +'@astrojs/markdown-remark': minor +--- + +Export remarkShiki and remarkPrism plugins diff --git a/.changeset/stupid-olives-push.md b/.changeset/stupid-olives-push.md new file mode 100644 index 000000000000..078e1c69bed5 --- /dev/null +++ b/.changeset/stupid-olives-push.md @@ -0,0 +1,5 @@ +--- +'@astrojs/mdx': patch +--- + +Use exported remarkShiki and remarkPrism plugins from `@astrojs/markdown-remark` diff --git a/packages/integrations/mdx/package.json b/packages/integrations/mdx/package.json index 4d99fed67ce7..5d9296f1ff4f 100644 --- a/packages/integrations/mdx/package.json +++ b/packages/integrations/mdx/package.json @@ -35,7 +35,6 @@ }, "dependencies": { "@astrojs/markdown-remark": "workspace:*", - "@astrojs/prism": "workspace:*", "@mdx-js/mdx": "^2.3.0", "acorn": "^8.10.0", "es-module-lexer": "^1.3.0", @@ -45,10 +44,8 @@ "hast-util-to-html": "^8.0.4", "kleur": "^4.1.4", "rehype-raw": "^6.1.1", - "remark-frontmatter": "^4.0.1", "remark-gfm": "^3.0.1", "remark-smartypants": "^2.0.0", - "shiki": "^0.14.3", "source-map": "^0.7.4", "unist-util-visit": "^4.1.2", "vfile": "^5.3.7" diff --git a/packages/integrations/mdx/src/plugins.ts b/packages/integrations/mdx/src/plugins.ts index 5d7b9b58cc25..a3d9e4ff30ee 100644 --- a/packages/integrations/mdx/src/plugins.ts +++ b/packages/integrations/mdx/src/plugins.ts @@ -1,4 +1,9 @@ -import { rehypeHeadingIds, remarkCollectImages } from '@astrojs/markdown-remark'; +import { + rehypeHeadingIds, + remarkCollectImages, + remarkPrism, + remarkShiki, +} from '@astrojs/markdown-remark'; import { InvalidAstroDataError, safelyGetAstroData, @@ -16,8 +21,6 @@ import { rehypeInjectHeadingsExport } from './rehype-collect-headings.js'; import rehypeMetaString from './rehype-meta-string.js'; import { rehypeOptimizeStatic } from './rehype-optimize-static.js'; import { remarkImageToComponent } from './remark-images-to-component.js'; -import remarkPrism from './remark-prism.js'; -import remarkShiki from './remark-shiki.js'; import { jsToTreeNode } from './utils.js'; // Skip nonessential plugins during performance benchmark runs @@ -112,7 +115,7 @@ export async function getRemarkPlugins(mdxOptions: MdxOptions): Promise - visit(tree, 'code', (node: any) => { - let { lang, value } = node; - node.type = 'html'; - - let { html, classLanguage } = runHighlighterWithAstro(lang, value); - let classes = [classLanguage]; - node.value = `
${html}
`; - return node; - }); -} diff --git a/packages/integrations/mdx/src/remark-shiki.ts b/packages/integrations/mdx/src/remark-shiki.ts deleted file mode 100644 index a241aaa4e5b1..000000000000 --- a/packages/integrations/mdx/src/remark-shiki.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { ShikiConfig } from 'astro'; -import type * as shiki from 'shiki'; -import { getHighlighter } from 'shiki'; -import { visit } from 'unist-util-visit'; - -/** - * getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page, - * cache it here as much as possible. Make sure that your highlighters can be cached, state-free. - * We make this async, so that multiple calls to parse markdown still share the same highlighter. - */ -const highlighterCacheAsync = new Map>(); - -const remarkShiki = async ({ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig) => { - const cacheID: string = typeof theme === 'string' ? theme : theme.name; - let highlighterAsync = highlighterCacheAsync.get(cacheID); - if (!highlighterAsync) { - highlighterAsync = getHighlighter({ theme }).then((hl) => { - hl.setColorReplacements({ - '#000001': 'var(--astro-code-color-text)', - '#000002': 'var(--astro-code-color-background)', - '#000004': 'var(--astro-code-token-constant)', - '#000005': 'var(--astro-code-token-string)', - '#000006': 'var(--astro-code-token-comment)', - '#000007': 'var(--astro-code-token-keyword)', - '#000008': 'var(--astro-code-token-parameter)', - '#000009': 'var(--astro-code-token-function)', - '#000010': 'var(--astro-code-token-string-expression)', - '#000011': 'var(--astro-code-token-punctuation)', - '#000012': 'var(--astro-code-token-link)', - }); - return hl; - }); - highlighterCacheAsync.set(cacheID, highlighterAsync); - } - const highlighter = await highlighterAsync; - - // NOTE: There may be a performance issue here for large sites that use `lang`. - // Since this will be called on every page load. Unclear how to fix this. - for (const lang of langs) { - await highlighter.loadLanguage(lang); - } - - return () => (tree: any) => { - visit(tree, 'code', (node) => { - let lang: string; - - if (typeof node.lang === 'string') { - const langExists = highlighter.getLoadedLanguages().includes(node.lang); - if (langExists) { - lang = node.lang; - } else { - console.warn(`The language "${node.lang}" doesn't exist, falling back to plaintext.`); - lang = 'plaintext'; - } - } else { - lang = 'plaintext'; - } - - let html = highlighter.codeToHtml(node.value, { lang }); - - // Q: Couldn't these regexes match on a user's inputted code blocks? - // A: Nope! All rendered HTML is properly escaped. - // Ex. If a user typed `([\+|\-])/g, - '$2' - ); - } - // Handle code wrapping - // if wrap=null, do nothing. - if (wrap === false) { - html = html.replace(/style="(.*?)"/, 'style="$1; overflow-x: auto;"'); - } else if (wrap === true) { - html = html.replace( - /style="(.*?)"/, - 'style="$1; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;"' - ); - } - - node.type = 'html'; - node.value = html; - node.children = []; - }); - }; -}; - -export default remarkShiki; diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index 43ab885b63a9..c54826bdc656 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -9,9 +9,8 @@ import { toRemarkInitializeAstroData } from './frontmatter-injection.js'; import { loadPlugins } from './load-plugins.js'; import { rehypeHeadingIds } from './rehype-collect-headings.js'; import { remarkCollectImages } from './remark-collect-images.js'; -import remarkPrism from './remark-prism.js'; -import scopedStyles from './remark-scoped-styles.js'; -import remarkShiki from './remark-shiki.js'; +import { remarkPrism } from './remark-prism.js'; +import { remarkShiki } from './remark-shiki.js'; import rehypeRaw from 'rehype-raw'; import rehypeStringify from 'rehype-stringify'; @@ -25,6 +24,8 @@ import { rehypeImages } from './rehype-images.js'; export { rehypeHeadingIds } from './rehype-collect-headings.js'; export { remarkCollectImages } from './remark-collect-images.js'; +export { remarkPrism } from './remark-prism.js'; +export { remarkShiki } from './remark-shiki.js'; export * from './types.js'; export const markdownConfigDefaults: Omit, 'drafts'> = { @@ -61,7 +62,6 @@ export async function renderMarkdown( frontmatter: userFrontmatter = {}, } = opts; const input = new VFile({ value: content, path: fileURL }); - const scopedClassName = opts.$?.scopedClassName; let parser = unified() .use(markdown) @@ -85,18 +85,14 @@ export async function renderMarkdown( }); if (!isPerformanceBenchmark) { - if (scopedClassName) { - parser.use([scopedStyles(scopedClassName)]); - } - if (syntaxHighlight === 'shiki') { - parser.use([await remarkShiki(shikiConfig, scopedClassName)]); + parser.use(remarkShiki, shikiConfig); } else if (syntaxHighlight === 'prism') { - parser.use([remarkPrism(scopedClassName)]); + parser.use(remarkPrism); } // Apply later in case user plugins resolve relative image paths - parser.use([remarkCollectImages]); + parser.use(remarkCollectImages); } parser.use([ diff --git a/packages/markdown/remark/src/remark-prism.ts b/packages/markdown/remark/src/remark-prism.ts index 6147d9ee9cf4..a3f476d6e4ab 100644 --- a/packages/markdown/remark/src/remark-prism.ts +++ b/packages/markdown/remark/src/remark-prism.ts @@ -1,31 +1,19 @@ import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter'; import { visit } from 'unist-util-visit'; +import type { RemarkPlugin } from './types.js'; -type MaybeString = string | null | undefined; - -/** */ -function transformer(className: MaybeString) { +export function remarkPrism(): ReturnType { return function (tree: any) { - const visitor = (node: any) => { + visit(tree, 'code', (node) => { let { lang, value } = node; node.type = 'html'; let { html, classLanguage } = runHighlighterWithAstro(lang, value); let classes = [classLanguage]; - if (className) { - classes.push(className); - } node.value = `
${html}
`; return node; - }; - return visit(tree, 'code', visitor); + }); }; } - -function plugin(className: MaybeString) { - return transformer.bind(null, className); -} - -export default plugin; diff --git a/packages/markdown/remark/src/remark-scoped-styles.ts b/packages/markdown/remark/src/remark-scoped-styles.ts deleted file mode 100644 index ba8780bb7026..000000000000 --- a/packages/markdown/remark/src/remark-scoped-styles.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { visit } from 'unist-util-visit'; -const noVisit = new Set(['root', 'html', 'text']); - -/** */ -export default function scopedStyles(className: string) { - const visitor = (node: any) => { - if (noVisit.has(node.type)) return; - - const { data } = node; - let currentClassName = data?.hProperties?.class ?? ''; - node.data = node.data || {}; - node.data.hProperties = node.data.hProperties || {}; - node.data.hProperties.class = `${className} ${currentClassName}`.trim(); - - return node; - }; - return () => (tree: any) => visit(tree, visitor); -} diff --git a/packages/markdown/remark/src/remark-shiki.ts b/packages/markdown/remark/src/remark-shiki.ts index 77cbf16c6515..6cd3861e5cf1 100644 --- a/packages/markdown/remark/src/remark-shiki.ts +++ b/packages/markdown/remark/src/remark-shiki.ts @@ -1,7 +1,7 @@ import type * as shiki from 'shiki'; import { getHighlighter } from 'shiki'; import { visit } from 'unist-util-visit'; -import type { ShikiConfig } from './types.js'; +import type { RemarkPlugin, ShikiConfig } from './types.js'; /** * getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page, @@ -10,10 +10,11 @@ import type { ShikiConfig } from './types.js'; */ const highlighterCacheAsync = new Map>(); -const remarkShiki = async ( - { langs = [], theme = 'github-dark', wrap = false }: ShikiConfig, - scopedClassName?: string | null -) => { +export function remarkShiki({ + langs = [], + theme = 'github-dark', + wrap = false, +}: ShikiConfig = {}): ReturnType { const cacheID: string = typeof theme === 'string' ? theme : theme.name; let highlighterAsync = highlighterCacheAsync.get(cacheID); if (!highlighterAsync) { @@ -35,15 +36,22 @@ const remarkShiki = async ( }); highlighterCacheAsync.set(cacheID, highlighterAsync); } - const highlighter = await highlighterAsync; - // NOTE: There may be a performance issue here for large sites that use `lang`. - // Since this will be called on every page load. Unclear how to fix this. - for (const lang of langs) { - await highlighter.loadLanguage(lang); - } + let highlighter: shiki.Highlighter; + + return async (tree: any) => { + // Lazily assign the highlighter as async can only happen within this function, + // and not on `remarkShiki` directly. + if (!highlighter) { + highlighter = await highlighterAsync!; + + // NOTE: There may be a performance issue here for large sites that use `lang`. + // Since this will be called on every page load. Unclear how to fix this. + for (const lang of langs) { + await highlighter.loadLanguage(lang); + } + } - return () => (tree: any) => { visit(tree, 'code', (node) => { let lang: string; @@ -69,10 +77,7 @@ const remarkShiki = async ( // <span class="line" // Replace "shiki" class naming with "astro" and add "is:raw". - html = html.replace( - /
/g, `;
 }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 3838f2b95545..605f793ec119 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -3984,9 +3984,6 @@ importers:
       '@astrojs/markdown-remark':
         specifier: workspace:*
         version: link:../../markdown/remark
-      '@astrojs/prism':
-        specifier: workspace:*
-        version: link:../../astro-prism
       '@mdx-js/mdx':
         specifier: ^2.3.0
         version: 2.3.0
@@ -4014,18 +4011,12 @@ importers:
       rehype-raw:
         specifier: ^6.1.1
         version: 6.1.1
-      remark-frontmatter:
-        specifier: ^4.0.1
-        version: 4.0.1
       remark-gfm:
         specifier: ^3.0.1
         version: 3.0.1
       remark-smartypants:
         specifier: ^2.0.0
         version: 2.0.0
-      shiki:
-        specifier: ^0.14.3
-        version: 0.14.3
       source-map:
         specifier: ^0.7.4
         version: 0.7.4
@@ -4083,7 +4074,7 @@ importers:
         version: 4.0.3
       rehype-pretty-code:
         specifier: ^0.10.0
-        version: 0.10.0(shiki@0.14.3)
+        version: 0.10.0
       remark-math:
         specifier: ^5.1.1
         version: 5.1.1
@@ -11595,12 +11586,6 @@ packages:
     dependencies:
       reusify: 1.0.4
 
-  /fault@2.0.1:
-    resolution: {integrity: sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==}
-    dependencies:
-      format: 0.2.2
-    dev: false
-
   /fenceparser@1.1.1:
     resolution: {integrity: sha512-VdkTsK7GWLT0VWMK5S5WTAPn61wJ98WPFwJiRHumhg4ESNUO/tnkU8bzzzc62o6Uk1SVhuZFLnakmDA4SGV7wA==}
     engines: {node: '>=12'}
@@ -11701,11 +11686,6 @@ packages:
       combined-stream: 1.0.8
       mime-types: 2.1.35
 
-  /format@0.2.2:
-    resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==}
-    engines: {node: '>=0.4.x'}
-    dev: false
-
   /formdata-polyfill@4.0.10:
     resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
     engines: {node: '>=12.20.0'}
@@ -13329,14 +13309,6 @@ packages:
     transitivePeerDependencies:
       - supports-color
 
-  /mdast-util-frontmatter@1.0.1:
-    resolution: {integrity: sha512-JjA2OjxRqAa8wEG8hloD0uTU0kdn8kbtOWpPP94NBkfAlbxn4S8gCGf/9DwFtEeGPXrDcNXdiDjVaRdUFqYokw==}
-    dependencies:
-      '@types/mdast': 3.0.12
-      mdast-util-to-markdown: 1.5.0
-      micromark-extension-frontmatter: 1.1.0
-    dev: false
-
   /mdast-util-gfm-autolink-literal@1.0.3:
     resolution: {integrity: sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==}
     dependencies:
@@ -13599,15 +13571,6 @@ packages:
       micromark-util-types: 1.0.2
       uvu: 0.5.6
 
-  /micromark-extension-frontmatter@1.1.0:
-    resolution: {integrity: sha512-0nLelmvXR5aZ+F2IL6/Ed4cDnHLpL/VD/EELKuclsTWHrLI8UgxGHEmeoumeX2FXiM6z2WrBIOEcbKUZR8RYNg==}
-    dependencies:
-      fault: 2.0.1
-      micromark-util-character: 1.1.0
-      micromark-util-symbol: 1.0.1
-      micromark-util-types: 1.0.2
-    dev: false
-
   /micromark-extension-gfm-autolink-literal@1.0.4:
     resolution: {integrity: sha512-WCssN+M9rUyfHN5zPBn3/f0mIA7tqArHL/EKbv3CZK+LT2rG77FEikIQEqBkv46fOqXQK4NEW/Pc7Z27gshpeg==}
     dependencies:
@@ -15584,7 +15547,7 @@ packages:
       unified: 10.1.2
     dev: false
 
-  /rehype-pretty-code@0.10.0(shiki@0.14.3):
+  /rehype-pretty-code@0.10.0:
     resolution: {integrity: sha512-qCD071Y+vUxEy9yyrATPk2+W9q7qCbzZgtc9suZhu75bmRQvOlBhJt4d3WvqSMTamkKoFkvqtCjyAk+ggH+aXQ==}
     engines: {node: '>=16'}
     peerDependencies:
@@ -15593,7 +15556,6 @@ packages:
       '@types/hast': 2.3.5
       hash-obj: 4.0.0
       parse-numeric-range: 1.3.0
-      shiki: 0.14.3
     dev: true
 
   /rehype-raw@6.1.1:
@@ -15652,15 +15614,6 @@ packages:
     dependencies:
       unist-util-visit: 1.4.1
 
-  /remark-frontmatter@4.0.1:
-    resolution: {integrity: sha512-38fJrB0KnmD3E33a5jZC/5+gGAC2WKNiPw1/fdXJvijBlhA7RCsvJklrYJakS0HedninvaCYW8lQGf9C918GfA==}
-    dependencies:
-      '@types/mdast': 3.0.12
-      mdast-util-frontmatter: 1.0.1
-      micromark-extension-frontmatter: 1.1.0
-      unified: 10.1.2
-    dev: false
-
   /remark-gfm@3.0.1:
     resolution: {integrity: sha512-lEFDoi2PICJyNrACFOfDD3JlLkuSbOa5Wd8EPt06HUdptv8Gn0bxYTdbU/XXQ3swAPkEaGxxPN9cbnMHvVu1Ig==}
     dependencies: