[MDX] Add Prism and Shiki support (#4002)

* deps: add rehype-prism, shiki, rehype-pretty-code

* wip: apply rehype plugins depending on config

* wip: cherry-pick jsx-runtime fix?

* deps: rehype-pretty-code -> shiki-twoslash, add rehype-raw

* wip: add jsx-runtime fix

* feat: get shiki working!

* deps: add @astrojs/prism, prismjs, unist-util-visit

* feat: add prism support

* example: add small syntax highlight demo to with-mdx

* deps: remove rehype-prism

* chore: remove unused async

* chore: add .test.js to all mdx tests

* test: shiki, shikiConfig, prism

* fix: remove "is:raw" from prism output

* docs: add syntax highlighting section

* chore: add changeset

* nit: "Shiki config" -> Shiki config

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

* Revert "wip: add jsx-runtime fix"

This reverts commit 07f4528.

* docs: link to integration README from example

Co-authored-by: Nate Moore <nate@astro.build>
Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

.changeset/real-camels-roll.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/mdx': minor
+---
+
+Support Prism and Shiki syntax highlighting based on project config

examples/with-mdx/src/pages/index.mdx

@@ -15,3 +15,15 @@ Written by: {new Intl.ListFormat('en').format(authors.map(d => d.name))}.
 Published on: {new Intl.DateTimeFormat('en', {dateStyle: 'long'}).format(published)}.
 
 <Counter client:idle>This is a **counter**!</Counter>
+
+## Syntax highlighting
+
+We also support syntax highlighting in MDX out-of-the-box! This example uses our default [Shiki theme](https://github.com/shikijs/shiki). See the [MDX integration docs](https://docs.astro.build/en/guides/integrations-guide/mdx/#syntax-highlighting) for configuration options.
+
+```astro
+---
+const weSupportAstro = true
+---
+
+<h1>Hey, what theme is that? Looks nice!</h1>
+```

packages/integrations/mdx/README.md

@@ -134,6 +134,42 @@ const posts = await Astro.glob('./*.mdx');
 ))}
 ```
 
+### Syntax highlighting
+
+The MDX integration respects [your project's `markdown.syntaxHighlight` configuration](https://docs.astro.build/en/guides/markdown-content/#syntax-highlighting).
+
+We will highlight your code blocks with [Shiki](https://github.com/shikijs/shiki) by default [using Shiki twoslash](https://shikijs.github.io/twoslash/). You can customize [this remark plugin](https://www.npmjs.com/package/remark-shiki-twoslash) using the `markdown.shikiConfig` option in your `astro.config`. For example, you can apply a different built-in theme like so:
+
+```js
+// astro.config.mjs
+export default {
+  markdown: {
+    shikiConfig: {
+      theme: 'dracula',
+    },
+  },
+  integrations: [mdx()],
+}
+```
+
+Visit [our Shiki configuration docs](https://docs.astro.build/en/guides/markdown-content/#shiki-configuration) for more on using Shiki with Astro.
+
+#### Switch to Prism
+
+You can also use the [Prism](https://prismjs.com/) syntax highlighter by setting `markdown.syntaxHighlight` to `'prism'` in your `astro.config` like so:
+
+```js
+// astro.config.mjs
+export default {
+  markdown: {
+    syntaxHighlight: 'prism',
+  },
+  integrations: [mdx()],
+}
+```
+
+This applies a minimal Prism renderer with added support for `astro` code blocks. Visit [our "Prism configuration" docs](https://docs.astro.build/en/guides/markdown-content/#prism-configuration) for more on using Prism with Astro.
+
 ## Configuration
 
 <details>

packages/integrations/mdx/package.json

@@ -30,12 +30,19 @@
     "test": "mocha --exit --timeout 20000"
   },
   "dependencies": {
+    "@astrojs/prism": "^0.6.1",
+    "@mdx-js/mdx": "^2.1.2",
     "@mdx-js/rollup": "^2.1.1",
     "es-module-lexer": "^0.10.5",
-    "remark-frontmatter": "^4.0.1",
+    "prismjs": "^1.28.0",
+    "rehype-raw": "^6.1.1",
     "remark-gfm": "^3.0.1",
-    "remark-mdx-frontmatter": "^2.0.2",
-    "remark-smartypants": "^2.0.0"
+    "remark-shiki-twoslash": "^3.1.0",
+    "remark-smartypants": "^2.0.0",
+    "shiki": "^0.10.1",
+    "unist-util-visit": "^4.1.0",
+    "remark-frontmatter": "^4.0.1",
+    "remark-mdx-frontmatter": "^2.0.2"
   },
   "devDependencies": {
     "@types/chai": "^4.3.1",

packages/integrations/mdx/src/index.ts

@@ -1,11 +1,15 @@
-import mdxPlugin, { Options as MdxRollupPluginOptions } from '@mdx-js/rollup';
+import type { RemarkMdxFrontmatterOptions } from 'remark-mdx-frontmatter';
 import type { AstroIntegration } from 'astro';
+import remarkShikiTwoslash from 'remark-shiki-twoslash';
+import { nodeTypes } from '@mdx-js/mdx';
+import rehypeRaw from 'rehype-raw';
+import mdxPlugin, { Options as MdxRollupPluginOptions } from '@mdx-js/rollup';
 import { parse as parseESM } from 'es-module-lexer';
 import remarkFrontmatter from 'remark-frontmatter';
 import remarkGfm from 'remark-gfm';
-import type { RemarkMdxFrontmatterOptions } from 'remark-mdx-frontmatter';
 import remarkMdxFrontmatter from 'remark-mdx-frontmatter';
 import remarkSmartypants from 'remark-smartypants';
+import remarkPrism from './remark-prism.js';
 import { getFileInfo } from './utils.js';
 
 type WithExtends<T> = T | { extends: T };
@@ -23,7 +27,10 @@ type MdxOptions = {
 
 const DEFAULT_REMARK_PLUGINS = [remarkGfm, remarkSmartypants];
 
-function handleExtends<T>(config: WithExtends<T[] | undefined>, defaults: T[] = []): T[] {
+function handleExtends<T>(
+	config: WithExtends<T[] | undefined>,
+	defaults: T[] = [],
+): T[] {
 	if (Array.isArray(config)) return config;
 
 	return [...defaults, ...(config?.extends ?? [])];
@@ -35,27 +42,43 @@ export default function mdx(mdxOptions: MdxOptions = {}): AstroIntegration {
 		hooks: {
 			'astro:config:setup': ({ updateConfig, config, addPageExtension, command }: any) => {
 				addPageExtension('.mdx');
+				let remarkPlugins = handleExtends(mdxOptions.remarkPlugins, DEFAULT_REMARK_PLUGINS);
+				let rehypePlugins = handleExtends(mdxOptions.rehypePlugins);
+
+				if (config.markdown.syntaxHighlight === 'shiki') {
+					remarkPlugins.push([
+						// Default export still requires ".default" chaining for some reason
+						// Workarounds tried:
+						// - "import * as remarkShikiTwoslash"
+						// - "import { default as remarkShikiTwoslash }"
+						(remarkShikiTwoslash as any).default,
+						config.markdown.shikiConfig,
+					]);
+					rehypePlugins.push([rehypeRaw, { passThrough: nodeTypes }]);
+				}
+
+				if (config.markdown.syntaxHighlight === 'prism') {
+					remarkPlugins.push(remarkPrism);
+					rehypePlugins.push([rehypeRaw, { passThrough: nodeTypes }]);
+				}
+
+				remarkPlugins.push(remarkFrontmatter);
+				remarkPlugins.push([
+					remarkMdxFrontmatter,
+					{
+						name: 'frontmatter',
+						...mdxOptions.frontmatterOptions,
+					},
+				]);
+
 				updateConfig({
 					vite: {
 						plugins: [
 							{
 								enforce: 'pre',
 								...mdxPlugin({
-									remarkPlugins: [
-										...handleExtends(mdxOptions.remarkPlugins, DEFAULT_REMARK_PLUGINS),
-										// Frontmatter plugins should always be applied!
-										// We can revisit this if a strong use case to *remove*
-										// YAML frontmatter via config is reported.
-										remarkFrontmatter,
-										[
-											remarkMdxFrontmatter,
-											{
-												name: 'frontmatter',
-												...mdxOptions.frontmatterOptions,
-											},
-										],
-									],
-									rehypePlugins: handleExtends(mdxOptions.rehypePlugins),
+									remarkPlugins,
+									rehypePlugins,
 									jsx: true,
 									jsxImportSource: 'astro',
 									// Note: disable `.md` support

packages/integrations/mdx/src/remark-prism.ts

@@ -0,0 +1,59 @@
+// TODO: discuss extracting this file to @astrojs/prism
+import { addAstro } from '@astrojs/prism/internal';
+import Prism from 'prismjs';
+import loadLanguages from 'prismjs/components/index.js';
+import { visit } from 'unist-util-visit';
+
+const languageMap = new Map([['ts', 'typescript']]);
+
+function runHighlighter(lang: string, code: string) {
+	let classLanguage = `language-${lang}`;
+
+	if (lang == null) {
+		lang = 'plaintext';
+	}
+
+	const ensureLoaded = (language: string) => {
+		if (language && !Prism.languages[language]) {
+			loadLanguages([language]);
+		}
+	};
+
+	if (languageMap.has(lang)) {
+		ensureLoaded(languageMap.get(lang)!);
+	} else if (lang === 'astro') {
+		ensureLoaded('typescript');
+		addAstro(Prism);
+	} else {
+		ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
+		ensureLoaded(lang);
+	}
+
+	if (lang && !Prism.languages[lang]) {
+		// eslint-disable-next-line no-console
+		console.warn(`Unable to load the language: ${lang}`);
+	}
+
+	const grammar = Prism.languages[lang];
+	let html = code;
+	if (grammar) {
+		html = Prism.highlight(code, grammar, lang);
+	}
+
+	return { classLanguage, html };
+}
+
+/**  */
+export default function remarkPrism() {
+	return (tree: any) => visit(tree, 'code', (node: any) => {
+		let { lang, value } = node;
+		node.type = 'html';
+
+		let { html, classLanguage } = runHighlighter(lang, value);
+		let classes = [classLanguage];
+		node.value = `<pre class="${classes.join(
+			' '
+		)}"><code class="${classLanguage}">${html}</code></pre>`;
+		return node;
+	});
+}

packages/integrations/mdx/test/fixtures/mdx-syntax-hightlighting/src/pages/index.mdx

@@ -0,0 +1,9 @@
+# Syntax highlighting
+
+```astro
+---
+const handlesAstroSyntax = true
+---
+
+<h1>{handlesAstroSyntax}</h1>
+```

packages/integrations/mdx/test/mdx-syntax-highlighting.test.js

@@ -0,0 +1,67 @@
+import mdx from '@astrojs/mdx';
+
+import { expect } from 'chai';
+import { parseHTML } from 'linkedom';
+import { loadFixture } from '../../../astro/test/test-utils.js';
+
+const FIXTURE_ROOT = new URL('./fixtures/mdx-syntax-hightlighting/', import.meta.url);
+
+describe('MDX syntax highlighting', () => {
+	describe('shiki', () => {
+		it('works', async () => {
+			const fixture = await loadFixture({
+				root: FIXTURE_ROOT,
+				markdown: {
+					syntaxHighlight: 'shiki',
+				},
+				integrations: [mdx()],
+			});
+			await fixture.build();
+
+			const html = await fixture.readFile('/index.html');
+			const { document } = parseHTML(html);
+
+			const shikiCodeBlock = document.querySelector('pre.shiki');
+			expect(shikiCodeBlock).to.not.be.null;
+		});
+
+		it('respects markdown.shikiConfig.theme', async () => {
+			const fixture = await loadFixture({
+				root: FIXTURE_ROOT,
+				markdown: {
+					syntaxHighlight: 'shiki',
+					shikiConfig: {
+						theme: 'dracula',
+					},
+				},
+				integrations: [mdx()],
+			});
+			await fixture.build();
+
+			const html = await fixture.readFile('/index.html');
+			const { document } = parseHTML(html);
+
+			const shikiCodeBlock = document.querySelector('pre.shiki.dracula');
+			expect(shikiCodeBlock).to.not.be.null;
+		});
+	});
+
+	describe('prism', () => {
+		it('works', async () => {
+			const fixture = await loadFixture({
+				root: FIXTURE_ROOT,
+				markdown: {
+					syntaxHighlight: 'prism',
+				},
+				integrations: [mdx()],
+			});
+			await fixture.build();
+
+			const html = await fixture.readFile('/index.html');
+			const { document } = parseHTML(html);
+
+			const prismCodeBlock = document.querySelector('pre.language-astro');
+			expect(prismCodeBlock).to.not.be.null;
+		});
+	});
+});