Refactor @astrojs/prism, fix Prism component import not working (#4114)

* Upgrade @astrojs/prism to a real package, fix component import not working

* Remove `@astrojs/prism` as a dependency of `astro`

* Update lock file

* Refactor to multiple files

* Oops, can't have astro imports run inside node

* Follow Nate's suggestion on being minors instead of patchs

* Update lockfile

.changeset/slow-spiders-fly.md

@@ -0,0 +1,8 @@
+---
+'astro': patch
+'@astrojs/prism': minor
+'@astrojs/mdx': minor
+'@astrojs/markdown-remark': minor
+---
+
+Refactor `@astrojs/mdx` and `@astrojs/markdown-remark` to use `@astrojs/prism` instead of duplicating the code

packages/astro-prism/Prism.astro

@@ -1,7 +1,5 @@
 ---
-import Prism from 'prismjs';
-import { addAstro } from './internal.mjs';
-import loadLanguages from 'prismjs/components/index.js';
+import { runHighlighterWithAstro } from './dist/highlighter';
 
 export interface Props {
 	class?: string;
@@ -10,40 +8,9 @@ export interface Props {
 }
 
 const { class: className, lang, code } = Astro.props as Props;
-
-let classLanguage = `language-${lang}`;
-
-const languageMap = new Map([['ts', 'typescript']]);
-
-if (lang == null) {
-	console.warn('Prism.astro: No language provided.');
-}
-
-const ensureLoaded = (lang) => {
-	if (lang && !Prism.languages[lang]) {
-		loadLanguages([lang]);
-	}
-};
-
-if (languageMap.has(lang)) {
-	ensureLoaded(languageMap.get(lang));
-} else if (lang === 'astro') {
-	ensureLoaded('typescript');
-	addAstro(Prism);
-} else {
-	ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
-	ensureLoaded(lang);
-}
-
-if (lang && !Prism.languages[lang]) {
-	console.warn(`Unable to load the language: ${lang}`);
-}
-
-const grammar = Prism.languages[lang];
-let html = code;
-if (grammar) {
-	html = Prism.highlight(code, grammar, lang);
-}
+const { classLanguage, html } = runHighlighterWithAstro(lang, code)
 ---
 
-<pre class={[className, classLanguage].join(' ')}><code class={classLanguage}><Fragment set:html={html} /></code></pre>
+<pre class={[className, classLanguage].join(' ')}>
+	<code class={classLanguage}><Fragment set:html={html} /></code>
+</pre>

packages/astro-prism/README.md

@@ -0,0 +1,31 @@
+# @astrojs/prism
+
+Supports Prism highlighting in Astro projects
+
+## Component
+
+This package exports a component to support highlighting inside an Astro file. Example:
+
+```astro
+---
+import { Prism } from "@astrojs/prism"
+---
+
+<Prism lang="js" code={`const foo = 'bar';`} />
+```
+
+## Internal
+
+This package exports a `runHighlighterWithAstro` function to highlight while making sure the Astro language is loaded beforehand
+
+```typescript
+import { runHighlighterWithAstro } from '@astrojs/prism';
+
+runHighlighterWithAstro(`
+  ---
+    const helloAstro = 'Hello, Astro!';
+  ---
+
+  <div>{helloAstro}</div>
+`, 'astro');
+```

packages/astro-prism/package.json

@@ -3,6 +3,7 @@
   "version": "0.6.1",
   "description": "Supports Prism highlighting in Astro projects",
   "author": "withastro",
+  "type": "module",
   "license": "MIT",
   "bugs": "https://github.com/withastro/astro/issues",
   "repository": {
@@ -11,17 +12,28 @@
     "directory": "packages/astro-prism"
   },
   "homepage": "https://astro.build",
-  "main": "index.js",
-  "scripts": {},
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "astro-scripts build \"src/**/*.ts\" && tsc",
+    "build:ci": "astro-scripts build \"src/**/*.ts\"",
+    "dev": "astro-scripts dev \"src/**/*.ts\""
+  },
   "exports": {
-    ".": "./index.js",
-    "./internal": "./internal.mjs"
+    ".": "./dist/index.js",
+    "./Prism.astro": "./Prism.astro",
+    "./dist/highlighter": "./dist/highlighter.js"
   },
-  "types": "./internal.d.ts",
-  "keywords": [],
-  "devDependencies": {
+  "keywords": [
+    "astro",
+    "astro-component"
+  ],
+  "dependencies": {
     "prismjs": "^1.28.0"
   },
+  "devDependencies": {
+    "astro-scripts": "workspace:*",
+    "@types/prismjs": "1.26.0"
+  },
   "engines": {
     "node": "^14.18.0 || >=16.12.0"
   }

packages/astro-prism/src/highlighter.ts

@@ -0,0 +1,42 @@
+import Prism from 'prismjs';
+import loadLanguages from 'prismjs/components/index.js';
+import { addAstro } from './plugin.js';
+
+const languageMap = new Map([['ts', 'typescript']]);
+
+export function runHighlighterWithAstro(lang: string | undefined, code: string) {
+	let classLanguage = `language-${lang}`;
+
+	if (!lang) {
+		lang = 'plaintext';
+	}
+
+	const ensureLoaded = (language: string) => {
+		if (language && !Prism.languages[language]) {
+			loadLanguages([language]);
+		}
+	};
+
+	if (languageMap.has(lang)) {
+		ensureLoaded(languageMap.get(lang)!);
+	} else if (lang === 'astro') {
+		ensureLoaded('typescript');
+		addAstro(Prism);
+	} else {
+		ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
+		ensureLoaded(lang);
+	}
+
+	if (lang && !Prism.languages[lang]) {
+		// eslint-disable-next-line no-console
+		console.warn(`Unable to load the language: ${lang}`);
+	}
+
+	const grammar = Prism.languages[lang];
+	let html = code;
+	if (grammar) {
+		html = Prism.highlight(code, grammar, lang);
+	}
+
+	return { classLanguage, html };
+}

packages/astro-prism/src/index.ts

@@ -0,0 +1,2 @@
+// @ts-expect-error
+export { default as Prism } from '../Prism.astro';

packages/astro-prism/internal.mjs → packages/astro-prism/src/plugin.ts

@@ -1,13 +1,14 @@
-export function addAstro(Prism) {
+export function addAstro(Prism: typeof import('prismjs')) {
 	if (Prism.languages.astro) {
 		return;
 	}
 
-	let scriptLang;
+	let scriptLang: string;
 	if (Prism.languages.typescript) {
 		scriptLang = 'typescript';
 	} else {
 		scriptLang = 'javascript';
+		// eslint-disable-next-line no-console
 		console.warn(
 			'Prism TypeScript language not loaded, Astro scripts will be treated as JavaScript.'
 		);
@@ -19,11 +20,7 @@ export function addAstro(Prism) {
 	let braces = /(?:\{(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])*\})/.source;
 	let spread = /(?:\{<S>*\.{3}(?:[^{}]|<BRACES>)*\})/.source;
 
-	/**
-	 * @param {string} source
-	 * @param {string} [flags]
-	 */
-	function re(source, flags) {
+	function re(source: string, flags?: string) {
 		source = source
 			.replace(/<S>/g, function () {
 				return space;
@@ -40,16 +37,18 @@ export function addAstro(Prism) {
 	spread = re(spread).source;
 
 	Prism.languages.astro = Prism.languages.extend('markup', script);
-	Prism.languages.astro.tag.pattern = re(
+
+	(Prism.languages.astro as any).tag.pattern = re(
 		/<\/?(?:[\w.:-]+(?:<S>+(?:[\w.:$-]+(?:=(?:"(?:\\[^]|[^\\"])*"|'(?:\\[^]|[^\\'])*'|[^\s{'"/>=]+|<BRACES>))?|<SPREAD>))*<S>*\/?)?>/
 			.source
 	);
 
-	Prism.languages.astro.tag.inside['tag'].pattern = /^<\/?[^\s>\/]*/i;
-	Prism.languages.astro.tag.inside['attr-value'].pattern =
+	(Prism.languages.astro as any).tag.inside['tag'].pattern = /^<\/?[^\s>\/]*/i;
+	(Prism.languages.astro as any).tag.inside['attr-value'].pattern =
 		/=(?!\{)(?:"(?:\\[^]|[^\\"])*"|'(?:\\[^]|[^\\'])*'|[^\s'">]+)/i;
-	Prism.languages.astro.tag.inside['tag'].inside['class-name'] = /^[A-Z]\w*(?:\.[A-Z]\w*)*$/;
-	Prism.languages.astro.tag.inside['comment'] = script['comment'];
+	(Prism.languages.astro as any).tag.inside['tag'].inside['class-name'] =
+		/^[A-Z]\w*(?:\.[A-Z]\w*)*$/;
+	(Prism.languages.astro as any).tag.inside['comment'] = script['comment'];
 
 	Prism.languages.insertBefore(
 		'inside',
@@ -60,7 +59,7 @@ export function addAstro(Prism) {
 				inside: Prism.languages.astro,
 			},
 		},
-		Prism.languages.astro.tag
+		(Prism.languages.astro as any).tag
 	);
 
 	Prism.languages.insertBefore(
@@ -80,11 +79,11 @@ export function addAstro(Prism) {
 				alias: `language-${scriptLang}`,
 			},
 		},
-		Prism.languages.astro.tag
+		(Prism.languages.astro as any).tag
 	);
 
 	// The following will handle plain text inside tags
-	let stringifyToken = function (token) {
+	let stringifyToken = function (token: any) {
 		if (!token) {
 			return '';
 		}
@@ -97,8 +96,8 @@ export function addAstro(Prism) {
 		return token.content.map(stringifyToken).join('');
 	};
 
-	let walkTokens = function (tokens) {
-		let openedTags = [];
+	let walkTokens = function (tokens: any) {
+		let openedTags: any[] = [];
 		for (let i = 0; i < tokens.length; i++) {
 			let token = tokens[i];
 
@@ -169,7 +168,7 @@ export function addAstro(Prism) {
 						i--;
 					}
 
-					tokens[i] = new Prism.Token('plain-text', plainText, null, plainText);
+					tokens[i] = new Prism.Token('plain-text', plainText, undefined, plainText);
 				}
 			}
 
@@ -179,7 +178,7 @@ export function addAstro(Prism) {
 		}
 	};
 
-	Prism.hooks.add('after-tokenize', function (env) {
+	Prism.hooks.add('after-tokenize', function (env: any) {
 		if (env.language !== 'astro') {
 			return;
 		}

packages/astro-prism/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "include": ["src"],
+  "compilerOptions": {
+    "allowJs": true,
+    "target": "ES2020",
+    "module": "ES2020",
+    "outDir": "./dist"
+  }
+}

packages/astro/package.json

@@ -87,7 +87,6 @@
     "@astrojs/compiler": "^0.22.1",
     "@astrojs/language-server": "^0.20.0",
     "@astrojs/markdown-remark": "^0.13.0",
-    "@astrojs/prism": "0.6.1",
     "@astrojs/telemetry": "^0.4.1",
     "@astrojs/webapi": "^0.12.0",
     "@babel/core": "^7.18.2",
@@ -124,7 +123,6 @@
     "postcss": "^8.4.14",
     "postcss-load-config": "^3.1.4",
     "preferred-pm": "^3.0.3",
-    "prismjs": "^1.28.0",
     "prompts": "^2.4.2",
     "recast": "^0.20.5",
     "rehype": "^12.0.1",

packages/integrations/mdx/package.json

@@ -37,8 +37,6 @@
     "es-module-lexer": "^0.10.5",
     "github-slugger": "^1.4.0",
     "gray-matter": "^4.0.3",
-    "mdast-util-mdx": "^2.0.0",
-    "prismjs": "^1.28.0",
     "rehype-raw": "^6.1.1",
     "remark-frontmatter": "^4.0.1",
     "remark-gfm": "^3.0.1",

packages/integrations/mdx/src/remark-prism.ts

@@ -1,56 +1,14 @@
-// TODO: discuss extracting this file to @astrojs/prism
-import { addAstro } from '@astrojs/prism/internal';
-import Prism from 'prismjs';
-import loadLanguages from 'prismjs/components/index.js';
+import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
 import { visit } from 'unist-util-visit';
 
-const languageMap = new Map([['ts', 'typescript']]);
-
-function runHighlighter(lang: string, code: string) {
-	let classLanguage = `language-${lang}`;
-
-	if (lang == null) {
-		lang = 'plaintext';
-	}
-
-	const ensureLoaded = (language: string) => {
-		if (language && !Prism.languages[language]) {
-			loadLanguages([language]);
-		}
-	};
-
-	if (languageMap.has(lang)) {
-		ensureLoaded(languageMap.get(lang)!);
-	} else if (lang === 'astro') {
-		ensureLoaded('typescript');
-		addAstro(Prism);
-	} else {
-		ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
-		ensureLoaded(lang);
-	}
-
-	if (lang && !Prism.languages[lang]) {
-		// eslint-disable-next-line no-console
-		console.warn(`Unable to load the language: ${lang}`);
-	}
-
-	const grammar = Prism.languages[lang];
-	let html = code;
-	if (grammar) {
-		html = Prism.highlight(code, grammar, lang);
-	}
-
-	return { classLanguage, html };
-}
-
 /**  */
 export default function remarkPrism() {
 	return (tree: any) =>
 		visit(tree, 'code', (node: any) => {
 			let { lang, value } = node;
 			node.type = 'html';
 
-			let { html, classLanguage } = runHighlighter(lang, value);
+			let { html, classLanguage } = runHighlighterWithAstro(lang, value);
 			let classes = [classLanguage];
 			node.value = `<pre class="${classes.join(
 				' '