feat: switch to xdm (#9)

* feat: switch to xdm * things work now * Update src/index.ts Co-authored-by: Titus <tituswormer@gmail.com> Co-authored-by: Titus <tituswormer@gmail.com> BREAKING CHANGE: We've switched from @mdx-js/js to xdm which is more capable, faster, and requires no runtime! We've also moved to an `xdmOptions` function to customize all the options for compilation rather than simply accepting a `remarkPlugins` option. Check the docs for the latest way to configure things. This is hopefully the last breaking change and we'll enter a pretty stable state from here on out.
kentcdodds · Mar 13, 2021 · c49e165 · c49e165
1 parent 01149e9
commit c49e165
Show file tree

Hide file tree

Showing 10 changed files with 132 additions and 104 deletions.
diff --git a/README.md b/README.md
@@ -26,7 +26,10 @@ get a bundled version of these files to eval in the browser.
 
 This is an async function that will compile and bundle your MDX files and their
 dependencies. It uses [esbuild](https://esbuild.github.io/), so it's VERY fast
-and supports TypeScript files (for the dependencies of your MDX files).
+and supports TypeScript files (for the dependencies of your MDX files). It also
+uses [xdm](https://github.com/wooorm/xdm) which is a more modern and powerful
+MDX compiler with fewer bugs and more features (and no extra runtime
+requirements).
 
 Your source files could be local, in a remote github repo, in a CMS, or wherever
 else and it doesn't matter. All `mdx-bundler` cares about is that you pass it
@@ -151,13 +154,25 @@ file source code. You could get these from the filesystem or from a remote
 database. If your MDX doesn't reference other files (or only imports things from
 `node_modules`), then you can omit this entirely.
 
-#### remarkPlugins
+#### xdmOptions
 
-If you need to customize anything about the MDX compilation you can use remark
-plugins.
+This allows you to modify the built-in xdm configuration (passed to
+xdm.compile). This can be helpful for specifying your own
+remarkPlugins/rehypePlugins.
 
-NOTE: Specifying this will override the default value for frontmatter support so
-if you want to keep that, you'll need to include `remark-frontmatter` yourself.
+```ts
+bundleMDX(mdxString, {
+  xdmOptions(input, options) {
+    // this is the recommended way to add custom remark/rehype plugins:
+    // The syntax might look weird, but it protects you in case we add/remove
+    // plugins in the future.
+    options.remarkPlugins = [...(options.remarkPlugins ?? []), myRemarkPlugin]
+    options.rehypePlugins = [...(options.rehypePlugins ?? []), myRehypePlugin]
+
+    return options
+  },
+})
+```
 
 #### esbuildOptions
 
@@ -166,19 +181,8 @@ takes a function which is passed the default esbuild options and expects an
 options object to be returned.
 
 ```typescript
-// server-side or build-time code that runs in Node:
-import {bundleMDX} from 'mdx-bundler'
-
-const mdxSource = `
-# This is the title
-
-import leftPad from 'left-pad'
-
-<div>{leftPad("Neat demo!", 12, '!')}</div>
-`.trim()
-
-const result = await bundleMDX(mdxSource, {
-  esbuildOptions: options => {
+bundleMDX(mdxSource, {
+  esbuildOptions(options) {
     options.minify = false
     options.target = [
       'es2020',

diff --git a/jest.config.js b/jest.config.js
@@ -1,5 +1,19 @@
 const configs = require('kcd-scripts/config')
 
+const esModules = [
+  'xdm',
+  'unist-util-position-from-estree',
+  'estree-walker',
+  'periscopic',
+  'remark-mdx-frontmatter',
+  'js-yaml',
+].join('|')
+
 module.exports = Object.assign(configs.jest, {
   testEnvironment: './tests/jest.environment.js',
+  transformIgnorePatterns: [`/node_modules/(?!${esModules}).+/`],
+  transform: {
+    '^.+\\.(js|ts|jsx|tsx|cjs|mjs)$': './tests/transform.js',
+  },
+  resolver: 'jest-module-field-resolver',
 })
diff --git a/package.json b/package.json
@@ -8,7 +8,8 @@
     "mdx",
     "bundler",
     "mdx-bundler",
-    "esbuild"
+    "esbuild",
+    "xdm"
   ],
   "author": "Kent C. Dodds <me@kentcdodds.com> (https://kentcdodds.com)",
   "license": "MIT",
@@ -42,18 +43,19 @@
     "@babel/runtime": "^7.13.10",
     "@esbuild-plugins/node-resolve": "0.0.14",
     "@fal-works/esbuild-plugin-global-externals": "^2.1.1",
-    "@mdx-js/mdx": "^1.6.22",
-    "@mdx-js/react": "^1.6.22",
     "esbuild": "0.9.0",
     "gray-matter": "^4.0.2",
-    "remark-frontmatter": "^2.0.0"
+    "remark-frontmatter": "^3.0.0",
+    "remark-mdx-frontmatter": "^1.0.0",
+    "xdm": "^1.5.1"
   },
   "devDependencies": {
     "@testing-library/react": "^11.2.5",
     "@types/jest": "^26.0.20",
     "@types/react": "^17.0.3",
     "@types/react-dom": "^17.0.2",
     "jest-environment-jsdom": "^26.6.2",
+    "jest-module-field-resolver": "0.0.1",
     "kcd-scripts": "^8.1.0",
     "left-pad": "^1.3.0",
     "react": "^17.0.1",

diff --git a/src/__tests__/index.tsx b/src/__tests__/index.tsx
@@ -1,6 +1,5 @@
 import * as React from 'react'
 import {render} from '@testing-library/react'
-import {MDXProvider} from '@mdx-js/react'
 import leftPad from 'left-pad'
 import {bundleMDX} from '..'
 import {getMDXComponent} from '../client'
@@ -60,7 +59,7 @@ export default ({children}) => <div className="sub-dir">{children}</div>
 title: This is frontmatter
 ---
 
-Frontmatter is ignored
+# Frontmatter title: {frontmatter.title}
       `.trim(),
       './data.json': `{"package": "mdx-bundler"}`,
     },
@@ -82,15 +81,15 @@ Frontmatter is ignored
   const Component = getMDXComponent(result.code, {myLeftPad})
 
   const {container} = render(
-    <MDXProvider>
+    <>
       <header>
         <h1>{frontmatter.title}</h1>
         <p>{frontmatter.description}</p>
       </header>
       <main>
         <Component />
       </main>
-    </MDXProvider>,
+    </>,
   )
   expect(container).toMatchInlineSnapshot(`
     <div>
@@ -106,13 +105,19 @@ Frontmatter is ignored
         <h1>
           This is the title
         </h1>
+        
+
+        
+
         <p>
           Here's a 
           <strong>
             neat
           </strong>
            demo:
         </p>
+        
+
         <div>
           $$Neat demo!
           <div
@@ -130,9 +135,10 @@ Frontmatter is ignored
           <div>
             jsx comp
           </div>
-          <p>
-            Frontmatter is ignored
-          </p>
+          <h1>
+            Frontmatter title: 
+            This is frontmatter
+          </h1>
         </div>
       </main>
     </div>
@@ -159,11 +165,7 @@ export default () => leftPad("Neat demo!", 12, '!')
   // this test ensures that *not* passing leftPad as a global here
   // will work because I didn't externalize the left-pad module
   const Component = getMDXComponent(result.code)
-  render(
-    <MDXProvider>
-      <Component />
-    </MDXProvider>,
-  )
+  render(<Component />)
 })
 
 test('gives a handy error when the entry imports a module that cannot be found', async () => {
@@ -179,7 +181,7 @@ import Demo from './demo'
 
   expect(error.message).toMatchInlineSnapshot(`
     "Build failed with 1 error:
-    __mdx_bundler_fake_dir__/index.mdx:1:17: error: [inMemory] Could not resolve \\"./demo\\" in the entry MDX file."
+    __mdx_bundler_fake_dir__/index.mdx:2:17: error: [inMemory] Could not resolve \\"./demo\\" in the entry MDX file."
   `)
 })
 
@@ -217,7 +219,7 @@ import Demo from './demo.blah'
 
   expect(error.message).toMatchInlineSnapshot(`
     "Build failed with 1 error:
-    __mdx_bundler_fake_dir__/index.mdx:1:17: error: [JavaScript plugins] Invalid loader: \\"blah\\" (valid: js, jsx, ts, tsx, css, json, text, base64, dataurl, file, binary)"
+    __mdx_bundler_fake_dir__/index.mdx:2:17: error: [JavaScript plugins] Invalid loader: \\"blah\\" (valid: js, jsx, ts, tsx, css, json, text, base64, dataurl, file, binary)"
   `)
 })
 
@@ -256,11 +258,7 @@ return leftPad(s, 12, '!')
 
   const Component = getMDXComponent(code)
 
-  const {container} = render(
-    <MDXProvider>
-      <Component />
-    </MDXProvider>,
-  )
+  const {container} = render(<Component />)
 
   expect(container).toMatchInlineSnapshot(`
     <div>
@@ -286,11 +284,7 @@ import LeftPad from 'left-pad-js'
 
   const Component = getMDXComponent(code)
 
-  const {container} = render(
-    <MDXProvider>
-      <Component />
-    </MDXProvider>,
-  )
+  const {container} = render(<Component />)
 
   expect(container).toMatchInlineSnapshot(`
     <div>

diff --git a/src/client.ts b/src/client.ts
@@ -1,11 +1,10 @@
 import * as React from 'react'
-import {mdx} from '@mdx-js/react'
 
 function getMDXComponent(
   code: string,
   globals?: Record<string, unknown>,
 ): React.FunctionComponent {
-  const scope = {React, mdx, ...globals}
+  const scope = {React, ...globals}
   // eslint-disable-next-line
   const fn = new Function(...Object.keys(scope), code)
   return fn(...Object.values(scope))

diff --git a/src/index.ts b/src/index.ts
@@ -1,13 +1,14 @@
 import path from 'path'
 import {StringDecoder} from 'string_decoder'
-import {createCompiler} from '@mdx-js/mdx'
 import remarkFrontmatter from 'remark-frontmatter'
+import {remarkMdxFrontmatter} from 'remark-mdx-frontmatter'
 import matter from 'gray-matter'
 import * as esbuild from 'esbuild'
 import type {Plugin, BuildOptions, Loader} from 'esbuild'
 import nodeResolve from '@esbuild-plugins/node-resolve'
 import {globalExternals} from '@fal-works/esbuild-plugin-global-externals'
 import type {ModuleInfo} from '@fal-works/esbuild-plugin-global-externals'
+import type {VFileCompatible, CompileOptions} from 'xdm/lib/compile'
 
 type ESBuildOptions = BuildOptions & {write: false}
 
@@ -36,12 +37,32 @@ type BundleMDXOptions = {
    */
   files?: Record<string, string>
   /**
-   * The remark plugins you want applied when compiling the MDX
+   * This allows you to modify the built-in xdm configuration (passed to xdm.compile).
+   * This can be helpful for specifying your own remarkPlugins/rehypePlugins.
    *
-   * NOTE: Specifying this will override the default value for stripping
-   * frontmatter remark-frontmatter
+   * @param vfileCompatible the path and contents of the mdx file being compiled
+   * @param options the default options which you are expected to modify and return
+   * @returns the options to be passed to xdm.compile
+   *
+   * @example
+   * ```
+   * bundleMDX(mdxString, {
+   *   xdmOptions(input, options) {
+   *     // this is the recommended way to add custom remark/rehype plugins:
+   *     // The syntax might look weird, but it protects you in case we add/remove
+   *     // plugins in the future.
+   *     options.remarkPlugins = [...(options.remarkPlugins ?? []), myRemarkPlugin]
+   *     options.rehypePlugins = [...(options.rehypePlugins ?? []), myRehypePlugin]
+   *
+   *     return options
+   *   }
+   * })
+   * ```
    */
-  remarkPlugins?: Array<unknown>
+  xdmOptions?: (
+    vfileCompatible: VFileCompatible,
+    options: CompileOptions,
+  ) => CompileOptions
   /**
    * This allows you to modify the built-in esbuild configuration. This can be
    * especially helpful for specifying the compilation target.
@@ -92,11 +113,15 @@ async function bundleMDX(
   mdxSource: string,
   {
     files = {},
+    xdmOptions = (vfileCompatible: VFileCompatible, options: CompileOptions) =>
+      options,
     esbuildOptions = (options: ESBuildOptions) => options,
-    remarkPlugins = [remarkFrontmatter],
     globals = {},
   }: BundleMDXOptions = {},
 ) {
+  // xdm is a native ESM, and we're running in a CJS context. This is the
+  // only way to import ESM within CJS
+  const {compile: compileMDX} = await import('xdm')
   // extract the frontmatter
   const {data: frontmatter} = matter(mdxSource)
 
@@ -147,12 +172,21 @@ async function bundleMDX(
 
           switch (fileType) {
             case 'mdx': {
-              // I do not want to take the time to type mdx...
-              // eslint-disable-next-line
-              const result = (await createCompiler({
-                remarkPlugins,
-              }).process(contents)) as {contents: string}
-              return {contents: result.contents, loader: 'jsx'}
+              const vFileCompatible: VFileCompatible = {
+                path: filePath,
+                contents,
+              }
+              const vfile = await compileMDX(
+                vFileCompatible,
+                xdmOptions(vFileCompatible, {
+                  jsx: true,
+                  remarkPlugins: [
+                    remarkFrontmatter,
+                    [remarkMdxFrontmatter, {name: 'frontmatter'}],
+                  ],
+                }),
+              )
+              return {contents: vfile.toString(), loader: 'jsx'}
             }
             default:
               return {contents, loader: fileType as Loader}
@@ -186,8 +220,7 @@ async function bundleMDX(
     bundle: true,
     format: 'iife',
     globalName: 'Component',
-    minify: false,
-    jsxFactory: 'mdx',
+    minify: true,
   })
 
   const bundled = await esbuild.build(buildOptions)

diff --git a/tests/transform.js b/tests/transform.js
@@ -0,0 +1,22 @@
+// because jest + TS + native es modules don't like each other, we have to keep
+// native es modules disabled and compile esm to cjs for the packages that are esm
+// total bummer...
+const path = require('path')
+const {transformSync} = require('esbuild')
+
+const extnameToLoader = {
+  mjs: 'js',
+  cjs: 'js',
+}
+
+module.exports = {
+  process(src, filename) {
+    const ext = path.extname(filename).slice(1)
+    const result = transformSync(src, {
+      loader: extnameToLoader[ext] || ext,
+      sourcemap: true,
+      format: 'cjs',
+    })
+    return result
+  },
+}
diff --git a/tsconfig.json b/tsconfig.json
@@ -2,6 +2,7 @@
   "extends": "./node_modules/kcd-scripts/shared-tsconfig.json",
   "compilerOptions": {
     "target": "ES2020",
+    "module": "CommonJS",
     "typeRoots": ["./types", "./node_modules/@types"]
   }
 }
diff --git a/types/mdx-js__mdx/index.d.ts b/types/mdx-js__mdx/index.d.ts