Add format to support MDX and plain markdown

Add `format` to support MDX and plain markdown

* Default to detecting whether input is `mdx` or `md` based on whether the given
  file has an extension in `mdExtensions` and to handle both `mdExtensions` and
  `mdxExtensions` in integrations
* Support `format: 'md'` to treat input as markdown and to handle `mdExtensions`
  in integrations
* Support `format: 'mdx'` to treat input as MDX and to handle `mdxExtensions`
  in integrations
* Switch which elements will turn into configurable components, previously that
  was everything that was written in the source markdown, now this is everything
  not explicitly written as JSX
* Add tests for integration with `rehype-raw`
* Fix docs on `evaluate` options

Closes GH-17.
Closes GH-19.

Reviewed-by: Christian Murphy <christian.murphy.42@gmail.com>
Reviewed-by: Remco Haszing <remcohaszing@gmail.com>

index.js

@@ -1,3 +1,4 @@
 export {createProcessor} from './lib/core.js'
 export {compile, compileSync} from './lib/compile.js'
 export {evaluate, evaluateSync} from './lib/evaluate.js'
+export {nodeTypes} from './lib/node-types.js'

lib/compile.js

@@ -1,36 +1,39 @@
-import vfile from 'vfile'
 import {createProcessor} from './core.js'
+import {resolveFileAndOptions} from './util/resolve-file-and-options.js'
 
 /**
  * @typedef {import('vfile').VFileCompatible} VFileCompatible
  * @typedef {import('vfile').VFile} VFile
- * @typedef {import('./core').ProcessorOptions} ProcessorOptions
+ * @typedef {import('./core').PluginOptions} PluginOptions
+ * @typedef {import('./core').BaseProcessorOptions} BaseProcessorOptions
+ * @typedef {Omit<BaseProcessorOptions, 'format'>} CoreProcessorOptions
+ *
+ * @typedef ExtraOptions
+ * @property {'detect' | 'mdx' | 'md'} [format='detect'] Format of `file`
+ *
+ * @typedef {CoreProcessorOptions & PluginOptions & ExtraOptions} CompileOptions
  */
 
 /**
  * Compile MDX to JS.
  *
  * @param {VFileCompatible} vfileCompatible MDX document to parse (`string`, `Buffer`, `vfile`, anything that can be given to `vfile`)
- * @param {ProcessorOptions} [options]
+ * @param {CompileOptions} [compileOptions]
  * @return {Promise<VFile>}
  */
-export function compile(vfileCompatible, options) {
-  var {settings, file} = configure(vfileCompatible, options)
-  return createProcessor(settings).process(file)
+export function compile(vfileCompatible, compileOptions) {
+  var {file, options} = resolveFileAndOptions(vfileCompatible, compileOptions)
+  return createProcessor(options).process(file)
 }
 
 /**
  * Synchronously compile MDX to JS.
  *
  * @param {VFileCompatible} vfileCompatible MDX document to parse (`string`, `Buffer`, `vfile`, anything that can be given to `vfile`)
- * @param {ProcessorOptions} [options]
+ * @param {CompileOptions} [compileOptions]
  * @return {VFile}
  */
-export function compileSync(vfileCompatible, options) {
-  var {settings, file} = configure(vfileCompatible, options)
-  return createProcessor(settings).processSync(file)
-}
-
-function configure(vfileCompatible, options) {
-  return {file: vfile(vfileCompatible), settings: options || {}}
+export function compileSync(vfileCompatible, compileOptions) {
+  var {file, options} = resolveFileAndOptions(vfileCompatible, compileOptions)
+  return createProcessor(options).processSync(file)
 }

lib/core.js

@@ -7,7 +7,9 @@ import {recmaJsxRewrite} from './plugin/recma-jsx-rewrite.js'
 import {recmaStringify} from './plugin/recma-stringify.js'
 import {rehypeMarkAndUnravel} from './plugin/rehype-mark-and-unravel.js'
 import {rehypeRecma} from './plugin/rehype-recma.js'
+import {rehypeRemoveRaw} from './plugin/rehype-remove-raw.js'
 import {remarkMdx} from './plugin/remark-mdx.js'
+import {nodeTypes} from './node-types.js'
 
 /**
  * @typedef {import('unified').Processor} Processor
@@ -18,6 +20,9 @@ import {remarkMdx} from './plugin/remark-mdx.js'
  *
  * @typedef BaseProcessorOptions
  * @property {boolean} [jsx=false] Whether to keep JSX
+ * @property {'mdx' | 'md'} [format='mdx'] Format of the files to be processed
+ * @property {string[]} [mdExtensions] Extensions (with `.`) for markdown
+ * @property {string[]} [mdxExtensions] Extensions (with `.`) for MDX
  * @property {PluggableList} [recmaPlugins] List of recma (esast, JavaScript) plugins
  * @property {PluggableList} [remarkPlugins] List of remark (mdast, markdown) plugins
  * @property {PluggableList} [rehypePlugins] List of rehype (hast, HTML) plugins
@@ -42,6 +47,7 @@ export function createProcessor(options = {}) {
   var {
     _contain,
     jsx,
+    format,
     providerImportSource,
     recmaPlugins,
     rehypePlugins,
@@ -50,25 +56,22 @@ export function createProcessor(options = {}) {
     ...rest
   } = options
 
+  // @ts-ignore Sure the types prohibit it but what if someone does it anyway?
+  if (format === 'detect') {
+    throw new Error(
+      "Incorrect `format: 'detect'`: `createProcessor` can support either `md` or `mdx`; it does not support detecting the format"
+    )
+  }
+
   return (
     unified()
       .use(remarkParse)
-      .use(remarkMdx)
+      .use(format === 'md' ? undefined : remarkMdx)
       .use(remarkPlugins)
-      .use(remarkRehype, {
-        allowDangerousHtml: true,
-        // List of node types made by `mdast-util-mdx`, which have to be passed
-        // through untouched from the mdast tree to the hast tree.
-        passThrough: [
-          'mdxFlowExpression',
-          'mdxJsxFlowElement',
-          'mdxJsxTextElement',
-          'mdxTextExpression',
-          'mdxjsEsm'
-        ]
-      })
+      .use(remarkRehype, {allowDangerousHtml: true, passThrough: nodeTypes})
       .use(rehypeMarkAndUnravel)
       .use(rehypePlugins)
+      .use(format === 'md' ? rehypeRemoveRaw : undefined)
       .use(rehypeRecma)
       .use(recmaDocument, {...rest, _contain})
       // @ts-ignore recma transformer uses an esast node rather than a unist node

lib/evaluate.js

@@ -1,22 +1,10 @@
 import {compile, compileSync} from './compile.js'
 import {run, runSync} from './run.js'
+import {resolveEvaluateOptions} from './util/resolve-evaluate-options.js'
 
 /**
  * @typedef {import('vfile').VFileCompatible} VFileCompatible
- * @typedef {import('./core.js').BaseProcessorOptions} BaseProcessorOptions
- *
- * @typedef RunnerOptions
- * @property {*} Fragment Symbol to use for fragments
- * @property {*} jsx Function to generate an element with static children
- * @property {*} jsxs Function to generate an element with dynamic children
- * @property {*} [useMDXComponents] Function to get `MDXComponents` from context
- *
- * @typedef {Omit<BaseProcessorOptions, 'jsx' | '_contain' | '_baseUrl'> } ProcessorOptions
- *
- * @typedef ExtraOptions
- * @property {string} baseUrl URL to resolve imports from (typically: pass `import.meta.url`)
- *
- * @typedef {ProcessorOptions & RunnerOptions & ExtraOptions} EvaluateOptions
+ * @typedef {import('./util/resolve-evaluate-options.js').EvaluateOptions} EvaluateOptions
  *
  * @typedef {{[name: string]: any}} ComponentMap
  * @typedef {{[props: string]: any, components?: ComponentMap}} MDXContentProps
@@ -26,59 +14,25 @@ import {run, runSync} from './run.js'
 /**
  * Evaluate MDX.
  *
- * @param {VFileCompatible} file MDX document to parse (`string`, `Buffer`, `vfile`, anything that can be given to `vfile`)
- * @param {EvaluateOptions} options
+ * @param {VFileCompatible} vfileCompatible MDX document to parse (`string`, `Buffer`, `vfile`, anything that can be given to `vfile`)
+ * @param {EvaluateOptions} evaluateOptions
  * @return {Promise<ExportMap>}
  */
-export async function evaluate(file, options) {
-  var config = splitEvaluateOptions(options)
+export async function evaluate(vfileCompatible, evaluateOptions) {
+  var {compiletime, runtime} = resolveEvaluateOptions(evaluateOptions)
   // V8 on Erbium.
   /* c8 ignore next 2 */
-  return run(await compile(file, config.compile), config.run)
+  return run(await compile(vfileCompatible, compiletime), runtime)
 }
 
 /**
  * Synchronously evaluate MDX.
  *
- * @param {VFileCompatible} file MDX document to parse (`string`, `Buffer`, `vfile`, anything that can be given to `vfile`)
- * @param {EvaluateOptions} options
+ * @param {VFileCompatible} vfileCompatible MDX document to parse (`string`, `Buffer`, `vfile`, anything that can be given to `vfile`)
+ * @param {EvaluateOptions} evaluateOptions
  * @return {ExportMap}
  */
-export function evaluateSync(file, options) {
-  var config = splitEvaluateOptions(options)
-  return runSync(compileSync(file, config.compile), config.run)
-}
-
-/**
- * Split processor/compiler options from runner options.
- *
- * @param {EvaluateOptions} options
- */
-export function splitEvaluateOptions(options) {
-  var {
-    Fragment,
-    jsx,
-    jsxs,
-    recmaPlugins,
-    rehypePlugins,
-    remarkPlugins,
-    useMDXComponents,
-    baseUrl
-  } = options || {}
-
-  if (!Fragment) throw new Error('Expected `Fragment` given to `evaluate`')
-  if (!jsx) throw new Error('Expected `jsx` given to `evaluate`')
-  if (!jsxs) throw new Error('Expected `jsxs` given to `evaluate`')
-
-  return {
-    compile: {
-      _contain: true,
-      _baseUrl: baseUrl,
-      providerImportSource: useMDXComponents ? '#' : undefined,
-      recmaPlugins,
-      rehypePlugins,
-      remarkPlugins
-    },
-    run: {Fragment, jsx, jsxs, useMDXComponents}
-  }
+export function evaluateSync(vfileCompatible, evaluateOptions) {
+  var {compiletime, runtime} = resolveEvaluateOptions(evaluateOptions)
+  return runSync(compileSync(vfileCompatible, compiletime), runtime)
 }

lib/integration/esbuild.js

@@ -1,6 +1,7 @@
 import vfile from 'vfile'
 import {promises as fs} from 'fs'
-import {createProcessor} from '../core.js'
+import {createFormatAwareProcessors} from '../util/create-format-aware-processors.js'
+import {extnamesToRegex} from '../util/extnames-to-regex.js'
 
 var eol = /\r\n|\r|\n|\u2028|\u2029/g
 
@@ -11,12 +12,12 @@ var eol = /\r\n|\r|\n|\u2028|\u2029/g
  * @return {import('esbuild').Plugin}
  */
 export function esbuild(options) {
-  var processor = createProcessor(options)
+  var {extnames, process} = createFormatAwareProcessors(options)
 
   return {name: 'esbuild-xdm', setup}
 
   function setup(build) {
-    build.onLoad({filter: /\.mdx$/}, onload)
+    build.onLoad({filter: extnamesToRegex(extnames)}, onload)
   }
 
   async function onload(data) {
@@ -38,7 +39,7 @@ export function esbuild(options) {
     var column
 
     try {
-      file = await processor.process(file)
+      file = await process(file)
       contents = file.contents
       messages = file.messages
     } catch (error) {

lib/integration/node.js

@@ -1,11 +1,9 @@
 import path from 'path'
 import vfile from 'vfile'
-import {createProcessor} from '../core.js'
+import {createFormatAwareProcessors} from '../util/create-format-aware-processors.js'
 
 export function createLoader(options) {
-  var processor = createProcessor(options)
-  // eslint-disable-next-line unicorn/prefer-set-has
-  var extnames = ['.mdx']
+  var {extnames, process} = createFormatAwareProcessors(options)
 
   return {getFormat, transformSource}
 
@@ -22,7 +20,7 @@ export function createLoader(options) {
       return defaultTransformSource(contents, context, defaultTransformSource)
     }
 
-    file = await processor.process(vfile({contents, path: context.url}))
+    file = await process(vfile({contents, path: context.url}))
     // V8 on Erbium.
     /* c8 ignore next 2 */
     return {source: String(file).replace(/\/jsx-runtime(?=["'])/g, '$&.js')}

lib/integration/require.cjs

@@ -3,25 +3,28 @@
 var fs = require('fs')
 var deasync = require('deasync')
 var {runSync} = deasync(load)('../run.js')
-var {splitEvaluateOptions} = deasync(load)('../evaluate.js')
-var {createProcessor} = deasync(load)('../core.js')
+var {createFormatAwareProcessors} = deasync(load)(
+  '../util/create-format-aware-processors.js'
+)
+var {resolveEvaluateOptions} = deasync(load)(
+  '../util/resolve-evaluate-options.js'
+)
 
 module.exports = register
 
 function register(options) {
-  var config = splitEvaluateOptions(options)
-  var processor = createProcessor(config.compile)
-  var extnames = ['.mdx']
+  var {compiletime, runtime} = resolveEvaluateOptions(options)
+  var {extnames, processSync} = createFormatAwareProcessors(compiletime)
   var index = -1
 
   while (++index < extnames.length) {
     // eslint-disable-next-line node/no-deprecated-api
-    require.extensions[extnames[index]] = mdx
+    require.extensions[extnames[index]] = xdm
   }
 
-  function mdx(module, path) {
-    var file = processor.processSync(fs.readFileSync(path))
-    var result = runSync(file, config.run)
+  function xdm(module, path) {
+    var file = processSync(fs.readFileSync(path))
+    var result = runSync(file, runtime)
     module.exports = result.default
     module.loaded = true
   }

lib/integration/rollup.js

@@ -1,31 +1,25 @@
 import vfile from 'vfile'
 import {createFilter} from '@rollup/pluginutils'
-import {createProcessor} from '../core.js'
+import {createFormatAwareProcessors} from '../util/create-format-aware-processors.js'
 
 /**
- * @typedef {import('../core.js').ProcessorOptions} ProcessorOptions
- * @typedef {import('@rollup/pluginutils').FilterPattern} FilterPattern
- * @typedef {import('rollup').Plugin} Plugin
- *
  * @typedef RollupPluginOptions
- * @property {FilterPattern} [include] List of picomatch patterns to include
- * @property {FilterPattern} [exclude] List of picomatch patterns to exclude
- * @property {Array.<string>} [extensions=['.mdx']] List of extensions to recognize (with `.`!)
+ * @property {import('@rollup/pluginutils').FilterPattern} [include] List of picomatch patterns to include
+ * @property {import('@rollup/pluginutils').FilterPattern} [exclude] List of picomatch patterns to exclude
  *
- * @typedef {ProcessorOptions & RollupPluginOptions} ProcessorAndRollupOptions
+ * @typedef {import('../compile').CompileOptions & RollupPluginOptions} ProcessorAndRollupOptions
  */
 
 /**
  * Compile MDX w/ rollup.
  *
  * @param {ProcessorAndRollupOptions} [options]
- * @return {Plugin}
+ * @return {import('rollup').Plugin}
  */
 export function rollup(options = {}) {
-  var {include, exclude, extensions, ...rest} = options
-  var processor = createProcessor(rest)
+  var {include, exclude, ...rest} = options
+  var {extnames, process} = createFormatAwareProcessors(rest)
   var filter = createFilter(include, exclude)
-  var extnames = extensions || ['.mdx']
 
   return {
     name: 'xdm',
@@ -34,11 +28,8 @@ export function rollup(options = {}) {
       var file = vfile({contents, path})
 
       if (filter(file.path) && extnames.includes(file.extname)) {
-        var compiled = await processor.process(file)
-        return {
-          code: String(compiled.contents),
-          map: compiled.map
-        }
+        var compiled = await process(file)
+        return {code: String(compiled.contents), map: compiled.map}
         // V8 on Erbium.
         /* c8 ignore next 2 */
       }

lib/node-types.js

@@ -0,0 +1,9 @@
+// List of node types made by `mdast-util-mdx`, which have to be passed
+// through untouched from the mdast tree to the hast tree.
+export var nodeTypes = [
+  'mdxFlowExpression',
+  'mdxJsxFlowElement',
+  'mdxJsxTextElement',
+  'mdxTextExpression',
+  'mdxjsEsm'
+]

lib/plugin/recma-jsx-rewrite.js

@@ -98,7 +98,11 @@ export function recmaJsxRewrite(options = {}) {
 
             scope.components.push(name.name)
           }
-        } else if (node.data && node.data._xdmInSource) {
+        } else if (node.data && node.data._xdmExplicitJsx) {
+          // Do not turn explicit JSX into components from `_components`.
+          // As in, a given `h1` component is used for `# heading` (next case),
+          // but not for `<h1>heading</h1>`.
+        } else {
           if (!scope.tags.includes(name.name)) {
             scope.tags.push(name.name)
           }