feat: back to jsx markdown parser we go

website/package.json

@@ -9,22 +9,16 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@mdx-js/mdx": "^2.1.1",
     "clsx": "^1.1.1",
     "copy-text-to-clipboard": "^3.0.1",
     "esbuild": "^0.14.36",
+    "mdx-bundler": "^9.0.1",
     "next": "12.1.5",
     "prism-react-renderer": "^1.3.1",
     "react": "17.0.2",
     "react-dom": "17.0.2",
-    "rehype-katex": "^6.0.2",
-    "rehype-raw": "^6.1.1",
-    "rehype-react": "^7.1.1",
-    "remark-gfm": "^3.0.1",
-    "remark-math": "^5.1.1",
-    "remark-parse": "^10.0.1",
-    "remark-rehype": "^10.1.0",
-    "sass": "^1.50.1",
-    "unified": "^10.1.2"
+    "sass": "^1.50.1"
   },
   "devDependencies": {
     "@babel/parser": "^7.17.9",
@@ -36,10 +30,12 @@
     "@types/glob": "^7.1.4",
     "@types/react": "17.0.27",
     "@types/unist": "^2.0.6",
+    "acorn": "^8.7.1",
     "escape-html": "^1.0.3",
     "eslint": "7.32.0",
     "eslint-config-next": "11.1.2",
     "glob": "^8.0.1",
+    "html-react-parser": "^1.4.12",
     "mdast": "^3.0.0",
     "mdast-util-from-markdown": "^1.2.0",
     "mdast-util-gfm": "^2.0.1",
@@ -49,10 +45,17 @@
     "mdast-util-to-string": "^3.1.0",
     "micromark-extension-gfm": "^2.0.1",
     "micromark-extension-math": "^2.0.2",
-    "micromark-extension-mdxjs-esm": "^1.0.2",
+    "micromark-extension-mdxjs-esm": "^1.0.3",
+    "rehype-katex": "^6.0.2",
+    "rehype-react": "^7.1.1",
     "rehype-stringify": "^9.0.3",
+    "remark-gfm": "^3.0.1",
+    "remark-math": "^5.1.1",
+    "remark-parse": "^10.0.1",
+    "remark-rehype": "^10.1.0",
     "remark-stringify": "^10.0.2",
     "typescript": "^4.6.3",
+    "unified": "^10.1.2",
     "unist-util-visit": "^4.1.0"
   }
 }

website/src/components/Sidebar/Desktop.tsx

@@ -9,7 +9,7 @@ import c from "clsx";
 
 import SidebarItems from "./SidebarItems";
 
-import { markdown } from "@/markdown/markdown.module.scss";
+import { markdown } from "@/mdx/markdown.module.scss";
 import s from "./styles.module.scss";
 
 import type { SidebarProps } from ".";

website/src/components/Sidebar/Mobile.tsx

@@ -9,7 +9,7 @@
 
  import SidebarItems from "./SidebarItems";
 
- import { markdown } from "@/markdown/markdown.module.scss";
+ import { markdown } from "@/mdx/markdown.module.scss";
  import s from "./styles.module.scss";
 
  import type { SidebarProps } from ".";

website/src/components/Toc/Desktop.tsx

@@ -9,7 +9,7 @@ import c from "clsx"
 
 import Toc from "."
 
-import { markdown } from "@/markdown/markdown.module.scss"
+import { markdown } from "@/mdx/markdown.module.scss"
 import s from "./styles.module.scss"
 
 import type { TocProps } from "../Toc"

website/src/layout/components/MDX/Provider.tsx

@@ -0,0 +1,35 @@
+import { createContext, useContext } from "react";
+
+import { useMDX } from "./useMDX";
+
+import type { FC, ReactNode } from "react";
+import type { MDXComponentProps } from "./useMDX";
+
+type MDXData = {
+  MDXComponent: FC<MDXComponentProps>
+}
+
+const MDXContext = createContext<MDXData | undefined>(undefined);
+
+export const useMDXContext = () => {
+  const context = useContext(MDXContext)
+  if (context === undefined) {
+    throw new Error('useMDXContext must be used in a MDXProvider')
+  }
+  return context
+}
+
+interface MDXProviderProps {
+  source: string
+  children: ReactNode
+}
+
+export default function MDXProvider ({ children, ...props }: MDXProviderProps) {
+  const { MDXComponent } = useMDX(props)
+
+  return (
+    <MDXContext.Provider value={{ MDXComponent }}>
+      {children}
+    </MDXContext.Provider>
+  )
+}

website/src/layout/components/MDX/useMDX.tsx

@@ -0,0 +1,24 @@
+import { useMemo, useCallback } from "react";
+import { ComponentMap, getMDXExport } from "mdx-bundler/client";
+
+import defaultComponents from "@/mdx/components";
+
+import type { FC } from "react";
+
+interface UseMDXProps {
+  source: string
+}
+
+export interface MDXComponentProps {
+  components?: ComponentMap
+}
+
+export function useMDX ({ source }: UseMDXProps) {
+  const { default: BaseComponent } = useMemo(() => getMDXExport(source), [source])
+
+  const MDXComponent: FC<MDXComponentProps> = useCallback(({ components, ...props}) => (
+    <BaseComponent components={{...defaultComponents, ...components}} {...props} />
+  ), [BaseComponent])
+
+  return { MDXComponent }
+}

website/src/layout/components/MarkdownWrapper/index.tsx

@@ -1,6 +1,6 @@
 import c from "clsx";
 
-import { markdown, content } from "@/markdown/markdown.module.scss";
+import { markdown, content } from "@/mdx/markdown.module.scss";
 
 import type { ReactNode } from "react";
 

website/src/layout/pages/DocPage/index.tsx

@@ -11,15 +11,15 @@ import TocContainer from "@/layout/components/TocContainer";
 import BeforeMarkdown from "@/layout/components/BeforeMarkdown";
 import OpenInColab from "@/layout/components/OpenInColab";
 import MarkdownWrapper from "@/layout/components/MarkdownWrapper";
-import { useMarkdown } from "@/layout/components/Markdown/useMarkdown";
 import PageProvider from "@/layout/context/Page";
 
 import type { DocPageProps } from "@/layout/pages/types";
+import { useMDX } from "@/layout/components/MDX/useMDX";
 
 export default function DocPage ({ source, slug, fsPath, sidebar, toc }: DocPageProps) {
   const { asPath } = useRouter();
 
-  const { markdownReactElement, MarkdownComponent } = useMarkdown(source)
+  const { MDXComponent } = useMDX({ source })
 
   return (
     <PageProvider sidebar={sidebar} toc={toc}>
@@ -34,7 +34,7 @@ export default function DocPage ({ source, slug, fsPath, sidebar, toc }: DocPage
               <OpenInColab fsPath={fsPath} />
             </BeforeMarkdown>
             <MarkdownWrapper>
-              <MarkdownComponent fallback={<h1>Loading...</h1>} />
+              <MDXComponent />
             </MarkdownWrapper>
           </ContentContainer>
           <TocContainer>

website/src/lib/bundle.ts

@@ -0,0 +1,77 @@
+import path from "path";
+import { bundleMDX } from "mdx-bundler";
+
+import remarkHeadingIds from "@/lib/unified/remark-heading-ids";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import rehypeKatex from "rehype-katex";
+
+import astDebug from "./unified/ast-debug";
+
+// kinda hacky
+import type { BundleMDX } from "mdx-bundler/dist/types";
+
+// https://www.alaycock.co.uk/2021/03/mdx-bundler#esbuild-executable
+const fixEsbuildPath = () => {
+  if (process.platform === 'win32') {
+    process.env.ESBUILD_BINARY_PATH = path.join(
+      process.cwd(),
+      'node_modules',
+      'esbuild-windows-64',
+      'esbuild.exe',
+    )
+  } else {
+    process.env.ESBUILD_BINARY_PATH = path.join(
+      process.cwd(),
+      'node_modules',
+      'esbuild',
+      'bin',
+      'esbuild',
+    )
+  }
+}
+
+type Options<Frontmatter> = Required<Pick<BundleMDX<Frontmatter>, 'source' | 'cwd'>> & {
+  baseUrl: `/${string}`
+  slug: string[]
+}
+export const bundle = async <Frontmatter>({ source, cwd, baseUrl, slug }: Options<Frontmatter>) => {
+  fixEsbuildPath();
+  const res = await bundleMDX({
+    source,
+    cwd,
+    mdxOptions: (options) => {
+      options.remarkPlugins = [
+        ...(options.remarkPlugins ?? []),
+        remarkHeadingIds,
+        remarkGfm,
+        remarkMath,
+      ]
+      options.rehypePlugins = [
+        ...(options.rehypePlugins ?? []),
+        rehypeKatex,
+      ]
+
+      return options
+    },
+    esbuildOptions: (options) => {
+      options.loader = {
+        ...(options.loader ?? {}),
+        ".png": 'file',
+        ".jpg": 'file',
+        ".gif": 'file',
+      }
+      // write assets locally to `~/public/static/content/[...slug]`,
+      // and they will correspondingly be served at /static/content/[...slug]
+      options.outdir = slug ? path.join(process.cwd(), 'public', 'static', 'content', ...slug) : undefined
+      options.publicPath = slug ? `/static/content${baseUrl}/${slug.join('/')}` : undefined
+      options.write = true
+
+      return options
+    },
+  });
+
+  if (res.errors.length > 0) { console.error(res.errors.map(({ detail }) => detail)); }
+
+  return res;
+}

website/src/lib/helpers/extractMdTitleDescription.ts → website/src/lib/helpers/extractTitleAndDescription.ts

@@ -4,14 +4,10 @@ import { gfm } from "micromark-extension-gfm";
 import { mathFromMarkdown } from "mdast-util-math";
 import { gfmFromMarkdown } from "mdast-util-gfm";
 import { toValue } from "@/lib/helpers/toc/utils";
+import { fromNotebook } from "../unified/nbast-util-from-notebook";
+import type { Root } from "mdast";
 
-export const extractTitleDescriptionFromMdString = async (md: string) => {
-
-  // expect the first two children to be an h1 heading (the title), and a paragraph (the description)
-  const mdast = fromMarkdown(md, 'utf-8', {
-    extensions: [math(), gfm()],
-    mdastExtensions: [mathFromMarkdown(), gfmFromMarkdown()],
-  });
+export const extractTitleAndDescriptionFromMdast = async (mdast: Root) => {
   const { children } = mdast;
   const titleNode = children?.[0]
   const descriptionNode = children?.[1];
@@ -31,4 +27,21 @@ export const extractTitleDescriptionFromMdString = async (md: string) => {
   ;
 
   return { title, description }
+}
+
+export const extractTitleAndDescriptionFromMdString = async (md: string) => {
+
+  // expect the first two children to be an h1 heading (the title), and a paragraph (the description)
+  const mdast = fromMarkdown(md, 'utf-8', {
+    extensions: [math(), gfm()],
+    mdastExtensions: [mathFromMarkdown(), gfmFromMarkdown()],
+  });
+
+  return extractTitleAndDescriptionFromMdast(mdast);
+}
+
+export const extractTitleAndDescriptionFromNotebook = async (notebook: string) => {
+  const mdast = fromNotebook(notebook) as Root
+
+  return extractTitleAndDescriptionFromMdast(mdast);
 }