Add math expression parser for markdown

buildutils/src/ensure-repo.ts

@@ -97,6 +97,10 @@ const UNUSED: Dict<string[]> = {
     '@codemirror/lang-xml',
     '@codemirror/legacy-modes'
   ],
+  '@jupyterlab/codemirror-extension': [
+    '@codemirror/lang-markdown',
+    '@codemirror/legacy-modes'
+  ],
   '@jupyterlab/coreutils': ['path-browserify'],
   '@jupyterlab/fileeditor': ['regexp-match-indices'],
   '@jupyterlab/services': ['ws'],

packages/codemirror-extension/package.json

@@ -38,6 +38,9 @@
     "watch": "tsc -b --watch"
   },
   "dependencies": {
+    "@codemirror/lang-markdown": "^6.1.1",
+    "@codemirror/language": "^6.6.0",
+    "@codemirror/legacy-modes": "^6.3.2",
     "@jupyter/ydoc": "^1.0.2",
     "@jupyterlab/application": "^4.0.0-rc.0",
     "@jupyterlab/codeeditor": "^4.0.0-rc.0",

packages/codemirror-extension/src/services.tsx

@@ -3,6 +3,7 @@
  * Distributed under the terms of the Modified BSD License.
  */
 
+import { StreamLanguage } from '@codemirror/language';
 import { IYText } from '@jupyter/ydoc';
 import {
   JupyterFrontEnd,
@@ -18,6 +19,7 @@ import {
   IEditorExtensionRegistry,
   IEditorLanguageRegistry,
   IEditorThemeRegistry,
+  parseMathIPython,
   ybinding
 } from '@jupyterlab/codemirror';
 import { ISettingRegistry } from '@jupyterlab/settingregistry';
@@ -60,10 +62,16 @@ export const languagePlugin: JupyterFrontEndPlugin<IEditorLanguageRegistry> = {
       name: 'ipythongfm',
       mime: 'text/x-ipythongfm',
       load: async () => {
-        // TODO: add support for LaTeX
-        const m = await import('@codemirror/lang-markdown');
+        const [m, tex] = await Promise.all([
+          import('@codemirror/lang-markdown'),
+          import('@codemirror/legacy-modes/mode/stex')
+        ]);
         return m.markdown({
-          codeLanguages: (info: string) => languages.findBest(info) as any
+          base: m.markdownLanguage,
+          codeLanguages: (info: string) => languages.findBest(info) as any,
+          extensions: [
+            parseMathIPython(StreamLanguage.define(tex.stexMath).parser)
+          ]
         });
       }
     });

packages/codemirror/package.json

@@ -67,6 +67,7 @@
     "@lezer/common": "^1.0.2",
     "@lezer/generator": "^1.2.2",
     "@lezer/highlight": "^1.1.4",
+    "@lezer/markdown": "^1.0.2",
     "@lumino/coreutils": "^2.1.1",
     "@lumino/disposable": "^2.1.1",
     "@lumino/signaling": "^2.1.1",

packages/codemirror/src/editor.ts

@@ -18,7 +18,6 @@ import { CodeEditor } from '@jupyterlab/codeeditor';
 import { SyntaxNodeRef } from '@lezer/common';
 import { UUID } from '@lumino/coreutils';
 import { Signal } from '@lumino/signaling';
-import './codemirror-ipythongfm';
 import { ExtensionsHandler } from './extension';
 import { EditorLanguageRegistry } from './language';
 import {

packages/codemirror/src/extensions/index.ts

@@ -4,5 +4,6 @@
  */
 
 export * from './customStyle';
+export * from './ipython-md';
 export * from './rulers';
 export * from './ybinding';

packages/codemirror/src/extensions/ipython-md.ts

@@ -0,0 +1,158 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+import { parseMixed, Parser } from '@lezer/common';
+import { tags } from '@lezer/highlight';
+import {
+  DelimiterType,
+  InlineContext,
+  MarkdownConfig,
+  NodeSpec
+} from '@lezer/markdown';
+
+// Mathematical expression delimiters
+const INLINE_MATH_DOLLAR = 'InlineMathDollar';
+const INLINE_MATH_BRACKET = 'InlineMathBracket';
+const BLOCK_MATH_DOLLAR = 'BlockMathDollar';
+const BLOCK_MATH_BRACKET = 'BlockMathBracket';
+
+/**
+ * Lengh of the delimiter for a math expression
+ */
+const DELIMITER_LENGTH: Record<string, number> = {
+  [INLINE_MATH_DOLLAR]: 1,
+  [INLINE_MATH_BRACKET]: 3,
+  [BLOCK_MATH_DOLLAR]: 2,
+  [BLOCK_MATH_BRACKET]: 3
+};
+
+/**
+ * Delimiters for math expressions
+ */
+// Delimiters must be defined as constant because they are used in object match tests
+const DELIMITERS = Object.keys(DELIMITER_LENGTH).reduce<
+  Record<string, DelimiterType>
+>((agg, name) => {
+  agg[name] = { mark: `${name}Mark`, resolve: name };
+  return agg;
+}, {});
+
+/**
+ * Define an IPython mathematical expression parser for Markdown.
+ *
+ * @param latexParser CodeMirror {@link Parser} for LaTeX mathematical expression
+ * @returns Markdown extension
+ */
+export function parseMathIPython(latexParser?: Parser): MarkdownConfig {
+  const defineNodes = new Array<NodeSpec>();
+  Object.keys(DELIMITER_LENGTH).forEach(name => {
+    defineNodes.push(
+      {
+        name,
+        style: tags.emphasis
+      },
+      { name: `${name}Mark`, style: tags.processingInstruction }
+    );
+  });
+  return {
+    defineNodes,
+    parseInline: [
+      {
+        name: BLOCK_MATH_DOLLAR,
+        parse(cx: InlineContext, next: number, pos: number): number {
+          if (next != 36 /* '$' */ || cx.char(pos + 1) != 36) {
+            return -1;
+          }
+
+          return cx.addDelimiter(
+            DELIMITERS[BLOCK_MATH_DOLLAR],
+            pos,
+            pos + DELIMITER_LENGTH[BLOCK_MATH_DOLLAR],
+            true,
+            true
+          );
+        }
+      },
+      {
+        name: INLINE_MATH_DOLLAR,
+        parse(cx: InlineContext, next: number, pos: number): number {
+          if (next != 36 /* '$' */ || cx.char(pos + 1) == 36) {
+            return -1;
+          }
+
+          return cx.addDelimiter(
+            DELIMITERS[INLINE_MATH_DOLLAR],
+            pos,
+            pos + DELIMITER_LENGTH[INLINE_MATH_DOLLAR],
+            true,
+            true
+          );
+        }
+      },
+      // Inline expression wrapped in \\( ... \\)
+      {
+        name: INLINE_MATH_BRACKET,
+        before: 'Escape', // Search for this delimiter before the escape character
+        parse(cx: InlineContext, next: number, pos: number): number {
+          if (
+            next != 92 /* '\' */ ||
+            cx.char(pos + 1) != 92 ||
+            ![40 /* '(' */, 41 /* ')' */].includes(cx.char(pos + 2))
+          ) {
+            return -1;
+          }
+
+          return cx.addDelimiter(
+            DELIMITERS[INLINE_MATH_BRACKET],
+            pos,
+            pos + DELIMITER_LENGTH[INLINE_MATH_BRACKET],
+            cx.char(pos + 2) == 40,
+            cx.char(pos + 2) == 41
+          );
+        }
+      },
+      // Block expression wrapped in \\[ ... \\]
+      {
+        name: BLOCK_MATH_BRACKET,
+        before: 'Escape', // Search for this delimiter before the escape character
+        parse(cx: InlineContext, next: number, pos: number): number {
+          if (
+            next != 92 /* '\' */ ||
+            cx.char(pos + 1) != 92 ||
+            ![91 /* '[' */, 93 /* ']' */].includes(cx.char(pos + 2))
+          ) {
+            return -1;
+          }
+
+          return cx.addDelimiter(
+            DELIMITERS[BLOCK_MATH_BRACKET],
+            pos,
+            pos + DELIMITER_LENGTH[BLOCK_MATH_BRACKET],
+            cx.char(pos + 2) == 91,
+            cx.char(pos + 2) == 93
+          );
+        }
+      }
+    ],
+    wrap: latexParser
+      ? parseMixed((node, input) => {
+          // Test if the node type is one of the math expression
+          const delimiterLength = DELIMITER_LENGTH[node.type.name];
+          if (delimiterLength) {
+            return {
+              parser: latexParser,
+              // Remove delimiter from LaTeX parser otherwise it won't be highlighted
+              overlay: [
+                {
+                  from: node.from + delimiterLength,
+                  to: node.to - delimiterLength
+                }
+              ]
+            };
+          }
+
+          return null;
+        })
+      : undefined
+  };
+}