Include position information

package.json

@@ -25,17 +25,20 @@
         "build": "mkdirp dist; webpack"
     },
     "dependencies": {
+        "@babel/plugin-transform-modules-commonjs": "^7.17.9",
         "@types/color": "^3.0.3",
         "@types/cssesc": "^3.0.0",
         "@types/prettier": "^2.6.0",
         "@types/trie-prefix-tree": "^1.5.0",
         "color": "^4.2.3",
         "cssesc": "^3.0.0",
+        "jest-esm-transformer": "^1.0.0",
         "prettier": "^2.6.2",
-        "trie-prefix-tree": "^1.5.1"
+        "trie-prefix-tree": "^1.5.1",
+        "unified": "^10.1.2"
     },
     "devDependencies": {
-        "@babel/core": "7.17.9",
+        "@babel/core": "^7.17.9",
         "@babel/preset-env": "7.16.11",
         "@types/jest": "^27.4.1",
         "@types/node": "^17.0.24",
@@ -55,8 +58,11 @@
         "webpack-cli": "^4.9.2"
     },
     "jest": {
+        "transformIgnorePatterns": [
+            "node_modules/(?!unified|bail|is-plain-obj|trough|vfile|unist.*)"
+        ],
         "transform": {
-            "\\.js$": "babel-jest",
+            "\\.m?jsx?$": "jest-esm-transformer",
             "latex.pegjs$": "<rootDir>/src/tests/pegjs-preprocessor-latex.js",
             "\\.pegjs$": "<rootDir>/src/tests/pegjs-preprocessor.js",
             "^.+\\.tsx?$": "ts-jest"

src/grammars/latex.pegjs

@@ -1,18 +1,23 @@
 {
+    function toString(e) {
+        if (typeof e === "string") {
+            return e;
+        }
+        if (typeof e.content === "string") {
+            return e.content;
+        }
+        return e;
+    }
+
     function compare_env(g1, g2) {
-        return g1.content.join("") == g2.content.join("");
+        return (
+            g1.content.map(toString).join("") ===
+            g2.content.map(toString).join("")
+        );
     }
 
     function createNode(type, extra = {}) {
-        const ret = { type, ...extra };
-        // Add a non-enumerable location property to `ret`. Since it is
-        // non-enumerable, it won't clutter up the syntax tree when printed.
-        Object.defineProperty(ret, "loc", {
-            value: location(),
-            enumerable: false,
-        });
-
-        return ret;
+        return { type, ...extra, position: location() };
     }
 }
 
@@ -36,7 +41,7 @@ token "token"
     / number
     / whitespace
     / punctuation
-    / $(!nonchar_token .)+
+    / s:$(!nonchar_token .)+ { return createNode("string", { content: s }); }
     // If all else fails, we allow special tokens. If one of these
     // is matched, it means there is an unbalanced group.
     / begin_group
@@ -67,7 +72,7 @@ math_token "math token"
         }
     / ignore
     / whitespace
-    / .
+    / s:. { return createNode("string", { content: s }); }
 
 nonchar_token "nonchar token"
     = escape
@@ -89,9 +94,11 @@ whitespace "whitespace"
         }
 
 number "number"
-    = a:num+ "." b:num+ { return a.join("") + "." + b.join(""); }
-    / "." b:num+ { return "." + b.join(""); }
-    / a:num+ "." { return a.join("") + "."; }
+    = s:(
+        a:num+ "." b:num+ { return a.join("") + "." + b.join(""); }
+        / "." b:num+ { return "." + b.join(""); }
+        / a:num+ "." { return a.join("") + "."; }
+    ) { return createNode("string", { content: s }); }
 
 special_macro "special macro" // for the special macros like \[ \] and \begin{} \end{} etc.
     // \verb|xxx| and \verb*|xxx|
@@ -229,36 +236,44 @@ begin_env = escape "begin"
 end_env = escape "end"
 
 math_env_name
-    = "equation*"
-    / "equation"
-    / "align*"
-    / "align"
-    / "alignat*"
-    / "alignat"
-    / "gather*"
-    / "gather"
-    / "multline*"
-    / "multline"
-    / "flalign*"
-    / "flalign"
-    / "split"
-    / "math"
-    / "displaymath"
+    = e:(
+        "equation*"
+        / "equation"
+        / "align*"
+        / "align"
+        / "alignat*"
+        / "alignat"
+        / "gather*"
+        / "gather"
+        / "multline*"
+        / "multline"
+        / "flalign*"
+        / "flalign"
+        / "split"
+        / "math"
+        / "displaymath"
+    ) { return createNode("string", { content: e }); }
+
+// FOR THE FOLLOWING ITEMS:
+//
+// Most of the time these are used as a match only. However, in the case
+// of errors, we match them as strings. Therefore, it is useful to have the returned
+// match be a string node.
 
 // catcode 0
-escape "escape" = "\\"
+escape "escape" = "\\" { return createNode("string", { content: "\\" }); }
 
 // catcode 1
-begin_group = "{"
+begin_group = s:"{" { return createNode("string", { content: s }); }
 
 // catcode 2
-end_group = "}"
+end_group = s:"}" { return createNode("string", { content: s }); }
 
 // catcode 3
-math_shift = "$"
+math_shift = s:"$" { return createNode("string", { content: s }); }
 
 // catcode 4
-alignment_tab = "&"
+alignment_tab = s:"&" { return createNode("string", { content: s }); }
 
 // catcode 5 (linux, os x, windows)
 nl "newline"
@@ -267,13 +282,13 @@ nl "newline"
     / "\r\n"
 
 // catcode 6
-macro_parameter = "#"
+macro_parameter = s:"#" { return createNode("string", { content: s }); }
 
 // catcode 7
-superscript = "^"
+superscript = s:"^" { return createNode("string", { content: s }); }
 
 // catcode 8
-subscript = "_"
+subscript = s:"_" { return createNode("string", { content: s }); }
 
 // catcode 9
 ignore = "\0"
@@ -288,7 +303,10 @@ char "letter" = c:[a-zA-Z]
 num "digit" = n:[0-9]
 
 // catcode 12
-punctuation "punctuation" = p:[.,;:\-\*/()!?=+<>\[\]`'\"~]
+punctuation "punctuation"
+    = p:[.,;:\-\*/()!?=+<>\[\]`'\"~] {
+            return createNode("string", { content: p });
+        }
 
 // catcode 14, including the newline
 comment_start = "%"

src/libs/ast-types.ts

@@ -10,7 +10,7 @@ export interface GenericNode {
 interface BaseNode {
     type: string;
     _renderInfo?: any;
-    loc?: {
+    position?: {
         start: { offset: number; line: number; column: number };
         end: { offset: number; line: number; column: number };
     };

src/libs/ast/render-info.ts

@@ -3,7 +3,7 @@ import { walkAst } from "./walkers";
 import { match } from "./matchers";
 
 /**
- * Removes any `_renderInfo` tags present in the AST.
+ * Removes any `_renderInfo` and `position` tags present in the AST.
  *
  * @export
  * @param {*} ast
@@ -13,13 +13,16 @@ export function trimRenderInfo(ast: Ast.Ast) {
     return walkAst(
         ast,
         (node) => {
-            const ret = { ...node };
-            delete ret._renderInfo;
-            return ret;
+            const { _renderInfo, position, ...ret } = node;
+            if (ret.type === "environment" || ret.type === "mathenv") {
+                ret.env = trimRenderInfo(ret.env) as Ast.Node[];
+            }
+            if (_renderInfo != null || position != null) {
+                return ret;
+            }
+            return node;
         },
-        ((node) =>
-            node != null &&
-            (node as Ast.Node)._renderInfo != null) as Ast.TypeGuard<Ast.Node>
+        ((node) => node != null) as Ast.TypeGuard<Ast.Node>
     );
 }
 

src/parsers/parser.ts

@@ -215,55 +215,6 @@ function attachSpecialMacroArgs<T extends Ast.Ast>(
     return ast;
 }
 
-/**
- * Recursively wraps all strings in the AST node in
- * a { type: "string", content: <original string> }
- * object.
- *
- * @param {*} node
- */
-function wrapStrings<T extends Ast.Ast | string>(
-    node: T
-): T extends string ? Ast.String : T {
-    if (node == null) {
-        return node;
-    }
-    if (typeof node === "string") {
-        return { type: "string", content: node } as any;
-    }
-    if (Array.isArray(node)) {
-        return node.map(wrapStrings) as any;
-    }
-    // At this point, `node` must be an object
-    // wrap strings that appear in children
-
-    // We don't want the `content` of a type == macro
-    // node to be wrapped, but wrap everything else
-    let childProps = ["content", "args", "env"];
-    switch (node.type) {
-        case "macro":
-            childProps = ["args"];
-            break;
-        case "comment":
-        case "string":
-        case "verb":
-        case "verbatim":
-            childProps = [];
-            break;
-        default:
-            break;
-    }
-
-    const ret: Ast.Node | Ast.Argument = { ...node };
-    for (const prop of childProps) {
-        if (prop in ret) {
-            (ret as any)[prop] = wrapStrings((ret as any)[prop]);
-        }
-    }
-
-    return ret as any;
-}
-
 for (const key in SPECIAL_MACROS) {
     if (key in LIB_SPECIAL_MACROS) {
         console.log(
@@ -341,7 +292,7 @@ export function parseMath(
     const pegAst: Ast.Node[] = PegParser.parse(str, {
         startRule: "math",
     });
-    let ast = wrapStrings(pegAst);
+    let ast = pegAst;
     ast = processMacrosAndEnvironments(ast, options);
     return ast;
 }
@@ -357,7 +308,7 @@ export function parse(
     options?: { macros?: SpecialMacroSpec; environments?: SpecialEnvSpec }
 ) {
     const pegAst: Ast.Root = PegParser.parse(str);
-    let ast = wrapStrings(pegAst);
+    let ast = pegAst;
     ast = processMacrosAndEnvironments(ast, options);
     // Now that arguments have been attached to environments and macros, we may need
     // to re-parse the contents of some environments/macro args in math mode

src/prettier-plugin-latex.ts

@@ -16,8 +16,8 @@ const parsers = {
     "latex-parser": {
         parse,
         astFormat: "latex-ast",
-        locStart: (node: Ast.Node) => (node.loc ? node.loc.start.offset : 0),
-        locEnd: (node: Ast.Node) => (node.loc ? node.loc.end.offset : 1),
+        locStart: (node: Ast.Node) => (node.position ? node.position.start.offset : 0),
+        locEnd: (node: Ast.Node) => (node.position ? node.position.end.offset : 1),
     },
 };