Implement special case modulus operator

**Output changes**

Now the modulus operator if written exactly `(mod` will yield a
significant lspace at the start of the group and no lspace at the
start of the operator.

Resolves: #107

demo/test-cases.html

@@ -174,7 +174,7 @@ <h2>Standard Functions</h2>
           det(A) = |(a, b; c, d)| = ad - cd
         </test-case>
 
-        <test-case>a^(p-1) == 1   (mod p)</test-case>
+        <test-case display="block">a^(p-1) == 1  (mod p)</test-case>
       </section>
 
       <!-- prettier-ignore -->

docs/index.html

@@ -1470,6 +1470,13 @@ <h3>Spaces</h3>
             the height of the <i>‘x’</i> character in your font.
           </p>
 
+          <p>
+            In addition <code>(mod p)</code> will add 1.65 <span class="unit"
+              >ex</span
+            >
+            of space to the left of the open parenthesis.
+          </p>
+
           <aside class="examples">
             <ul>
               <li>
@@ -1479,6 +1486,11 @@ <h3>Spaces</h3>
                 </use-example>
               </li>
 
+              <li>
+                <!-- prettier-ignore -->
+                <use-example>a^(p-1) == 1  (mod p)</use-example>
+              </li>
+
               <li>
                 <!-- prettier-ignore -->
                 <use-example>
@@ -1635,10 +1647,10 @@ <h3>Sub and superscripts</h3>
           <h3>Fenced Groups</h3>
 
           <p>
-            We can use parentesis to group terms together. An any unicode open
+            We can use parenthesis to group terms together. An any unicode open
             parenthesis (unicode category <code>Ps</code>) starts a fence, and
             any closing parenthesis (category <code>Pe</code>) closes it. Note
-            that parentesis don’t have to match, but they do have to come in
+            that parenthesis don’t have to match, but they do have to come in
             open/close pairs.
           </p>
 

eslint.config.js

@@ -106,6 +106,7 @@ export default [
       "no-throw-literal": "error",
       "no-unmodified-loop-condition": "error",
       "no-unused-expressions": "error",
+      "no-unused-vars": ["error", { varsIgnorePattern: "^_" }],
       "no-useless-backreference": "error",
       "no-useless-concat": "error",
       "no-useless-return": "error",

src/compiler/parser/handlers/group.js

@@ -3,11 +3,22 @@ import { addZeroLSpaceToOperator } from "../utils.js";
 import expr from "./expr.js";
 
 /**
+ * @typedef {import("../../tokenizer/index.js").Token} Token
  * @typedef {import("../index.js").Node} Node
  * @typedef {import("../index.js").FencedGroup} FencedGroup
  * @typedef {import("../index.js").MatrixGroup} MatrixGroup
  */
 
+/**
+ * @param {Token} token
+ * @returns {Omit<Token, "type">}
+ */
+function omitType(token) {
+  const { type: _type, ...rest } = token;
+
+  return rest;
+}
+
 /**
  * @param {import("../parse.js").State} state
  * @returns {{ node: FencedGroup | MatrixGroup, end: number }}
@@ -108,8 +119,8 @@ export default function group(state) {
   const close = token && token.type === "paren.close" ? token : null;
 
   const attrs = {
-    open: open.value,
-    close: (close && token.value) || "",
+    open: omitType(open),
+    close: close ? omitType(close) : null,
     seps,
   };
 

src/compiler/parser/handlers/group.test.js

@@ -15,8 +15,8 @@ test("empty unfenced group", (t) => {
     type: "FencedGroup",
     items: [],
     attrs: {
-      open: "",
-      close: "",
+      open: { value: "" },
+      close: { value: "" },
       seps: [],
     },
   });
@@ -35,8 +35,8 @@ test("empty fenced group", (t) => {
     type: "FencedGroup",
     items: [],
     attrs: {
-      open: "(",
-      close: ")",
+      open: { value: "(" },
+      close: { value: ")" },
       seps: [],
     },
   });
@@ -52,8 +52,8 @@ test("empty unclosed group", (t) => {
     type: "FencedGroup",
     items: [],
     attrs: {
-      open: "",
-      close: "",
+      open: { value: "" },
+      close: null,
       seps: [],
     },
   });
@@ -74,8 +74,50 @@ test("group with one sep", (t) => {
     items: [[]],
     attrs: {
       seps: [","],
-      open: "",
-      close: "",
+      open: { value: "" },
+      close: { value: "" },
+    },
+  });
+});
+
+test("group with open attrs", (t) => {
+  const tokens = [
+    { type: "paren.open", value: "(", attrs: { foo: "bar" } },
+    { type: "sep.col", value: "," },
+    { type: "paren.close", value: "" },
+  ];
+
+  const { end, node } = group({ start: 0, tokens });
+
+  t.is(end, 3);
+  t.deepEqual(node, {
+    type: "FencedGroup",
+    items: [[]],
+    attrs: {
+      seps: [","],
+      open: { value: "(", attrs: { foo: "bar" } },
+      close: { value: "" },
+    },
+  });
+});
+
+test("group with close attrs", (t) => {
+  const tokens = [
+    { type: "paren.open", value: "" },
+    { type: "sep.col", value: "," },
+    { type: "paren.close", value: ")", attrs: { foo: "bar" } },
+  ];
+
+  const { end, node } = group({ start: 0, tokens });
+
+  t.is(end, 3);
+  t.deepEqual(node, {
+    type: "FencedGroup",
+    items: [[]],
+    attrs: {
+      seps: [","],
+      open: { value: "" },
+      close: { value: ")", attrs: { foo: "bar" } },
     },
   });
 });
@@ -94,8 +136,8 @@ test("unclosed group with one sep", (t) => {
     items: [[]],
     attrs: {
       seps: [","],
-      open: "foo",
-      close: "",
+      open: { value: "foo" },
+      close: null,
     },
   });
 });
@@ -117,8 +159,8 @@ test("group with multiple seps", (t) => {
   t.is(node.items[0].length, 0);
   t.is(node.items[1].length, 0);
   t.true(node.items[2].length > 0);
-  t.is(node.attrs.open, "");
-  t.is(node.attrs.close, "");
+  t.deepEqual(node.attrs.open, { value: "" });
+  t.deepEqual(node.attrs.close, { value: "" });
   t.deepEqual(node.attrs.seps, [",", ","]);
 });
 
@@ -138,8 +180,8 @@ test("unclosed group with multiple seps", (t) => {
   t.is(node.items[0].length, 0);
   t.is(node.items[1].length, 0);
   t.true(node.items[2].length > 0);
-  t.is(node.attrs.open, "");
-  t.is(node.attrs.close, "");
+  t.deepEqual(node.attrs.open, { value: "" });
+  t.is(node.attrs.close, null);
   t.deepEqual(node.attrs.seps, [",", ","]);
 });
 
@@ -160,8 +202,8 @@ test("ignores leading spaces", (t) => {
     items: [[]],
     attrs: {
       seps: [","],
-      open: "",
-      close: "",
+      open: { value: "" },
+      close: { value: "" },
     },
   });
 });
@@ -177,8 +219,8 @@ test("matrix groups", (t) => {
 
   t.is(end, 3);
   t.is(node.type, "MatrixGroup");
-  t.is(node.attrs.open, "(");
-  t.is(node.attrs.close, ")");
+  t.deepEqual(node.attrs.open, { value: "(" });
+  t.deepEqual(node.attrs.close, { value: ")" });
   t.deepEqual(node.items, [[[]]]);
 });
 
@@ -199,8 +241,8 @@ test("matrix groups with items", (t) => {
 
   t.is(end, 9);
   t.is(node.type, "MatrixGroup");
-  t.is(node.attrs.open, "(");
-  t.is(node.attrs.close, ")");
+  t.deepEqual(node.attrs.open, { value: "(" });
+  t.deepEqual(node.attrs.close, { value: ")" });
   t.is(node.items.length, 2);
   t.is(node.items[0].length, 2);
   t.is(node.items[1].length, 2);
@@ -220,8 +262,8 @@ test("unclosed matrix with one sep", (t) => {
 
   t.true(end >= 2);
   t.is(node.type, "MatrixGroup");
-  t.is(node.attrs.open, "foo");
-  t.is(node.attrs.close, "");
+  t.deepEqual(node.attrs.open, { value: "foo" });
+  t.is(node.attrs.close, null);
   t.deepEqual(node.items, [[[]]]);
 });
 
@@ -240,8 +282,8 @@ test("sparce matrix groups", (t) => {
 
   t.is(end, 7);
   t.is(node.type, "MatrixGroup");
-  t.is(node.attrs.open, "(");
-  t.is(node.attrs.close, ")");
+  t.deepEqual(node.attrs.open, { value: "(" });
+  t.deepEqual(node.attrs.close, { value: ")" });
   t.is(node.items.length, 2);
   t.is(node.items[0].length, 2);
   t.is(node.items[1].length, 1);
@@ -264,8 +306,8 @@ test("single line matrix groups", (t) => {
 
   t.is(end, 6);
   t.is(node.type, "MatrixGroup");
-  t.is(node.attrs.open, "(");
-  t.is(node.attrs.close, ")");
+  t.deepEqual(node.attrs.open, { value: "(" });
+  t.deepEqual(node.attrs.close, { value: ")" });
   t.is(node.items.length, 1);
   t.is(node.items[0].length, 2);
   t.true(node.items[0][0].length > 0);
@@ -285,8 +327,8 @@ test("unclosed matrix groups", (t) => {
 
   t.true(end >= 5);
   t.is(node.type, "MatrixGroup");
-  t.is(node.attrs.open, "(");
-  t.is(node.attrs.close, "");
+  t.deepEqual(node.attrs.open, { value: "(" });
+  t.is(node.attrs.close, null);
   t.is(node.items.length, 1);
   t.is(node.items[0].length, 2);
   t.true(node.items[0][0].length > 0);
@@ -308,7 +350,7 @@ test("ignores leading spaces in matrix groups", (t) => {
 
   t.is(end, 7);
   t.is(node.type, "MatrixGroup");
-  t.is(node.attrs.open, "");
-  t.is(node.attrs.close, "");
+  t.deepEqual(node.attrs.open, { value: "" });
+  t.deepEqual(node.attrs.close, { value: "" });
   t.deepEqual(node.items, [[[], []]]);
 });

src/compiler/parser/handlers/infix.test.js

@@ -534,8 +534,8 @@ test("but keeps a singleton exponent in fences", (t) => {
   t.is(node.items.length, 2);
   t.is(node.items[0].value, "a");
   t.is(node.items[1].type, "FencedGroup");
-  t.is(node.items[1].attrs.open, "(");
-  t.is(node.items[1].attrs.close, ")");
+  t.is(node.items[1].attrs.open.value, "(");
+  t.is(node.items[1].attrs.close.value, ")");
 });
 
 test("but keeps multi-celled in fences", (t) => {
@@ -559,8 +559,8 @@ test("but keeps multi-celled in fences", (t) => {
   t.is(node.items.length, 2);
   t.is(node.items[0].value, "a");
   t.is(node.items[1].type, "FencedGroup");
-  t.is(node.items[1].attrs.open, "(");
-  t.is(node.items[1].attrs.close, ")");
+  t.is(node.items[1].attrs.open.value, "(");
+  t.is(node.items[1].attrs.close.value, ")");
 });
 
 test("chains sub sup into subsup ternary", (t) => {

src/compiler/parser/index.js

@@ -1,12 +1,12 @@
 /**
- * @typedef {object} GroupAttrs
- * @property {string} open
- * @property {string} close
- * @property {string[]} seps
+ * @typedef {Record<string, string | boolean | number | null | undefined>} LiteralAttrs
  */
 
 /**
- * @typedef {Record<string, string | boolean | number | null | undefined>} LiteralAttrs
+ * @typedef {object} GroupAttrs
+ * @property {{ value: string, attrs?: LiteralAttrs }} open
+ * @property {{ value: string, attrs?: LiteralAttrs } | null} close
+ * @property {string[]} seps
  */
 
 /**

src/compiler/tokenizer/index.js

@@ -23,6 +23,7 @@ import scanners, { unhandled } from "./scanners/index.js";
  * @property {string} [name]
  * @property {boolean} [split]
  * @property {Record<string, string | number | boolean | null | undefined>} [attrs]
+ * @property {Token[]} [extraTokensAfter]
  *
  * @typedef {object} State
  * @property {number} start
@@ -67,10 +68,13 @@ export default function createTokenizer(options) {
     let nestLevel = 0;
 
     while (pos < input.length) {
-      const { type, value, end, split, ...attrs } = nextToken(input, {
-        start: pos,
-        grouping: nestLevel > 0,
-      });
+      const { type, value, end, split, extraTokensAfter, ...attrs } = nextToken(
+        input,
+        {
+          start: pos,
+          grouping: nestLevel > 0,
+        },
+      );
 
       if (split) {
         let char = "";
@@ -104,6 +108,12 @@ export default function createTokenizer(options) {
         };
       }
 
+      if (extraTokensAfter) {
+        for (const token of extraTokensAfter) {
+          yield token;
+        }
+      }
+
       pos = end;
 
       if (type === "paren.open") {

src/compiler/tokenizer/index.test.js

@@ -65,3 +65,22 @@ test("operators", (t) => {
     ],
   );
 });
+
+test("extra tokens after", (t) => {
+  const tokenize = tokenizer(options);
+  const tokens = tokenize("(mod");
+
+  t.deepEqual(tokens.next().value, {
+    type: "paren.open",
+    value: "(",
+    attrs: { lspace: "1.65ex" },
+  });
+
+  t.deepEqual(tokens.next().value, {
+    type: "operator",
+    value: "mod",
+    attrs: { lspace: 0 },
+  });
+
+  t.is(tokens.next().done, true);
+});