Merge pull request #45 from matthijsgroen/experiment-generic-ast

Experiment generic ast

CHANGELOG.md

@@ -6,9 +6,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.12.0] - UNRELEASED
+
+### Changes
+
+- Deduplication of choices is improved. Now also works for text output, and can
+  deduplicate items when one of them has a comment.
+
+  ```
+  a | a | a (* comment *) | b => a (* comment *) | b
+  ```
+
+- Improve height of overview diagrams
+- Update color scheme of dark theme
+- Improved styling of blockquotes in markdown `>`
+
+### Fixes
+
+- Exception when non-terminal lacks definition
+
 ## [1.11.1] - 2020-09-23
 
-## Fixes
+### Fixes
 
 - Overview diagrams should also optimize its sub elements
 

examples/optimizations.ebnf

@@ -18,7 +18,7 @@
   Also, links to sites, like https://github.com/matthijsgroen/ebnf2railroad
   will be converted into a clickable link.
 
-  ### Text decration
+  ### Text decoration
 
   Text markup is also a **breeze**. Using asterisks and _underscores_ to place
   emphasis.
@@ -119,6 +119,9 @@ nested choices with multiple optionals = "a" | [ "b" | "c"] | [ "d" | "e" ];
 *)
 
 nested choices = "a" | ( "b" | "c" );
+
+duplicate choices = "a" | ( "b" | "c" | "a" ) | "b";
+
 (*
 
   ## Optimization: Ungrouping
@@ -130,6 +133,7 @@ nested choices = "a" | ( "b" | "c" );
 
   ```
   nested choices = "a" | ( "b" | "c" );
+  duplicate choices = "a" | ( "b" | "c" | "a" ) | "b";
   ```
 *)
 
@@ -180,13 +184,14 @@ number value = "fixing ref";
 
 comments = comment in choice with optional | comment in one or more before | comment_in_one_or_more_in
   | comment in one or more after | comment in one or more with repeater
-  | comment in nested choices with multiple optionals | comment in one or more with repeater after;
+  | comment in nested choices with multiple optionals | comment in one or more with repeater after | comment before optional ;
 
 comment in choice with optional = [ "a" | "b" (* comment *) ];
 comment in one or more before = "a" (* comment *), { "a" };
 comment_in_one_or_more_in = "a", { "a" (* comment *) };
 comment in one or more after = "a", { "a" } (* comment *);
 comment in one or more with repeater = "a", "b" (* comment *), "c", { "d", "e", "b", "c" };
+comment before optional = "a", (* comment *) [ "d", "e", "b", "c" ];
 comment in nested choices with multiple optionals = "a" | [ "b" (* comment *)
   | "c"] (* comment *) | [ "d" | "e" ];
 

package.json

@@ -22,7 +22,7 @@
   },
   "scripts": {
     "build-parser": "jison src/ebnf.jison -o src/ebnf-parser.js",
-    "test": "mocha",
+    "test": "mocha --recursive",
     "lint": "eslint src/ test/",
     "update-examples": "bin/ebnf2railroad.js examples/json.ebnf --title JSON; bin/ebnf2railroad.js examples/ebnf.ebnf --title EBNF; bin/ebnf2railroad.js examples/optimizations.ebnf",
     "publish": ".travis/publish-site.sh"

src/ast/ebnf-transform.js

@@ -0,0 +1,83 @@
+const { traverse } = require("./traverse");
+
+const NodeTypes = {
+  Root: 0,
+  Production: 1,
+  Comment: 2,
+  Terminal: 3,
+  NonTerminal: 4,
+  Choice: 5,
+  Group: 6,
+  Sequence: 7,
+  Optional: 8,
+  Repetition: 9,
+  Special: 10,
+  ExceptTerminal: 11,
+  ExceptNonTerminal: 12
+};
+
+const identifyNode = node => {
+  if (Array.isArray(node)) return NodeTypes.Root;
+  if (node.definition) return NodeTypes.Production;
+  if (node.choice) return NodeTypes.Choice;
+  if (node.group) return NodeTypes.Group;
+  if (node.comment) return NodeTypes.Comment;
+  if (node.sequence) return NodeTypes.Sequence;
+  if (node.optional) return NodeTypes.Optional;
+  if (node.repetition) return NodeTypes.Repetition;
+  // leafs
+  if (node.specialSequence) return NodeTypes.Special;
+  if (node.terminal) return NodeTypes.Terminal;
+  if (node.nonTerminal) return NodeTypes.NonTerminal;
+  if (node.exceptTerminal) return NodeTypes.ExceptTerminal;
+  if (node.exceptNonTerminal) return NodeTypes.ExceptNonTerminal;
+};
+
+const travelers = {
+  [NodeTypes.Root]: (node, next) => node.map(next),
+  [NodeTypes.Production]: (node, next) => ({
+    ...node,
+    definition: next(node.definition)
+  }),
+  [NodeTypes.Choice]: (node, next) => ({
+    ...node,
+    choice: node.choice.map(next)
+  }),
+  [NodeTypes.Group]: (node, next) => ({
+    ...node,
+    group: next(node.group)
+  }),
+  [NodeTypes.Sequence]: (node, next) => ({
+    ...node,
+    sequence: node.sequence.map(next)
+  }),
+  [NodeTypes.Optional]: (node, next) => ({
+    ...node,
+    optional: next(node.optional)
+  }),
+  [NodeTypes.Repetition]: (node, next) => ({
+    ...node,
+    repetition: next(node.repetition)
+  })
+};
+
+const ebnfTransform = traverse(identifyNode)(travelers);
+
+const ebnfOptimizer = transformers => ast => {
+  const optimize = ebnfTransform(transformers);
+  let current = ast;
+  let transformed = optimize(ast);
+  while (current !== transformed) {
+    current = transformed;
+    transformed = optimize(current);
+  }
+  return transformed;
+};
+
+module.exports = {
+  ebnfTransform,
+  ebnfOptimizer,
+  NodeTypes,
+  identifyNode,
+  travelers
+};

src/ast/optimizers/choice-clustering.js

@@ -0,0 +1,180 @@
+const { NodeTypes } = require("../ebnf-transform");
+
+const skipFirst = list =>
+  [
+    list.some(e => e === "skip" || e.skip) && { skip: true },
+    ...list.filter(e => e !== "skip" && !e.skip)
+  ].filter(Boolean);
+
+const equalElements = (first, second) =>
+  JSON.stringify(first) === JSON.stringify(second);
+
+module.exports = {
+  [NodeTypes.Choice]: current => {
+    if (!current.choice) return current;
+
+    const isCertain = elem =>
+      (elem.terminal && elem) || (elem.nonTerminal && elem);
+
+    const groupElements = elements => {
+      const allSet = elements.every(f => f);
+      if (!allSet) return {};
+      return elements.reduce((acc, elem) => {
+        const key = JSON.stringify(elem);
+        acc[key] = (acc[key] || 0) + 1;
+        return acc;
+      }, {});
+    };
+    const countSame = groupElements => {
+      const amounts = Object.values(groupElements);
+      return Math.max(...amounts);
+    };
+
+    const collectCertainFirstElements = current.choice.map(
+      elem => isCertain(elem) || (elem.sequence && isCertain(elem.sequence[0]))
+    );
+    const collectCertainLastElements = current.choice.map(
+      elem =>
+        isCertain(elem) ||
+        (elem.sequence && isCertain(elem.sequence[elem.sequence.length - 1]))
+    );
+    const groupFirsts = groupElements(collectCertainFirstElements);
+    const groupLasts = groupElements(collectCertainLastElements);
+
+    // most wins, optimize, repeat
+    const maxFirstsEqual = countSame(groupFirsts);
+    const maxLastsEqual = countSame(groupLasts);
+    if (Math.max(maxFirstsEqual, maxLastsEqual) > 1) {
+      const beforeChoices = [];
+      const afterChoices = [];
+      if (maxFirstsEqual >= maxLastsEqual) {
+        const firstElement = Object.entries(groupFirsts).find(
+          ([, value]) => value === maxFirstsEqual
+        )[0];
+
+        // now filter all choices that have this as first element, placing
+        // the others in 'leftOverChoices'
+        let hasEmpty = false;
+        let found = false;
+        const newChoices = collectCertainFirstElements
+          .map((elem, index) => {
+            // if not match, add production choice to leftOverChoices.
+            if (JSON.stringify(elem) === firstElement) {
+              found = true;
+              // strip off element of chain.
+              const choice = current.choice[index];
+              if (choice.sequence) {
+                return { ...choice, sequence: choice.sequence.slice(1) };
+              } else {
+                hasEmpty = true;
+              }
+            } else {
+              (found ? afterChoices : beforeChoices).push(
+                current.choice[index]
+              );
+            }
+          })
+          .filter(Boolean);
+        const newElements = [
+          JSON.parse(firstElement),
+          newChoices.length > 0 &&
+            hasEmpty && {
+              optional:
+                newChoices.length == 1 ? newChoices[0] : { choice: newChoices }
+            },
+          newChoices.length > 0 &&
+            !hasEmpty &&
+            (newChoices.length == 1 ? newChoices[0] : { choice: newChoices })
+        ].filter(Boolean);
+        const replacementElement =
+          newElements.length > 1 ? { sequence: newElements } : newElements[0];
+
+        const finalResult =
+          beforeChoices.length + afterChoices.length > 0
+            ? {
+                choice: []
+                  .concat(beforeChoices)
+                  .concat(replacementElement)
+                  .concat(afterChoices)
+              }
+            : replacementElement;
+
+        return finalResult;
+      } else {
+        const lastElement = Object.entries(groupLasts).find(
+          ([, value]) => value === maxLastsEqual
+        )[0];
+
+        // now filter all choices that have this as first element, placing
+        // the others in 'leftOverChoices'
+        let hasEmpty = false;
+        let found = false;
+        const newChoices = collectCertainLastElements
+          .map((elem, index) => {
+            // if not match, add production choice to leftOverChoices.
+            if (JSON.stringify(elem) === lastElement) {
+              found = true;
+              // strip off element of chain.
+              const choice = current.choice[index];
+              if (choice.sequence) {
+                return { ...choice, sequence: choice.sequence.slice(0, -1) };
+              } else {
+                hasEmpty = true;
+              }
+            } else {
+              (found ? afterChoices : beforeChoices).push(
+                current.choice[index]
+              );
+            }
+          })
+          .filter(Boolean);
+        const newElements = [
+          newChoices.length > 0 &&
+            hasEmpty && {
+              optional:
+                newChoices.length == 1 ? newChoices[0] : { choice: newChoices }
+            },
+          newChoices.length > 0 &&
+            !hasEmpty &&
+            (newChoices.length == 1 ? newChoices[0] : { choice: newChoices }),
+          JSON.parse(lastElement)
+        ].filter(Boolean);
+        const replacementElement =
+          newElements.length > 1 ? { sequence: newElements } : newElements[0];
+
+        const finalResult =
+          beforeChoices.length + afterChoices.length > 0
+            ? {
+                choice: []
+                  .concat(beforeChoices)
+                  .concat(replacementElement)
+                  .concat(afterChoices)
+              }
+            : replacementElement;
+
+        return finalResult;
+      }
+    }
+
+    // Merge remaining choices
+    const result = {
+      ...current,
+      choice: skipFirst(
+        current.choice
+          .map(item => {
+            const optimizedItem = item;
+            if (optimizedItem.choice) {
+              return optimizedItem.choice;
+            } else {
+              return [optimizedItem];
+            }
+          })
+          .reduce((acc, item) => acc.concat(item), [])
+      )
+    };
+    if (equalElements(result, current)) {
+      return current;
+    }
+    return result;
+  }
+};

src/ast/optimizers/choice-with-skip.js

@@ -0,0 +1,36 @@
+const { NodeTypes } = require("../ebnf-transform");
+
+module.exports = {
+  [NodeTypes.Optional]: current => {
+    if (!current.optional || !current.optional.choice) {
+      return current;
+    }
+    return {
+      choice: [{ skip: true }].concat(
+        current.optional.choice
+          .filter(node => !node.skip)
+          .map(node => (node.repetition ? { ...node, skippable: false } : node))
+      )
+    };
+  },
+  [NodeTypes.Choice]: current => {
+    if (!current.choice) {
+      return current;
+    }
+    const hasSkippableRepetition = current.choice.some(
+      node => node.repetition && node.skippable
+    );
+    if (hasSkippableRepetition) {
+      return {
+        choice: [{ skip: true }].concat(
+          current.choice
+            .filter(node => !node.skip)
+            .map(
+              node => (node.repetition ? { ...node, skippable: false } : node)
+            )
+        )
+      };
+    }
+    return current;
+  }
+};