Import (pegjs/pegjs#38): Implement syntax for clause and base support…

… in the compiler:

```
import { rule1, rule2 as alias2, ..., ruleN } from '<string with path to the .peggy file>';
```

Import clauses expected before (top) initializer code block.

All import clauses appeared in the `grammar` AST node in the `imports` node property.
This property contains array of AST `import` nodes with the`rules` and `path` properties.
`rules` contains array of the `imported_rule` AST nodes.

lib/compiler/asts.js

@@ -4,10 +4,20 @@ const visitor = require("./visitor");
 
 // AST utilities.
 const asts = {
-  findRule(ast, name) {
-    for (let i = 0; i < ast.rules.length; i++) {
-      if (ast.rules[i].name === name) {
-        return ast.rules[i];
+  findRule(ast, name, includeImports = false) {
+    for (const rule of ast.rules) {
+      if (rule.name === name) {
+        return rule;
+      }
+    }
+
+    if (includeImports && ast.imports) {
+      for (const _import of ast.imports) {
+        for (const rule of _import.rules) {
+          if (rule.alias === name) {
+            return rule;
+          }
+        }
       }
     }
 

lib/compiler/opcodes.js

@@ -53,6 +53,7 @@ const opcodes = {
   // Rules
 
   RULE:              27,   // RULE r
+  IMPORTED_RULE:     43,   // IMPORTED_RULE i, r
 
   // Failure Reporting
 

lib/compiler/passes/generate-bytecode.js

@@ -218,6 +218,10 @@ const { ALWAYS_MATCH, SOMETIMES_MATCH, NEVER_MATCH } = require("./inference-matc
 //
 //        stack.push(parseRule(r));
 //
+// [43] IMPORTED_RULE i, r
+//
+//        stack.push(parseImportedRule(i, r));
+//
 // Failure Reporting
 // -----------------
 //
@@ -249,7 +253,7 @@ const { ALWAYS_MATCH, SOMETIMES_MATCH, NEVER_MATCH } = require("./inference-matc
 // that is equivalent of an unknown match result and signals the generator that
 // runtime check for the |FAILED| is required. Trick is explained on the
 // Wikipedia page (https://en.wikipedia.org/wiki/Asm.js#Code_generation)
-function generateBytecode(ast) {
+function generateBytecode(ast, options, session) {
   const literals = [];
   const classes = [];
   const expectations = [];
@@ -882,7 +886,20 @@ function generateBytecode(ast) {
     },
 
     rule_ref(node) {
-      return [op.RULE, asts.indexOfRule(ast, node.name)];
+      const rule = asts.indexOfRule(ast, node.name);
+      if (rule < 0) {
+        for (let i = 0; i < ast.imports.length; ++i) {
+          const rule = ast.imports[i].rules.findIndex(
+            rule => rule.alias === node.name
+          );
+          if (rule >= 0) {
+            return [op.IMPORTED_RULE, i, rule];
+          }
+        }
+        session.error(`Unknown imported rule ${node.name}`, node.location);
+      }
+
+      return [op.RULE, rule];
     },
 
     literal(node) {

lib/compiler/passes/generate-js.js

@@ -531,6 +531,13 @@ function generateJS(ast, options) {
             ip += 2;
             break;
 
+          case op.IMPORTED_RULE: {    // IMPORTED_RULE i, r
+            const _import = ast.imports[bc[ip + 1]];
+            const rule = _import.rules[bc[ip + 2]];
+            parts.push(stack.push(rule.name + "()"));
+            ip += 3;
+            break;
+          }
           case op.SILENT_FAILS_ON:    // SILENT_FAILS_ON
             parts.push("peg$silentFails++;");
             ip++;

lib/compiler/passes/inference-match-result.js

@@ -46,6 +46,7 @@ function inferenceMatchResult(ast) {
   }
 
   const inference = visitor.build({
+    imported_rule: sometimesMatch,
     rule(node) {
       let oldResult;
       let count = 0;
@@ -151,7 +152,7 @@ function inferenceMatchResult(ast) {
     semantic_and: sometimesMatch,
     semantic_not: sometimesMatch,
     rule_ref(node) {
-      const rule = asts.findRule(ast, node.name);
+      const rule = asts.findRule(ast, node.name, true);
 
       return (node.match = inference(rule));
     },

lib/compiler/passes/remove-proxy-rules.js

@@ -20,7 +20,7 @@ function removeProxyRules(ast, options, session) {
             node.location,
             [{
               message: "This rule will be used",
-              location: asts.findRule(ast, to).nameLocation,
+              location: asts.findRule(ast, to, true).nameLocation,
             }]
           );
         }

lib/compiler/passes/report-duplicate-rules.js

@@ -1,28 +1,71 @@
 "use strict";
 
+const asts = require("../asts");
 const visitor = require("../visitor");
 
 // Checks that each rule is defined only once.
 function reportDuplicateRules(ast, options, session) {
-  const rules = {};
+  const defined = {};
+  const imported = {};
+
+  function checkRepository(repository, node, errorMessage, detailMessage) {
+    if (Object.prototype.hasOwnProperty.call(repository, node.name)) {
+      session.error(
+        errorMessage,
+        node.nameLocation,
+        [{
+          message: detailMessage,
+          location: repository[node.name],
+        }]
+      );
+
+      return true;
+    }
+
+    return false;
+  }
 
   const check = visitor.build({
-    rule(node) {
-      if (Object.prototype.hasOwnProperty.call(rules, node.name)) {
+    imported_rule(node) {
+      if (checkRepository(
+        imported,
+        node,
+        `Rule "${node.name}" is already imported`,
+        "Original import location"
+      )) {
+        // Do not rewrite original rule location
+        return;
+      }
+
+      imported[node.name] = node.location;
+
+      const rule = asts.findRule(ast, node.name);
+      if (rule) {
         session.error(
-          `Rule "${node.name}" is already defined`,
+          node.aliasLocation
+            ? `Rule with the same name "${node.name}" is already defined in the grammar`
+            : `Rule with the same name "${node.name}" is already defined in the grammar, try to add \`as <alias_name>\` to the imported one`,
           node.nameLocation,
           [{
-            message: "Original rule location",
-            location: rules[node.name],
+            message: "Rule defined here",
+            location: rule.nameLocation,
           }]
         );
+      }
+    },
 
+    rule(node) {
+      if (checkRepository(
+        defined,
+        node,
+        `Rule "${node.name}" is already defined`,
+        "Original rule location"
+      )) {
         // Do not rewrite original rule location
         return;
       }
 
-      rules[node.name] = node.nameLocation;
+      defined[node.name] = node.nameLocation;
     },
   });
 

lib/compiler/passes/report-undefined-rules.js

@@ -7,9 +7,9 @@ const visitor = require("../visitor");
 function reportUndefinedRules(ast, options, session) {
   const check = visitor.build({
     rule_ref(node) {
-      if (!asts.findRule(ast, node.name)) {
+      if (!asts.findRule(ast, node.name, true)) {
         session.error(
-          `Rule "${node.name}" is not defined`,
+          `Rule "${node.name}" is not defined or imported`,
           node.location
         );
       }

lib/compiler/visitor.js

@@ -28,6 +28,10 @@ const visitor = {
 
     const DEFAULT_FUNCTIONS = {
       grammar(node, ...args) {
+        if (node.imports) {
+          node.imports.forEach(_import => visit(_import, ...args));
+        }
+
         if (node.topLevelInitializer) {
           visit(node.topLevelInitializer, ...args);
         }
@@ -39,6 +43,8 @@ const visitor = {
         node.rules.forEach(rule => visit(rule, ...args));
       },
 
+      import: visitChildren("rules"),
+      imported_rule: visitNop,
       top_level_initializer: visitNop,
       initializer: visitNop,
       rule: visitExpression,

lib/peg.d.ts

@@ -45,6 +45,8 @@ declare namespace ast {
 
   /** The main Peggy AST class returned by the parser. */
   interface Grammar extends Node<"grammar"> {
+    /** List of import clauses with additional rules. */
+    imports: Import[];
     /** Initializer that run once when importing generated parser module. */
     topLevelInitializer?: TopLevelInitializer;
     /** Initializer that run each time when `parser.parse()` method in invoked. */
@@ -59,6 +61,40 @@ declare namespace ast {
     code?: SourceNode;
   }
 
+  /** The single `import` clause. */
+  interface Import extends Node<"import"> {
+    /** Path to the grammar, relative to the current file. */
+    path: string;
+    /** List of imported rules from another grammar. */
+    rules: ImportedRule[];
+  }
+
+  /**
+   * A rule with an optional alias imported from another grammar.
+   *
+   * Alias name shares the same naming scope as the ordinal rules,
+   * so they should be unique within that scope.
+   */
+  interface ImportedRule extends Node<"imported_rule"> {
+    /** Name of rule in the imported grammar. */
+    name: string;
+    /**
+     * Name of the rule to use in the grammar. This property will be set to
+     * `name` if alias was not defined in the grammar. To distinguish between
+     * values, defined in the grammar and constructed from the rule name use
+     * the `aliasLocation` property. It will be `null` in case of constructed
+     * name.
+     */
+    alias: string;
+    /** Span of the `name` string definition in the grammar. */
+    nameLocation: LocationRange;
+    /**
+     * Span of the `alias` string definition in the grammar.
+     * `null` if the alias is not defined in the grammar.
+     */
+    aliasLocation: LocationRange | null;
+  }
+
   /**
    * Base interface for all initializer nodes with the code.
    *
@@ -533,6 +569,7 @@ export namespace compiler {
     interface NodeTypes {
       /**
        * Default behavior: run visitor:
+       * - on each element in `imports`
        * - on the top level initializer, if it is defined
        * - on the initializer, if it is defined
        * - on each element in `rules`
@@ -544,6 +581,22 @@ export namespace compiler {
        */
       grammar?(node: ast.Grammar, ...args: any[]): any;
 
+      /**
+       * Default behavior: run visitor on each element in `rules`,
+       * return `undefined`
+       *
+       * @param node Node, representing one clause of imports from one another grammar
+       * @param args Any arguments passed to the `Visitor`
+       */
+      import?(node: ast.Import, ...args: any[]): any;
+      /**
+       * Default behavior: do nothing
+       *
+       * @param node Node, representing imported one and possible renamed rule
+       * @param args Any arguments passed to the `Visitor`
+       */
+      imported_rule?(node: ast.ImportedRule, ...args: any[]): any;
+
       /**
        * Default behavior: do nothing
        *

src/parser.pegjs

@@ -46,16 +46,42 @@
 // ---- Syntactic Grammar -----
 
 Grammar
-  = __ topLevelInitializer:(@TopLevelInitializer __)? initializer:(@Initializer __)? rules:(@Rule __)+ {
+  = __ imports:(@Import __)* topLevelInitializer:(@TopLevelInitializer __)? initializer:(@Initializer __)? rules:(@Rule __)+ {
       return {
         type: "grammar",
+        imports,
         topLevelInitializer,
         initializer,
         rules,
         location: location()
       };
     }
 
+Import
+  = 'import' __
+    '{' __ rules:ImportedRule|.., __ ',' __| (__ ',')? __ '}' __
+    'from' __ path:StringLiteral EOS
+  {
+    return {
+      type: "import",
+      path,
+      rules,
+      location: location(),
+    };
+  }
+
+ImportedRule
+  = name:IdentifierName alias:(__ 'as' __ @IdentifierName)? {
+    return {
+      type: "imported_rule",
+      name: name[0],
+      alias: alias !== null ? alias[0] : name[0],
+      nameLocation: name[1],
+      aliasLocation: alias !== null ? alias[1] : null,
+      location: location(),
+    };
+  }
+
 TopLevelInitializer
   = "{" code:CodeBlock "}" EOS {
       return {

test/behavior/generated-parser-behavior.spec.js

@@ -2937,7 +2937,7 @@ Error: Expected "{", code block, comment, end of line, identifier, or whitespace
 
         // Check that all messages present in the list
         expect(messages).to.include.members([
-          "Rule \"missingRule\" is not defined",
+          "Rule \"missingRule\" is not defined or imported",
           "Rule \"duplicatedRule\" is already defined",
           "Label \"duplicatedLabel\" is already defined",
           "Possible infinite loop when parsing (left recursion: duplicatedRule -> start -> leftRecursion -> duplicatedRule)",

test/unit/compiler/passes/report-undefined-rules.spec.js

@@ -11,7 +11,7 @@ const expect = chai.expect;
 describe("compiler pass |reportUndefinedRules|", () => {
   it("reports undefined rules", () => {
     expect(pass).to.reportError("start = undefined", {
-      message: "Rule \"undefined\" is not defined",
+      message: "Rule \"undefined\" is not defined or imported",
       location: {
         source: undefined,
         start: { offset: 8, line: 1, column: 9 },