Text nodes: Implement text nodes

Implement a new syntax to extract matched strings from expressions. For
example, instead of:

  identifier = first:[a-zA-Z_] rest:[a-zA-Z0-9_]* { return first + rest.join(""); }

you can now just write:

  identifier = $([a-zA-Z_] [a-zA-Z0-9_]*)

This is useful mostly for "lexical" rules at the bottom of many
grammars.

Note that structured match results are still built for the expressions
prefixed by "$", they are just ignored. I plan to optimize this later
(sometime after the code generator rewrite).

README.md

@@ -314,6 +314,11 @@ the `options` variable.
 
 Note that curly braces in the predicate code must be balanced.
 
+#### $ *expression*
+
+Try to match the expression. If the match succeeds, return the matched string
+instead of the match result.
+
 #### *label* : *expression*
 
 Match the expression and remember its match result under given label. The label

lib/compiler/passes/allocate-registers.js

@@ -212,6 +212,7 @@ module.exports = function(ast) {
         computeExpressionScopedReuseResult(node);
       },
 
+    text:         computeExpressionScopedReuseResultSavePos,
     simple_and:   computeExpressionScopedReuseResultSavePos,
     simple_not:   computeExpressionScopedReuseResultSavePos,
     semantic_and: computeParams,

lib/compiler/passes/generate-code.js

@@ -604,6 +604,13 @@ module.exports = function(ast, options) {
           "sequence.inner": [
             '#{r(node.resultIndex)} = [#{map(pluck(node.elements, "resultIndex"), r).join(", ")}];'
           ],
+          text: [
+            '#{r(node.posIndex)} = pos;',
+            '#block emit(node.expression)',
+            'if (#{r(node.resultIndex)} !== null) {',
+            '  #{r(node.resultIndex)} = input.substring(pos, #{r(node.posIndex)});',
+            '}'
+          ],
           simple_and: [
             '#{r(node.posIndex)} = pos;',
             'reportFailures++;',
@@ -813,6 +820,7 @@ module.exports = function(ast, options) {
 
     labeled: function(node) { return emit(node.expression); },
 
+    text:         emitSimple("text"),
     simple_and:   emitSimple("simple_and"),
     simple_not:   emitSimple("simple_not"),
     semantic_and: emitSimple("semantic_and"),

lib/compiler/passes/remove-proxy-rules.js

@@ -30,6 +30,7 @@ module.exports = function(ast) {
       choice:       replaceInSubnodes("alternatives"),
       sequence:     replaceInSubnodes("elements"),
       labeled:      replaceInExpression,
+      text:         replaceInExpression,
       simple_and:   replaceInExpression,
       simple_not:   replaceInExpression,
       semantic_and: nop,

lib/compiler/passes/report-left-recursion.js

@@ -36,6 +36,7 @@ module.exports = function(ast) {
       },
 
     labeled:      checkExpression,
+    text:         checkExpression,
     simple_and:   checkExpression,
     simple_not:   checkExpression,
     semantic_and: nop,

lib/compiler/passes/report-missing-rules.js

@@ -18,6 +18,7 @@ module.exports = function(ast) {
     action:       checkExpression,
     sequence:     checkSubnodes("elements"),
     labeled:      checkExpression,
+    text:         checkExpression,
     simple_and:   checkExpression,
     simple_not:   checkExpression,
     semantic_and: nop,

spec/compiler/passes/allocate-registers.spec.js

@@ -177,6 +177,27 @@ describe("compiler pass |allocateRegisters|", function() {
     });
   });
 
+  describe("for text", function() {
+    it("allocates a position register", function() {
+      expect(pass).toChangeAST('start = $"a"', savePosDetails);
+    });
+
+    it("reuses its own result register for the expression", function() {
+      expect(pass).toChangeAST('start = $"a"', reuseResultDetails);
+    });
+
+    it("creates a new scope", function() {
+      expect(pass).toChangeAST('start = $(a:"a") { }', scopedDetails);
+    });
+
+    it("unblocks registers blocked by its children", function() {
+      expect(pass).toChangeAST(
+        'start = ($(a:"a") "b") ("c" "d")',
+        unblockedDetails
+      );
+    });
+  });
+
   describe("for simple and", function() {
     it("allocates a position register", function() {
       expect(pass).toChangeAST('start = &"a"', savePosDetails);

spec/compiler/passes/remove-proxy-rules.spec.js

@@ -60,6 +60,10 @@ describe("compiler pass |removeProxyRules|", function() {
     expect(pass).toChangeAST(proxyGrammar('start = label:proxy'), simpleDetails);
   });
 
+  it("removes proxy rule from a text", function() {
+    expect(pass).toChangeAST(proxyGrammar('start = $proxy'), simpleDetails);
+  });
+
   it("removes proxy rule from a simple and", function() {
     expect(pass).toChangeAST(proxyGrammar('start = &proxy'), simpleDetails);
   });

spec/compiler/passes/report-left-recursion.spec.js

@@ -63,6 +63,10 @@ describe("compiler pass |reportLeftRecursion|", function() {
     expect(pass).toReportLeftRecursionIn('start = label:start');
   });
 
+  it("reports left recursion inside a text", function() {
+    expect(pass).toReportLeftRecursionIn('start = $start');
+  });
+
   it("reports left recursion inside a simple and", function() {
     expect(pass).toReportLeftRecursionIn('start = &start');
   });

spec/compiler/passes/report-missing-rules.spec.js

@@ -64,6 +64,10 @@ describe("compiler pass |reportMissingRules|", function() {
     expect(pass).toReportMissingRuleIn('start = label:missing');
   });
 
+  it("reports missing rule referenced from a text", function() {
+    expect(pass).toReportMissingRuleIn('start = $missing');
+  });
+
   it("reports missing rule referenced from a simple and", function() {
     expect(pass).toReportMissingRuleIn('start = &missing');
   });

spec/generated-parser.spec.js

@@ -353,6 +353,14 @@ describe("generated parser", function() {
       });
     });
 
+    describe("text matching", function() {
+      it("matches correctly", function() {
+        var parser = PEG.buildParser('start = $("a" "b" "c")', options);
+
+        expect(parser).toParse("abc", "abc");
+      });
+    });
+
     describe("simple and matching", function() {
       it("matches correctly", function() {
         var parser = PEG.buildParser('start = &"a" "a"', options);

spec/parser.spec.js

@@ -272,6 +272,10 @@ describe("PEG.js grammar parser", function() {
 
   /* Canonical prefixed is "!\"abcd\"". */
   it("parses prefixed", function() {
+    expect('start = $"abcd"?' ).toParseAs(oneRuleGrammar({
+      type:       "text",
+      expression: optionalLiteral
+    }));
     expect('start = &{ code }').toParseAs(oneRuleGrammar({
       type: "semantic_and",
       code: " code "

src/parser.pegjs

@@ -88,7 +88,13 @@ labeled
   / prefixed
 
 prefixed
-  = and code:action {
+  = dollar expression:suffixed {
+      return {
+        type:       "text",
+        expression: expression
+      };
+    }
+  / and code:action {
       return {
         type: "semantic_and",
         code: code
@@ -169,6 +175,7 @@ semicolon = ";" __ { return ";"; }
 slash     = "/" __ { return "/"; }
 and       = "&" __ { return "&"; }
 not       = "!" __ { return "!"; }
+dollar    = "$" __ { return "$"; }
 question  = "?" __ { return "?"; }
 star      = "*" __ { return "*"; }
 plus      = "+" __ { return "+"; }