Enhanced EOF handling

.vscode/launch.json

@@ -8,7 +8,7 @@
       "program": "${workspaceRoot}/node_modules/.bin/_mocha",
       "stopOnEntry": false,
       "args": [
-        "test/JSONRecovery.spec.js",
+        "test/Lookahead.spec.js",
         "--ui",
         "bdd",
         "--reporter",

src/Grammars/W3CEBNF.ts

@@ -18,8 +18,6 @@
 // RULE_S	::=	#x9 | #xA | #xD | #x20
 // Comment	::=	'/*' ( [^*] | '*'+ [^*/] )* '*'* '*/'
 
-import { findChildrenByType } from '../SemanticHelpers';
-
 import { IRule, Parser as _Parser, IToken } from '..';
 import { findRuleByName } from '../Parser';
 
@@ -36,12 +34,12 @@ namespace BNF {
       fragment: true
     }, {
       name: 'Production',
-      bnf: [['NCName', 'RULE_S*', '"::="', 'RULE_WHITESPACE*', '%Choice', 'RULE_WHITESPACE*', 'RULE_EOL+', 'RULE_S*']]
+      bnf: [['NCName', 'RULE_S*', '"::="', 'RULE_WHITESPACE*', 'Choice', 'RULE_WHITESPACE*', 'RULE_EOL+', 'RULE_S*']]
     }, {
       name: 'NCName',
       bnf: [[/[a-zA-Z][a-zA-Z_0-9]*/]]
     }, {
-      name: '%Choice',
+      name: 'Choice',
       bnf: [['SequenceOrDifference', '%_Choice_1*']],
       fragment: true
     }, {
@@ -50,27 +48,21 @@ namespace BNF {
       fragment: true
     }, {
       name: 'SequenceOrDifference',
-      bnf: [['RecoverRule?', '%Item', 'RULE_WHITESPACE*', '%_Item_1?']]
+      bnf: [['Item', 'RULE_WHITESPACE*', '%_Item_1?']]
     }, {
       name: '%_Item_1',
-      bnf: [['Minus', '%Item'], ['%Item*']],
+      bnf: [['Minus', 'Item'], ['Item*']],
       fragment: true
     }, {
       name: 'Minus',
       bnf: [['"-"']]
     }, {
-      name: '%Item',
-      bnf: [['RULE_WHITESPACE*', 'PrimaryPreDecoration?', '%Primary', 'PrimaryDecoration?']],
+      name: 'Item',
+      bnf: [['RULE_WHITESPACE*', '%Primary', 'PrimaryDecoration?']],
       fragment: true
     }, {
       name: 'PrimaryDecoration',
       bnf: [['"?"'], ['"*"'], ['"+"']]
-    }, {
-      name: 'PrimaryPreDecoration',
-      bnf: [['&"[ebnf://"', "'['", 'DecorationName', '%Url1?', '"]"']]
-    }, {
-      name: 'RecoverRule',
-      bnf: [['RULE_WHITESPACE*', '"[recover://"', 'NCName', '"]"']]
     }, {
       name: 'DecorationName',
       bnf: [['"ebnf://"', /[^\x5D#]+/]]
@@ -86,10 +78,11 @@ namespace BNF {
       fragment: true
     }, {
       name: 'SubItem',
-      bnf: [['"("', 'RULE_WHITESPACE*', '%Choice', 'RULE_WHITESPACE*', '")"']]
+      bnf: [['"("', 'RULE_WHITESPACE*', 'Choice', 'RULE_WHITESPACE*', '")"']]
     }, {
       name: 'StringLiteral',
-      bnf: [[`'"'`, /[^"]*/, `'"'`], [`"'"`, /[^']*/, `"'"`]]
+      bnf: [[`'"'`, /[^"]*/, `'"'`], [`"'"`, /[^']*/, `"'"`]],
+      pinned: 1
     }, {
       name: 'CharCode',
       bnf: [['"#x"', /[0-9a-zA-Z]+/]]
@@ -126,7 +119,7 @@ namespace BNF {
       bnf: [['"/*"', '%RULE_Comment_Body*', '"*/"']]
     }, {
       name: '%RULE_Comment_Body',
-      bnf: [[/[^*]/], ['"*"+', /[^/]*/]],
+      bnf: [['!"*/"', /[^*]/]],
       fragment: true
     }, {
       name: 'RULE_EOL',
@@ -153,26 +146,23 @@ namespace BNF {
 
   function getBNFRule(name: string | RegExp, parser: Parser): string {
     if (typeof name == 'string') {
-      let decoration = decorationRE.exec(name);
-      let preDecoration = preDecorationRE.exec(name);
-
-      let preDecorationText = preDecoration ? ' /* ' + preDecoration[0] : '';
-      let decorationText = decoration ? decoration[0] + ' ' : '';
-
-      if (preDecorationText) decorationText = decorationText + ' */';
+      if (preDecorationRE.test(name))
+        return '';
 
       let subexpression = subExpressionRE.test(name);
 
       if (subexpression) {
+        let decoration = decorationRE.exec(name);
+        let decorationText = decoration ? decoration[0] + ' ' : '';
         let lonely = isLonelyRule(name, parser);
 
         if (lonely)
-          return preDecorationText + getBNFBody(name, parser) + decorationText;
+          return getBNFBody(name, parser) + decorationText;
 
-        return preDecorationText + '(' + getBNFBody(name, parser) + ')' + decorationText;
+        return '(' + getBNFBody(name, parser) + ')' + decorationText;
       }
 
-      return name.replace(preDecorationRE, preDecorationText) + (preDecorationText ? ' */' : '');
+      return name;
     } else {
       return name.source
         .replace(/\\(?:x|u)([a-zA-Z0-9]+)/g, '#x$1')
@@ -257,10 +247,6 @@ namespace BNF {
         case 'StringLiteral':
           bnfSeq.push(preDecoration + x.text + decoration);
           break;
-
-        case 'RecoverRule':
-          bnfSeq["recover"] = findChildrenByType(x, 'NCName')[0].text;
-          break;
         case 'CharCode':
         case 'CharClass':
           if (decoration || preDecoration) {
@@ -276,8 +262,6 @@ namespace BNF {
             bnfSeq.push(convertRegex(x.text));
           }
           break;
-
-        case 'PrimaryPreDecoration':
         case 'PrimaryDecoration':
           break;
         default:

src/Parser.ts

@@ -214,6 +214,9 @@ export class Parser {
     return 'CANNOT EMIT SOURCE FROM BASE Parser';
   }
 
+  // lookup(txt: string, target: string): boolean {
+
+  // }
   parse(txt: string, target: string, recursion = 0): IToken {
     let out = null;
 
@@ -229,22 +232,22 @@ export class Parser {
 
     let targetLex = findRuleByName(type.name, this);
 
-    if (!targetLex && type.name == 'EOF' && txt.length) {
-      return null;
-    }
-
-    if (txt === "") {
-      return {
-        type: 'EOF',
-        text: '',
-        rest: '',
-        start: 0,
-        end: 0,
-        fullText: '',
-        errors: [],
-        children: [],
-        parent: null
-      };
+    if (type.name == 'EOF') {
+      if (txt.length) {
+        return null;
+      } else if (txt.length == 0) {
+        return {
+          type: 'EOF',
+          text: '',
+          rest: '',
+          start: 0,
+          end: 0,
+          fullText: '',
+          errors: [],
+          children: [],
+          parent: null
+        };
+      }
     }
 
 
@@ -321,35 +324,26 @@ export class Parser {
 
               let foundAtLeastOne = false;
 
-              if (localTarget.lookupNegative) {
-                if (!tmpTxt.length)
-                  continue;
-              }
-
               do {
                 got = this.parse(tmpTxt, localTarget.name, recursion + 1);
 
                 // rule ::= "true" ![a-zA-Z]
                 // negative lookup, if it does not matches, we should continue
                 if (localTarget.lookupNegative) {
                   if (got)
-                    return;
+                    return /* cancel this path */;
                   break;
                 }
 
                 if (localTarget.lookupPositive) {
-                  if (!got || got.type == 'EOF')
+                  if (!got)
                     return;
                 }
 
-                if (got && got.type == 'EOF')
-                  continue;
-
-                if (!got && localTarget.isOptional)
-                  break;
-
                 if (!got) {
-                  if (foundAtLeastOne && localTarget.atLeastOne ? tmp : null)
+                  if (localTarget.isOptional)
+                    break;
+                  if (localTarget.atLeastOne && foundAtLeastOne)
                     break;
                 }
 
@@ -456,6 +450,10 @@ export class Parser {
       }
     }
 
+    if (!out) {
+      printable && console.log(target + ' NOT RESOLVED FROM ' + txt);
+    }
+
     return out;
   }
 

test/EOF.spec.ts

@@ -0,0 +1,39 @@
+declare var describe, it, require;
+
+import { Grammars, Parser, IToken } from '../dist';
+import { testParseTokenFailsafe, describeTree, printBNF, testParseToken } from './TestHelpers';
+
+let inspect = require('util').inspect;
+let expect = require('expect');
+
+describe('EOF', function () {
+  let parser = new Grammars.Custom.Parser(`
+Rule ::= Item* EOF
+Item ::= Space? Rules {recoverUntil=Space, fragment=true}
+Rules ::= "true" | "false"
+Space ::= " "+ | EOF
+`, {});
+
+  testParseTokenFailsafe(parser, 'true', null, (doc) => {
+    expect(doc.errors.length).toEqual(0);
+  });
+
+  testParseTokenFailsafe(parser, 'true false true', null, (doc) => {
+    expect(doc.errors.length).toEqual(0);
+  });
+});
+
+describe('EOF1', function () {
+  let parser = new Grammars.Custom.Parser(`
+Rule ::= Rules EOF {pin=1}
+Rules ::= "true" | "false"
+`, {});
+
+  testParseTokenFailsafe(parser, 'true', null, (doc) => {
+    expect(doc.errors.length).toEqual(0);
+  });
+
+  testParseTokenFailsafe(parser, 'true false true', null, (doc) => {
+    expect(doc.errors.length).toEqual(1);
+  });
+});