Splat comprehensions

commit 93088f6
Author: William C. Johnson <wcjohnson@oigroup.net>
Date:   Thu Sep 28 17:07:19 2017 -0400

    Splat comprehension parsing

src/parser/expression.js

@@ -747,7 +747,7 @@ pp.parseExprAtom = function (refShorthandDefaultPos) {
       if (this.state.inMatchAtom) {
         this.unexpected(null, "Illegal expression in match atom.");
       }
-      if (this.hasPlugin("enhancedComprehension")) {
+      if (this.hasPlugin("splatComprehension")) {
         return this.parseComprehensionArray(refShorthandDefaultPos);
       }
       node = this.startNode();
@@ -1074,7 +1074,7 @@ pp.parseObj = function (isPattern, refShorthandDefaultPos) {
   // `for` keyword begins an object comprehension.
   if (
     this.hasPlugin("lightscript") &&
-    !this.hasPlugin("enhancedComprehension") &&
+    !this.hasPlugin("splatComprehension") &&
     this.match(tt._for)
   ) {
     // ...however, `{ for: x }` is a legal JS object.
@@ -1097,18 +1097,13 @@ pp.parseObj = function (isPattern, refShorthandDefaultPos) {
       if (this.eat(tt.braceR)) break;
     }
 
-    if (
-      this.hasPlugin("enhancedComprehension") &&
-      (this.match(tt._for) || this.match(tt._case))
-    ) {
-      if (this.lookahead().type !== tt.colon) {
-        if (isPattern) {
-          this.unexpected(null, "Comprehensions are illegal in patterns.");
-        }
-        node.properties.push(this.parseSomeComprehension());
-        hasComprehension = true;
-        continue;
+    if (this.hasPlugin("splatComprehension") && this.match(tt.splatComprehension)) {
+      if (isPattern) {
+        this.unexpected(null, "Comprehensions are illegal in patterns.");
       }
+      node.properties.push(this.parseSomeComprehension());
+      hasComprehension = true;
+      continue;
     }
 
     while (this.match(tt.at)) {

src/plugins/enhancedComprehension.js → src/plugins/splatComprehension.js

@@ -1,10 +1,12 @@
 import Parser from "../parser";
-import { types as tt } from "../tokenizer/types";
+import { types as tt, TokenType } from "../tokenizer/types";
 const pp = Parser.prototype;
 
 export default function(parser) {
-  if (parser.__enhancedComprehensionPluginInstalled) return;
-  parser.__enhancedComprehensionPluginInstalled = true;
+  if (parser.__splatComprehensionPluginInstalled) return;
+  parser.__splatComprehensionPluginInstalled = true;
+
+  tt.splatComprehension = new TokenType("...for");
 
   pp.parseComprehensionArray = function(refShorthandDefaultPos) {
     const node = this.startNode();
@@ -34,10 +36,10 @@ export default function(parser) {
         if (this.eat(tt.bracketR)) break;
       }
 
-      if (this.match(tt._for)) {
+      if (this.match(tt.splatComprehension) && this.state.value === "for") {
         hasComprehension = true;
         elts.push(this.parseLoopComprehension());
-      } else if (this.match(tt._case)) {
+      } else if (this.match(tt.splatComprehension) && this.state.value === "if") {
         hasComprehension = true;
         elts.push(this.parseCaseComprehension());
       } else {
@@ -64,12 +66,12 @@ export default function(parser) {
   };
 
   pp.parseSomeComprehension = function() {
-    if (this.match(tt._for)) {
+    if (this.match(tt.splatComprehension) && this.state.value === "for") {
       return this.parseLoopComprehension();
-    } else if (this.match(tt._case)) {
+    } else if (this.match(tt.splatComprehension) && this.state.value === "if") {
       return this.parseCaseComprehension();
     } else {
-      this.unexpected(null, "Unexpected token, expected `for` or `case`");
+      this.unexpected();
     }
   };
 }

src/registerPlugins.js

@@ -6,7 +6,7 @@ import tildeCallPlugin from "./plugins/tildeCall";
 import safeCallExistentialPlugin from "./plugins/safeCallExistential";
 import bangCallPlugin from "./plugins/bangCall";
 import significantWhitespacePlugin from "./plugins/significantWhitespace";
-import enhancedComprehensionPlugin from "./plugins/enhancedComprehension";
+import splatComprehensionPlugin from "./plugins/splatComprehension";
 import syntacticPlaceholderPlugin from "./plugins/syntacticPlaceholder";
 import pipeCallPlugin from "./plugins/pipeCall";
 import { matchCoreSyntax, match } from "./plugins/match";
@@ -69,7 +69,7 @@ export default function registerPlugins(plugins, metadata) {
     dependencies: ["lightscript", "matchCoreSyntax"]
   });
 
-  registerPlugin("enhancedComprehension", enhancedComprehensionPlugin, {
+  registerPlugin("splatComprehension", splatComprehensionPlugin, {
     dependencies: [
       "lightscript", // needed for `parseIf`
       "seqExprRequiresParen"

src/tokenizer/index.js

@@ -282,6 +282,25 @@ export default class Tokenizer {
     const next2 = this.input.charCodeAt(this.state.pos + 2);
     if (next === 46 && next2 === 46) { // 46 = dot '.'
       this.state.pos += 3;
+      // splatComprehension: parse ...for and ...if
+      if (this.hasPlugin("splatComprehension")) {
+        const next3 = this.input.charCodeAt(this.state.pos);
+        const next4 = this.input.charCodeAt(this.state.pos + 1);
+        const next5 = this.input.charCodeAt(this.state.pos + 2);
+        // "...if"
+        if (next3 === 105 && next4 === 102 && !isIdentifierChar(next5)) {
+          this.state.pos += 2;
+          return this.finishToken(tt.splatComprehension, "if");
+        } else if ( // "...for"
+          next3 === 102 &&
+          next4 === 111 &&
+          next5 === 114 &&
+          !isIdentifierChar(this.input.charCodeAt(this.state.pos + 3))
+        ) {
+          this.state.pos += 3;
+          return this.finishToken(tt.splatComprehension, "for");
+        }
+      }
       return this.finishToken(tt.ellipsis);
     } else {
       ++this.state.pos;

test/fixtures/comments/options.json

@@ -7,7 +7,7 @@
     },
     "noLsc": {
       "allPlugins": true,
-      "excludePlugins": ["lightscript", "match", "enhancedComprehension", "whiteblockOnly"]
+      "excludePlugins": ["lightscript", "match", "splatComprehension", "whiteblockOnly"]
     }
   }
 }

test/fixtures/comprehension/array/for/actual.js

@@ -1 +1 @@
-[for idx i in Array(10): i]
+[ ...for idx i in Array(10): [i] ]

test/fixtures/comprehension/array/for/expected.json

@@ -1,165 +1,182 @@
 {
   "type": "File",
   "start": 0,
-  "end": 27,
+  "end": 34,
   "loc": {
     "start": {
       "line": 1,
       "column": 0
     },
     "end": {
       "line": 1,
-      "column": 27
+      "column": 34
     }
   },
   "program": {
     "type": "Program",
     "start": 0,
-    "end": 27,
+    "end": 34,
     "loc": {
       "start": {
         "line": 1,
         "column": 0
       },
       "end": {
         "line": 1,
-        "column": 27
+        "column": 34
       }
     },
     "sourceType": "script",
     "body": [
       {
         "type": "ExpressionStatement",
         "start": 0,
-        "end": 27,
+        "end": 34,
         "loc": {
           "start": {
             "line": 1,
             "column": 0
           },
           "end": {
             "line": 1,
-            "column": 27
+            "column": 34
           }
         },
         "expression": {
           "type": "ArrayComprehension",
           "start": 0,
-          "end": 27,
+          "end": 34,
           "loc": {
             "start": {
               "line": 1,
               "column": 0
             },
             "end": {
               "line": 1,
-              "column": 27
+              "column": 34
             }
           },
           "elements": [
             {
               "type": "LoopComprehension",
-              "start": 1,
-              "end": 26,
+              "start": 2,
+              "end": 32,
               "loc": {
                 "start": {
                   "line": 1,
-                  "column": 1
+                  "column": 2
                 },
                 "end": {
                   "line": 1,
-                  "column": 26
+                  "column": 32
                 }
               },
               "loop": {
                 "type": "ForInArrayStatement",
-                "start": 1,
-                "end": 26,
+                "start": 2,
+                "end": 32,
                 "loc": {
                   "start": {
                     "line": 1,
-                    "column": 1
+                    "column": 2
                   },
                   "end": {
                     "line": 1,
-                    "column": 26
+                    "column": 32
                   }
                 },
                 "idx": {
                   "type": "Identifier",
-                  "start": 9,
-                  "end": 10,
+                  "start": 13,
+                  "end": 14,
                   "loc": {
                     "start": {
                       "line": 1,
-                      "column": 9
+                      "column": 13
                     },
                     "end": {
                       "line": 1,
-                      "column": 10
+                      "column": 14
                     },
                     "identifierName": "i"
                   },
                   "name": "i"
                 },
                 "body": {
                   "type": "ExpressionStatement",
-                  "start": 25,
-                  "end": 26,
+                  "start": 29,
+                  "end": 32,
                   "loc": {
                     "start": {
                       "line": 1,
-                      "column": 25
+                      "column": 29
                     },
                     "end": {
                       "line": 1,
-                      "column": 26
+                      "column": 32
                     }
                   },
                   "expression": {
-                    "type": "Identifier",
-                    "start": 25,
-                    "end": 26,
+                    "type": "ArrayExpression",
+                    "start": 29,
+                    "end": 32,
                     "loc": {
                       "start": {
                         "line": 1,
-                        "column": 25
+                        "column": 29
                       },
                       "end": {
                         "line": 1,
-                        "column": 26
-                      },
-                      "identifierName": "i"
+                        "column": 32
+                      }
                     },
-                    "name": "i"
+                    "elements": [
+                      {
+                        "type": "Identifier",
+                        "start": 30,
+                        "end": 31,
+                        "loc": {
+                          "start": {
+                            "line": 1,
+                            "column": 30
+                          },
+                          "end": {
+                            "line": 1,
+                            "column": 31
+                          },
+                          "identifierName": "i"
+                        },
+                        "name": "i"
+                      }
+                    ]
                   }
                 },
                 "array": {
                   "type": "CallExpression",
-                  "start": 14,
-                  "end": 23,
+                  "start": 18,
+                  "end": 27,
                   "loc": {
                     "start": {
                       "line": 1,
-                      "column": 14
+                      "column": 18
                     },
                     "end": {
                       "line": 1,
-                      "column": 23
+                      "column": 27
                     }
                   },
                   "callee": {
                     "type": "Identifier",
-                    "start": 14,
-                    "end": 19,
+                    "start": 18,
+                    "end": 23,
                     "loc": {
                       "start": {
                         "line": 1,
-                        "column": 14
+                        "column": 18
                       },
                       "end": {
                         "line": 1,
-                        "column": 19
+                        "column": 23
                       },
                       "identifierName": "Array"
                     },
@@ -168,16 +185,16 @@
                   "arguments": [
                     {
                       "type": "NumericLiteral",
-                      "start": 20,
-                      "end": 22,
+                      "start": 24,
+                      "end": 26,
                       "loc": {
                         "start": {
                           "line": 1,
-                          "column": 20
+                          "column": 24
                         },
                         "end": {
                           "line": 1,
-                          "column": 22
+                          "column": 26
                         }
                       },
                       "extra": {

test/fixtures/comprehension/array/if/actual.js

@@ -0,0 +1 @@
+[...if true: [ 1 ] ]