Add support for invalid escapes in tagged templates

Per the stage-3 TC39 proposal:
https://github.com/tc39/proposal-template-literal-revision

README.md

@@ -118,3 +118,4 @@ require("babylon").parse("code", {
  - `functionBind`
  - `functionSent`
  - `dynamicImport`
+ - `templateInvalidEscapes`

ast/spec.md

@@ -949,7 +949,7 @@ interface TemplateElement <: Node {
   type: "TemplateElement";
   tail: boolean;
   value: {
-    cooked: string;
+    cooked: string | null;
     raw: string;
   };
 }

src/parser/expression.js

@@ -321,7 +321,7 @@ pp.parseSubscripts = function (base, startPos, startLoc, noCalls) {
     } else if (this.match(tt.backQuote)) {
       const node = this.startNodeAt(startPos, startLoc);
       node.tag = base;
-      node.quasi = this.parseTemplate();
+      node.quasi = this.parseTemplate(true);
       base = this.finishNode(node, "TaggedTemplateExpression");
     } else {
       return base;
@@ -510,7 +510,7 @@ pp.parseExprAtom = function (refShorthandDefaultPos) {
       return this.parseNew();
 
     case tt.backQuote:
-      return this.parseTemplate();
+      return this.parseTemplate(false);
 
     case tt.doubleColon:
       node = this.startNode();
@@ -680,8 +680,11 @@ pp.parseNew = function () {
 
 // Parse template expression.
 
-pp.parseTemplateElement = function () {
+pp.parseTemplateElement = function (isTagged) {
   const elem = this.startNode();
+  if (!isTagged && this.state.value === null) {
+    this.raise(this.state.start, "Invalid escape sequence in template"); // TODO more precise error locations
+  }
   elem.value = {
     raw: this.input.slice(this.state.start, this.state.end).replace(/\r\n?/g, "\n"),
     cooked: this.state.value
@@ -691,17 +694,17 @@ pp.parseTemplateElement = function () {
   return this.finishNode(elem, "TemplateElement");
 };
 
-pp.parseTemplate = function () {
+pp.parseTemplate = function (isTagged) {
   const node = this.startNode();
   this.next();
   node.expressions = [];
-  let curElt = this.parseTemplateElement();
+  let curElt = this.parseTemplateElement(isTagged);
   node.quasis = [curElt];
   while (!curElt.tail) {
     this.expect(tt.dollarBraceL);
     node.expressions.push(this.parseExpression());
     this.expect(tt.braceR);
-    node.quasis.push(curElt = this.parseTemplateElement());
+    node.quasis.push(curElt = this.parseTemplateElement(isTagged));
   }
   this.next();
   return this.finishNode(node, "TemplateLiteral");

src/tokenizer/index.js

@@ -600,17 +600,17 @@ export default class Tokenizer {
 
   // Read a string value, interpreting backslash-escapes.
 
-  readCodePoint() {
+  readCodePoint(throwOnInvalid = true) {
     const ch = this.input.charCodeAt(this.state.pos);
     let code;
 
-    if (ch === 123) {
+    if (ch === 123) { // '{'
       const codePos = ++this.state.pos;
-      code = this.readHexChar(this.input.indexOf("}", this.state.pos) - this.state.pos);
+      code = this.readHexChar(this.input.indexOf("}", this.state.pos) - this.state.pos, throwOnInvalid);
       ++this.state.pos;
-      if (code > 0x10FFFF) this.raise(codePos, "Code point out of bounds");
+      if (throwOnInvalid && code > 0x10FFFF) this.raise(codePos, "Code point out of bounds");
     } else {
-      code = this.readHexChar(4);
+      code = this.readHexChar(4, throwOnInvalid);
     }
     return code;
   }
@@ -623,7 +623,7 @@ export default class Tokenizer {
       if (ch === quote) break;
       if (ch === 92) { // '\'
         out += this.input.slice(chunkStart, this.state.pos);
-        out += this.readEscapedChar(false);
+        out += this.readEscapedChar(false, true);
         chunkStart = this.state.pos;
       } else {
         if (isNewLine(ch)) this.raise(this.state.start, "Unterminated string constant");
@@ -637,7 +637,7 @@ export default class Tokenizer {
   // Reads template string tokens.
 
   readTmplToken() {
-    let out = "", chunkStart = this.state.pos;
+    let out = "", chunkStart = this.state.pos, containsInvalid = false;
     for (;;) {
       if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated template");
       const ch = this.input.charCodeAt(this.state.pos);
@@ -652,11 +652,16 @@ export default class Tokenizer {
           }
         }
         out += this.input.slice(chunkStart, this.state.pos);
-        return this.finishToken(tt.template, out);
+        return this.finishToken(tt.template, containsInvalid ? null : out);
       }
       if (ch === 92) { // '\'
         out += this.input.slice(chunkStart, this.state.pos);
-        out += this.readEscapedChar(true);
+        const escaped = this.readEscapedChar(true, !this.hasPlugin("templateInvalidEscapes"));
+        if (escaped === null) {
+          containsInvalid = true;
+        } else {
+          out += escaped;
+        }
         chunkStart = this.state.pos;
       } else if (isNewLine(ch)) {
         out += this.input.slice(chunkStart, this.state.pos);
@@ -682,14 +687,20 @@ export default class Tokenizer {
 
   // Used to read escaped characters
 
-  readEscapedChar(inTemplate) {
+  readEscapedChar(inTemplate, throwOnInvalid) {
     const ch = this.input.charCodeAt(++this.state.pos);
     ++this.state.pos;
     switch (ch) {
       case 110: return "\n"; // 'n' -> '\n'
       case 114: return "\r"; // 'r' -> '\r'
-      case 120: return String.fromCharCode(this.readHexChar(2)); // 'x'
-      case 117: return codePointToString(this.readCodePoint()); // 'u'
+      case 120: { // 'x'
+        const code = this.readHexChar(2, throwOnInvalid);
+        return code === null ? null : String.fromCharCode(code);
+      }
+      case 117: { // 'u'
+        const code = this.readCodePoint(throwOnInvalid);
+        return code === null ? null : codePointToString(code);
+      }
       case 116: return "\t"; // 't' -> '\t'
       case 98: return "\b"; // 'b' -> '\b'
       case 118: return "\u000b"; // 'v' -> '\u000b'
@@ -708,13 +719,19 @@ export default class Tokenizer {
             octal = parseInt(octalStr, 8);
           }
           if (octal > 0) {
-            if (!this.state.containsOctal) {
+            if (inTemplate) {
+              if (throwOnInvalid) {
+                this.raise(this.state.pos - 2, "Octal literal in template");
+              }
+              return null;
+            } else if (this.state.strict) {
+              this.raise(this.state.pos - 2, "Octal literal in strict mode");
+            } else {
+              // These properties are only used to throw an error for an octal which occurs
+              // in a directive which occurs prior to a "use strict" directive.
               this.state.containsOctal = true;
               this.state.octalPosition = this.state.pos - 2;
             }
-            if (this.state.strict || inTemplate) {
-              this.raise(this.state.pos - 2, "Octal literal in strict mode");
-            }
           }
           this.state.pos += octalStr.length - 1;
           return String.fromCharCode(octal);
@@ -723,12 +740,12 @@ export default class Tokenizer {
     }
   }
 
-  // Used to read character escape sequences ('\x', '\u', '\U').
+  // Used to read character escape sequences ('\x', '\u').
 
-  readHexChar(len) {
+  readHexChar(len, throwOnInvalid) {
     const codePos = this.state.pos;
     const n = this.readInt(16, len);
-    if (n === null) this.raise(codePos, "Bad character escape sequence");
+    if (throwOnInvalid && n === null) this.raise(codePos, "Bad character escape sequence");
     return n;
   }
 

test/fixtures/es2015/uncategorised/290/options.json

@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:22)"
+  "throws": "Octal literal in template (1:22)"
 }

test/fixtures/es2015/uncategorised/339/options.json

@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:1)"
+  "throws": "Octal literal in template (1:1)"
 }

test/fixtures/esprima/es2015-template-literals/invalid-escape/options.json

@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:1)"
+  "throws": "Octal literal in template (1:1)"
 }