Port all babel-parser changes from 2018-07-10 to 2018-09-28

In chronological order:
301db1b92 TypeScript: Support type arguments on JSX opening and self-closing tags (#7799)
✅ Required a different implementation since JSX parsing has changed so much.

b89234b1a v7.0.0-beta.53
🚫 Release only.

84282f199 v7.0.0-beta.54
🚫 Release only.

8b10a44fe remove .then from dynamic import parser exception message (#8355)
🚫 Error message only.

a811cf78e Correctly parse interface methods named 'static' (#8374)
✅ Ported the fix, though the test case wasn't breaking for me before.

8ee24fdfc TypeScript: Support type arguments on tagged templates (#7754)
✅ Ported change over with new test.

adca165ee v7.0.0-beta.55
🚫 Release only.

55ca90b3f Allow TSInterfaceDeclaration to be default export (#8408)
✅ Ported with a test and a few modifications.

5c728ea60 Fix private property parsing in Flow (#8340)
✅ Added a test, but the code paths were already consolidated.

6695f5e2f v7.0.0-beta.56
🚫 Release only.

8b1406127 v7.0.0-rc.0
🚫 Release only.

0a958861c v7.0.0-rc.1
🚫 Release only.

d79b5eeef Require decoratorsBeforeExport option for decorators (#8465)
🚫 Only affects plugin system.

ab8555a86 Update dependencies (#8364)
✅ Already working as expected in sucrase.

b439013cd Fix trailingComments for FunctionExpression that is CallExpression arguments (#8488)
🚫 Comments aren't tracked in Sucrase.

f1d774b34 v7.0.0-rc.2
🚫 Release only.

cada040be v7.0.0-rc.3
🚫 Release only.

589994015 Update parser whitespace for clarity (#8539)
✅ Ported refactor.

524d84776 Flatten TokenType class hierarchy (#8537)
🚫 Not relevant in Sucrase.

72ee1816a Update to ES6 String methods (#8541)
🚫 Relevant code was removed from Sucrase.

edbffda09 Cleanup getLineInfo (#8540)
🚫 Relevant code was removed from Sucrase.

814c564c4 v7.0.0-rc.4
🚫 Release only.

90fb82a53 v7.0.0
🚫 Release only.

04d09cc75 add access public to all packages [skip ci] (#8573)
🚫 package.json only.

9d244ae66 More helpful error message for missing decoratorsBeforeExport in parser (#8576)
🚫 Error message only.

6893b7e7d fix: [skip ci] consistent naming
🚫 Types only.

fc9becfdf Merge pull request #8170 from AviVahl/master
🚫 Merge commit.

07fae82a8 fix(types): missing `unambiguous` sourceType (#8610)
🚫 Types only.

58017044a Bump flow to 0.80 and fix sourceType error (#8630)
🚫 Types only.

79b2af599 Format fixture JSON with Prettier. (#8658)
🚫 Formatting only.

380f2a029 Fix typescript parsing typed object shorthand methods (#8677)
🚫 Already applied in a previous bug fix.

9b4b436e1 Fix parsing of newline between 'async' and 'function' (#8698)
✅ Added a test, but I didn't port the code since I think this is obscure and
will be hard to implement right in Sucrase.

fad74959f v7.1.0
🚫 Release only.

38cf318f6 Bump lint-related deps (#8765)
🚫 Lint only.

f38be1311 TypeScript: reserve `unknown` as TSUnknownKeyword (#8755)
🚫 Not relevant for Sucrase.

ead23d711 v7.1.1
🚫 Release only.

3f5b7554b v7.1.2
🚫 Release only.

script/lint.ts

@@ -39,4 +39,5 @@ async function checkIntegration(path: string): Promise<void> {
 main().catch((e) => {
   console.error("Unhandled error:");
   console.error(e);
+  process.exitCode = 1;
 });

src/parser/plugins/flow.ts

@@ -242,7 +242,7 @@ function flowParseInterfaceish(isClass: boolean = false): void {
     } while (eat(tt.comma));
   }
 
-  flowParseObjectType(true, false, isClass);
+  flowParseObjectType(isClass, false, isClass);
 }
 
 function flowParseInterfaceExtends(): void {
@@ -338,7 +338,7 @@ function flowParseInterfaceType(): void {
       flowParseInterfaceExtends();
     } while (eat(tt.comma));
   }
-  flowParseObjectType(true, false, false);
+  flowParseObjectType(false, false, false);
 }
 
 function flowParseObjectPropertyKey(): void {

src/parser/plugins/jsx/index.ts

@@ -10,11 +10,12 @@ import {
   Token,
 } from "../../tokenizer/index";
 import {TokenType as tt} from "../../tokenizer/types";
-import {input, raise, state} from "../../traverser/base";
+import {input, isTypeScriptEnabled, raise, state} from "../../traverser/base";
 import {parseExpression, parseMaybeAssign} from "../../traverser/expression";
 import {expect, unexpected} from "../../traverser/util";
 import {charCodes} from "../../util/charcodes";
 import {isIdentifierChar, isIdentifierStart} from "../../util/identifier";
+import {tsTryParseJSXTypeArgument} from "../typescript";
 
 // Reads inline JSX contents token.
 function jsxReadToken(): void {
@@ -178,6 +179,9 @@ function jsxParseOpeningElement(): boolean {
     return false;
   }
   jsxParseElementName();
+  if (isTypeScriptEnabled) {
+    tsTryParseJSXTypeArgument();
+  }
   while (!match(tt.slash) && !match(tt.jsxTagEnd)) {
     jsxParseAttribute();
   }
@@ -252,7 +256,7 @@ export function jsxParseElement(): void {
 // Overrides
 // ==================================
 
-function nextJSXTagToken(): void {
+export function nextJSXTagToken(): void {
   state.tokens.push(new Token());
   skipSpace();
   state.start = state.pos;

src/parser/plugins/typescript.ts

@@ -23,6 +23,7 @@ import {
   parseMaybeAssign,
   parseMaybeUnary,
   parsePropertyName,
+  parseTemplate,
   StopState,
 } from "../traverser/expression";
 import {parseBindingList} from "../traverser/lval";
@@ -49,6 +50,7 @@ import {
   semicolon,
   unexpected,
 } from "../traverser/util";
+import {nextJSXTagToken} from "./jsx";
 
 function assert(x: boolean): void {
   if (!x) {
@@ -673,17 +675,15 @@ export function tsParseTypeAssertion(): void {
   parseMaybeUnary();
 }
 
-// Returns true if parsing was successful.
-function tsTryParseTypeArgumentsInExpression(): boolean {
-  return tsTryParseAndCatch(() => {
-    const oldIsType = pushTypeContext(0);
-    expect(tt.lessThan);
+export function tsTryParseJSXTypeArgument(): void {
+  if (eat(tt.jsxTagStart)) {
     state.tokens[state.tokens.length - 1].type = tt.typeParameterStart;
+    const oldIsType = pushTypeContext(1);
     tsParseDelimitedList(ParsingContext.TypeParametersOrArguments, tsParseType);
-    expect(tt.greaterThan);
+    // Process >, but the one after needs to be parsed JSX-style.
+    nextJSXTagToken();
     popTypeContext(oldIsType);
-    expect(tt.parenL);
-  });
+  }
 }
 
 function tsParseHeritageClause(): void {
@@ -1093,22 +1093,32 @@ export function tsParseSubscript(
     return;
   }
 
-  if (!noCalls && match(tt.lessThan)) {
-    if (atPossibleAsync()) {
-      // Almost certainly this is a generic async function `async <T>() => ...
-      // But it might be a call with a type argument `async<T>();`
-      const asyncArrowFn = tsTryParseGenericAsyncArrowFunction();
-      if (asyncArrowFn) {
+  // There are number of things we are going to "maybe" parse, like type arguments on
+  // tagged template expressions. If any of them fail, walk it back and continue.
+  const success = tsTryParseAndCatch(() => {
+    if (match(tt.lessThan)) {
+      if (!noCalls && atPossibleAsync()) {
+        // Almost certainly this is a generic async function `async <T>() => ...
+        // But it might be a call with a type argument `async<T>();`
+        const asyncArrowFn = tsTryParseGenericAsyncArrowFunction();
+        if (asyncArrowFn) {
+          return;
+        }
+      }
+      tsParseTypeArguments();
+      if (!noCalls && eat(tt.parenL)) {
+        parseCallExpressionArguments(tt.parenR);
+        return;
+      } else if (match(tt.backQuote)) {
+        // Tagged template with a type argument.
+        parseTemplate();
         return;
       }
     }
-
-    // May be passing type arguments. But may just be the `<` operator.
-    const typeArguments = tsTryParseTypeArgumentsInExpression(); // Also eats the "("
-    if (typeArguments) {
-      // possibleAsync always false here, because we would have handled it above.
-      parseCallExpressionArguments(tt.parenR);
-    }
+    unexpected();
+  });
+  if (success) {
+    return;
   }
   baseParseSubscript(startPos, noCalls, stopState);
 }
@@ -1157,6 +1167,13 @@ export function tsTryParseExportDefaultExpression(): boolean {
     parseClass(true, true);
     return true;
   }
+  if (isContextual(ContextualKeyword._interface)) {
+    // Make sure "export default" are considered type tokens so the whole thing is removed.
+    const oldIsType = pushTypeContext(2);
+    tsParseDeclaration(ContextualKeyword._interface, true);
+    popTypeContext(oldIsType);
+    return true;
+  }
   return false;
 }
 

src/parser/tokenizer/index.ts

@@ -4,7 +4,7 @@ import {input, isFlowEnabled, raise, state} from "../traverser/base";
 import {unexpected} from "../traverser/util";
 import {charCodes} from "../util/charcodes";
 import {isIdentifierChar, isIdentifierStart} from "../util/identifier";
-import {nonASCIIwhitespace} from "../util/whitespace";
+import {isWhitespace} from "../util/whitespace";
 import readWord from "./readWord";
 import {TokenType, TokenType as tt} from "./types";
 
@@ -267,11 +267,6 @@ export function skipSpace(): void {
   while (state.pos < input.length) {
     const ch = input.charCodeAt(state.pos);
     switch (ch) {
-      case charCodes.space:
-      case charCodes.nonBreakingSpace:
-        ++state.pos;
-        break;
-
       case charCodes.carriageReturn:
         if (input.charCodeAt(state.pos + 1) === charCodes.lineFeed) {
           ++state.pos;
@@ -299,10 +294,7 @@ export function skipSpace(): void {
         break;
 
       default:
-        if (
-          (ch > charCodes.backSpace && ch < charCodes.shiftOut) ||
-          (ch >= charCodes.oghamSpaceMark && nonASCIIwhitespace.test(String.fromCharCode(ch)))
-        ) {
+        if (isWhitespace(ch)) {
           ++state.pos;
         } else {
           return;

src/parser/traverser/expression.ts

@@ -669,7 +669,7 @@ function parseNewArguments(): void {
   }
 }
 
-function parseTemplate(): void {
+export function parseTemplate(): void {
   // Finish `, read quasi
   nextTemplateToken();
   // Finish quasi, read ${

src/parser/util/whitespace.ts

@@ -1,5 +1,36 @@
 // Matches a whole line break (where CRLF is considered a single
 // line break). Used to count lines.
+import {charCodes} from "./charcodes";
+
 export const lineBreak = /\r\n?|\n|\u2028|\u2029/;
 
-export const nonASCIIwhitespace = /[\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]/;
+// https://tc39.github.io/ecma262/#sec-white-space
+export function isWhitespace(code: number): boolean {
+  switch (code) {
+    case 0x0009: // CHARACTER TABULATION
+    case 0x000b: // LINE TABULATION
+    case 0x000c: // FORM FEED
+    case charCodes.space:
+    case charCodes.nonBreakingSpace:
+    case charCodes.oghamSpaceMark:
+    case 0x2000: // EN QUAD
+    case 0x2001: // EM QUAD
+    case 0x2002: // EN SPACE
+    case 0x2003: // EM SPACE
+    case 0x2004: // THREE-PER-EM SPACE
+    case 0x2005: // FOUR-PER-EM SPACE
+    case 0x2006: // SIX-PER-EM SPACE
+    case 0x2007: // FIGURE SPACE
+    case 0x2008: // PUNCTUATION SPACE
+    case 0x2009: // THIN SPACE
+    case 0x200a: // HAIR SPACE
+    case 0x202f: // NARROW NO-BREAK SPACE
+    case 0x205f: // MEDIUM MATHEMATICAL SPACE
+    case 0x3000: // IDEOGRAPHIC SPACE
+    case 0xfeff: // ZERO WIDTH NO-BREAK SPACE
+      return true;
+
+    default:
+      return false;
+  }
+}

src/transformers/JSXTransformer.ts

@@ -150,15 +150,18 @@ export default class JSXTransformer extends Transformer {
   processTagIntro(): void {
     // Walk forward until we see one of these patterns:
     // jsxName to start the first prop, preceded by another jsxName to end the tag name.
+    // jsxName to start the first prop, preceded by greaterThan to end the type argument.
     // [open brace] to start the first prop.
     // [jsxTagEnd] to end the open-tag.
     // [slash, jsxTagEnd] to end the self-closing tag.
     let introEnd = this.tokens.currentIndex() + 1;
     while (
-      !this.tokens.matchesAtIndex(introEnd - 1, [tt.jsxName, tt.jsxName]) &&
-      !this.tokens.matchesAtIndex(introEnd, [tt.braceL]) &&
-      !this.tokens.matchesAtIndex(introEnd, [tt.jsxTagEnd]) &&
-      !this.tokens.matchesAtIndex(introEnd, [tt.slash, tt.jsxTagEnd])
+      this.tokens.tokens[introEnd].isType ||
+      (!this.tokens.matchesAtIndex(introEnd - 1, [tt.jsxName, tt.jsxName]) &&
+        !this.tokens.matchesAtIndex(introEnd - 1, [tt.greaterThan, tt.jsxName]) &&
+        !this.tokens.matchesAtIndex(introEnd, [tt.braceL]) &&
+        !this.tokens.matchesAtIndex(introEnd, [tt.jsxTagEnd]) &&
+        !this.tokens.matchesAtIndex(introEnd, [tt.slash, tt.jsxTagEnd]))
     ) {
       introEnd++;
     }

test/flow-test.ts

@@ -298,4 +298,34 @@ describe("transform flow", () => {
     `,
     );
   });
+
+  it("allows interface methods named 'static'", () => {
+    assertFlowResult(
+      `
+      type T = interface { static(): number }
+    `,
+      `"use strict";
+      
+    `,
+    );
+  });
+
+  // Note that we don't actually transform private fields at the moment, this just makes sure it
+  // parses.
+  it("allows private properties with type annotations", () => {
+    assertFlowResult(
+      `
+      class A {
+        #prop1: string;
+        #prop2: number = value;
+      }
+    `,
+      `"use strict";const __init = Symbol();
+      class A {constructor() { this[__init](); }
+        
+        [__init]() {this.prop2 = value}
+      }
+    `,
+    );
+  });
 });

test/imports-test.ts

@@ -1043,4 +1043,20 @@ module.exports = exports.default;
       {transforms: ["imports", "typescript"]},
     );
   });
+
+  // Skipping since this is reasonably obscure and seems hard to implement right in Sucrase.
+  // Babel fix: https://github.com/babel/babel/pull/8698
+  it.skip("treats 'export default async' as a complete statement", () => {
+    assertResult(
+      `
+      export default async
+      function bar() {}
+    `,
+      `"use strict";${ESMODULE_PREFIX}
+      exports. default = async
+      function bar() {}
+    `,
+      {transforms: ["imports", "typescript"]},
+    );
+  });
 });

test/typescript-test.ts

@@ -1045,4 +1045,43 @@ describe("typescript transform", () => {
     `,
     );
   });
+
+  it("allows type arguments in JSX elements", () => {
+    assertTypeScriptResult(
+      `
+      const e1 = <Foo<number> x="1" />
+      const e2 = <Foo<string>><span>Hello</span></Foo>
+    `,
+      `"use strict";const _jsxFileName = "";
+      const e1 = React.createElement(Foo, { x: "1", __self: this, __source: {fileName: _jsxFileName, lineNumber: 2}} )
+      const e2 = React.createElement(Foo, {__self: this, __source: {fileName: _jsxFileName, lineNumber: 3}}, React.createElement('span', {__self: this, __source: {fileName: _jsxFileName, lineNumber: 3}}, "Hello"))
+    `,
+    );
+  });
+
+  it("allows type arguments tagged templates", () => {
+    assertTypeScriptResult(
+      `
+      f<T>\`\`;
+      new C<T>
+      \`\`;
+    `,
+      `"use strict";
+      f\`\`;
+      new C
+      \`\`;
+    `,
+    );
+  });
+
+  it("allows export default interface", () => {
+    assertTypeScriptResult(
+      `
+      export default interface A {}
+    `,
+      `"use strict";
+      
+    `,
+    );
+  });
 });