Tokens before template literals are sometimes missing

[not-an-aardvark]

Example code:

if (foo) { bar }
`template literal`;

// using espree 3.2.0
require('espree').parse('if (foo) { bar }\n`template literal`;', {ecmaVersion: 6, tokens: true}).tokens

// output:
[ Token { type: 'Keyword', value: 'if', start: 0, end: 2 },
  Token { type: 'Punctuator', value: '(', start: 3, end: 4 },
  Token { type: 'Identifier', value: 'foo', start: 4, end: 7 },
  Token { type: 'Punctuator', value: ')', start: 7, end: 8 },
  Token { type: 'Punctuator', value: '{', start: 9, end: 10 },
  Token { type: 'Identifier', value: 'bar', start: 11, end: 14 },
  { type: 'Template', value: '`template literal`' },
  Token { type: 'Punctuator', value: ';', start: 35, end: 36 } ]

Note that the closing } for the if statement is missing in the list of tokens.

This does not appear to be a problem when using acorn directly:

// using acorn 4.0.3
var tokens = [];
require('acorn').parse('if (foo) { bar }\n`template literal`;', {onToken: tokens});
tokens

// output:

[
  // ... other tokens
  Token {
    type:
     TokenType {
       label: '}',
       keyword: undefined,
       beforeExpr: false,
       startsExpr: false,
       isLoop: false,
       isAssign: false,
       prefix: false,
       postfix: false,
       binop: null,
       updateContext: [Function] },
    value: undefined,
    start: 15,
    end: 16 },
  // ... other tokens
]

[nzakas]

Yeah, that's our bug. Acorn handles templates very different from Esprima, so we do some magic to get the correct output.

[mysticatea]

I'm working on this.