Skip to content

Commit

Permalink
fix: Maintain onNewLine state on subsequent lookahead (#2613)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dcodeIO
dcodeIO committed Jan 12, 2023
1 parent 9971708 commit fc52230
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 38 deletions.
40 changes: 20 additions & 20 deletions src/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,12 +290,12 @@ export class Parser extends DiagnosticEmitter {
tn.next();
let abstractStart = tn.tokenPos;
let abstractEnd = tn.pos;
let next = tn.peek(true);
if (tn.nextTokenOnNewLine) {
if (tn.peekOnNewLine()) {
tn.reset(state);
statement = this.parseStatement(tn, true);
break;
}
let next = tn.peek();
if (next != Token.Class) {
if (next == Token.Interface) {
this.error(
Expand All @@ -322,7 +322,7 @@ export class Parser extends DiagnosticEmitter {
case Token.Namespace: {
let state = tn.mark();
tn.next();
if (tn.peek(false, IdentifierHandling.Prefer) == Token.Identifier) {
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) {
tn.discard(state);
statement = this.parseNamespace(tn, flags, decorators, startPos);
decorators = null;
Expand All @@ -345,7 +345,7 @@ export class Parser extends DiagnosticEmitter {
case Token.Type: { // also identifier
let state = tn.mark();
tn.next();
if (tn.peek(false, IdentifierHandling.Prefer) == Token.Identifier) {
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) {
tn.discard(state);
statement = this.parseTypeDeclaration(tn, flags, decorators, startPos);
decorators = null;
Expand All @@ -358,7 +358,7 @@ export class Parser extends DiagnosticEmitter {
case Token.Module: { // also identifier
let state = tn.mark();
tn.next();
if (tn.peek(true) == Token.StringLiteral && !tn.nextTokenOnNewLine) {
if (tn.peek() == Token.StringLiteral && !tn.peekOnNewLine()) {
tn.discard(state);
statement = this.parseModuleDeclaration(tn, flags);
} else {
Expand Down Expand Up @@ -1113,10 +1113,11 @@ export class Parser extends DiagnosticEmitter {

let startPos = tn.tokenPos;
let expr: Expression | null = null;
let nextToken = tn.peek();
if (
tn.peek(true) != Token.Semicolon &&
tn.nextToken != Token.CloseBrace &&
!tn.nextTokenOnNewLine
nextToken != Token.Semicolon &&
nextToken != Token.CloseBrace &&
!tn.peekOnNewLine()
) {
if (!(expr = this.parseExpression(tn))) return null;
}
Expand Down Expand Up @@ -2042,7 +2043,7 @@ export class Parser extends DiagnosticEmitter {
let setEnd = 0;
if (!isInterface) {
if (tn.skip(Token.Get)) {
if (tn.peek(true, IdentifierHandling.Prefer) == Token.Identifier && !tn.nextTokenOnNewLine) {
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier && !tn.peekOnNewLine()) {
flags |= CommonFlags.Get;
isGetter = true;
getStart = tn.tokenPos;
Expand All @@ -2058,7 +2059,7 @@ export class Parser extends DiagnosticEmitter {
tn.reset(state);
}
} else if (tn.skip(Token.Set)) {
if (tn.peek(true, IdentifierHandling.Prefer) == Token.Identifier && !tn.nextTokenOnNewLine) {
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier && !tn.peekOnNewLine()) {
flags |= CommonFlags.Set;
isSetter = true;
setStart = tn.tokenPos;
Expand Down Expand Up @@ -2966,7 +2967,7 @@ export class Parser extends DiagnosticEmitter {
break;
}
case Token.Type: { // also identifier
if (tn.peek(false, IdentifierHandling.Prefer) == Token.Identifier) {
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) {
statement = this.parseTypeDeclaration(tn, CommonFlags.None, null, tn.tokenPos);
break;
}
Expand Down Expand Up @@ -3020,7 +3021,7 @@ export class Parser extends DiagnosticEmitter {
// at 'break': Identifier? ';'?

let identifier: IdentifierExpression | null = null;
if (tn.peek(true) == Token.Identifier && !tn.nextTokenOnNewLine) {
if (tn.peek() == Token.Identifier && !tn.peekOnNewLine()) {
tn.next(IdentifierHandling.Prefer);
identifier = Node.createIdentifierExpression(tn.readIdentifier(), tn.range());
}
Expand All @@ -3036,7 +3037,7 @@ export class Parser extends DiagnosticEmitter {
// at 'continue': Identifier? ';'?

let identifier: IdentifierExpression | null = null;
if (tn.peek(true) == Token.Identifier && !tn.nextTokenOnNewLine) {
if (tn.peek() == Token.Identifier && !tn.peekOnNewLine()) {
tn.next(IdentifierHandling.Prefer);
identifier = Node.createIdentifierExpression(tn.readIdentifier(), tn.range());
}
Expand Down Expand Up @@ -3948,7 +3949,7 @@ export class Parser extends DiagnosticEmitter {
if (tn.skip(Token.TemplateLiteral)) {
return this.parseTemplateLiteral(tn, identifier);
}
if (tn.peek(true) == Token.Equals_GreaterThan && !tn.nextTokenOnNewLine) {
if (tn.peek() == Token.Equals_GreaterThan && !tn.peekOnNewLine()) {
return this.parseFunctionExpressionCommon(
tn,
Node.createEmptyIdentifierExpression(tn.range(startPos)),
Expand Down Expand Up @@ -4405,8 +4406,8 @@ export class Parser extends DiagnosticEmitter {
tn: Tokenizer
): void {
// see: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion
let token = tn.peek(true);
if (tn.nextTokenOnNewLine || token == Token.EndOfFile || token == Token.CloseBrace) return;
let nextToken = tn.peek();
if (nextToken == Token.EndOfFile || nextToken == Token.CloseBrace || tn.peekOnNewLine()) return;
this.error(
DiagnosticCode.Unexpected_token,
tn.range(tn.nextTokenPos)
Expand All @@ -4415,18 +4416,17 @@ export class Parser extends DiagnosticEmitter {

/** Skips over a statement on errors in an attempt to reduce unnecessary diagnostic noise. */
skipStatement(tn: Tokenizer): void {
tn.peek(true);
if (tn.nextTokenOnNewLine) tn.next(); // if reset() to the previous line
if (tn.peekOnNewLine()) tn.next(); // if reset() to the previous line
do {
let nextToken = tn.peek(true);
let nextToken = tn.peek();
if (
nextToken == Token.EndOfFile || // next step should handle this
nextToken == Token.Semicolon // end of the statement for sure
) {
tn.next();
break;
}
if (tn.nextTokenOnNewLine) break; // end of the statement maybe
if (tn.peekOnNewLine()) break; // end of the statement maybe
switch (tn.next()) {
case Token.Identifier: {
tn.readIdentifier();
Expand Down
54 changes: 37 additions & 17 deletions src/tokenizer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,13 @@ export function operatorTokenToString(token: Token): string {
/** Handler for intercepting comments while tokenizing. */
export type CommentHandler = (kind: CommentKind, text: string, range: Range) => void;

/** Whether a token begins on a new line, if known. */
enum OnNewLine {
No,
Yes,
Unknown
}

/** Tokenizes a source to individual {@link Token}s. */
export class Tokenizer extends DiagnosticEmitter {

Expand All @@ -461,7 +468,7 @@ export class Tokenizer extends DiagnosticEmitter {

nextToken: Token = -1;
nextTokenPos: i32 = 0;
nextTokenOnNewLine: bool = false;
nextTokenOnNewLine: OnNewLine = OnNewLine.Unknown;

onComment: CommentHandler | null = null;

Expand Down Expand Up @@ -504,7 +511,7 @@ export class Tokenizer extends DiagnosticEmitter {
}

next(identifierHandling: IdentifierHandling = IdentifierHandling.Default): Token {
this.nextToken = -1;
this.clearNextToken();
let token: Token;
do token = this.unsafeNext(identifierHandling);
while (token == Token.Invalid);
Expand Down Expand Up @@ -959,34 +966,41 @@ export class Tokenizer extends DiagnosticEmitter {
}

peek(
checkOnNewLine: bool = false,
identifierHandling: IdentifierHandling = IdentifierHandling.Default,
maxCompoundLength: i32 = i32.MAX_VALUE
): Token {
let text = this.source.text;
if (this.nextToken < 0) {
let nextToken = this.nextToken;
if (nextToken < 0) {
let posBefore = this.pos;
let tokenBefore = this.token;
let tokenPosBefore = this.tokenPos;
let nextToken: Token;
do nextToken = this.unsafeNext(identifierHandling, maxCompoundLength);
while (nextToken == Token.Invalid);
this.nextToken = nextToken;
this.nextTokenPos = this.tokenPos;
if (checkOnNewLine) {
this.nextTokenOnNewLine = false;
for (let pos = posBefore, end = this.nextTokenPos; pos < end; ++pos) {
if (isLineBreak(text.charCodeAt(pos))) {
this.nextTokenOnNewLine = true;
break;
}
}
}
this.nextTokenOnNewLine = OnNewLine.Unknown;
this.pos = posBefore;
this.token = tokenBefore;
this.tokenPos = tokenPosBefore;
}
return this.nextToken;
return nextToken;
}

peekOnNewLine(): bool {
switch (this.nextTokenOnNewLine) {
case OnNewLine.No: return false;
case OnNewLine.Yes: return true;
}
this.peek();
let text = this.source.text;
for (let pos = this.pos, end = this.nextTokenPos; pos < end; ++pos) {
if (isLineBreak(text.charCodeAt(pos))) {
this.nextTokenOnNewLine = OnNewLine.Yes;
return true;
}
}
this.nextTokenOnNewLine = OnNewLine.No;
return false;
}

skipIdentifier(identifierHandling: IdentifierHandling = IdentifierHandling.Prefer): bool {
Expand All @@ -1006,7 +1020,7 @@ export class Tokenizer extends DiagnosticEmitter {
while (nextToken == Token.Invalid);
if (nextToken == token) {
this.token = token;
this.nextToken = -1;
this.clearNextToken();
return true;
} else {
this.pos = posBefore;
Expand Down Expand Up @@ -1037,7 +1051,13 @@ export class Tokenizer extends DiagnosticEmitter {
this.pos = state.pos;
this.token = state.token;
this.tokenPos = state.tokenPos;
this.clearNextToken();
}

clearNextToken(): void {
this.nextToken = -1;
this.nextTokenPos = 0;
this.nextTokenOnNewLine = OnNewLine.Unknown;
}

range(start: i32 = -1, end: i32 = -1): Range {
Expand Down
8 changes: 7 additions & 1 deletion tests/parser/asi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,10 @@ function successCloseBrace(): i32 {

function successCloseParen(): i32 {
return ( 123 )
}
}

function successAfterLet(): i32 {
// multiple tn.peeks
let a = 0
return a
}
4 changes: 4 additions & 0 deletions tests/parser/asi.ts.fixture.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ function successCloseBrace(): i32 {
function successCloseParen(): i32 {
return (123);
}
function successAfterLet(): i32 {
let a = 0;
return a;
}
// ERROR 1012: "Unexpected token." in asi.ts(2,13+0)
// ERROR 1012: "Unexpected token." in asi.ts(7,14+0)
// ERROR 1012: "Unexpected token." in asi.ts(11,13+0)

0 comments on commit fc52230

Please sign in to comment.