From 24b58e3859fba2d8880f55179a8a8521c518dd57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 23 Mar 2023 13:41:15 -0400 Subject: [PATCH 1/2] refactor: introduce lookaheadInLineCharCode --- .../babel-parser/src/parser/expression.ts | 3 +- packages/babel-parser/src/parser/statement.ts | 17 +++++------ packages/babel-parser/src/tokenizer/index.ts | 29 +++++++++++++++++++ packages/babel-parser/src/util/whitespace.ts | 2 +- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/packages/babel-parser/src/parser/expression.ts b/packages/babel-parser/src/parser/expression.ts index d9f8a8d66e03..59bf43e28ebc 100644 --- a/packages/babel-parser/src/parser/expression.ts +++ b/packages/babel-parser/src/parser/expression.ts @@ -1287,8 +1287,7 @@ export default abstract class ExpressionParser extends LValParser { if (tokenIsIdentifier(type)) { if ( this.isContextual(tt._module) && - this.lookaheadCharCode() === charCodes.leftCurlyBrace && - !this.hasFollowingLineBreak() + this.lookaheadInLineCharCode() === charCodes.leftCurlyBrace ) { return this.parseModuleExpression(); } diff --git a/packages/babel-parser/src/parser/statement.ts b/packages/babel-parser/src/parser/statement.ts index 7fab4fa1d5fa..d7b9b9c6023c 100644 --- a/packages/babel-parser/src/parser/statement.ts +++ b/packages/babel-parser/src/parser/statement.ts @@ -329,10 +329,11 @@ export default abstract class StatementParser extends ExpressionParser { /** * Assuming we have seen a contextual `using` and declaration is allowed, check if it - * starts a variable declaration so that it should be interpreted as a keyword. + * starts a variable declaration in the same line so that it should be interpreted as + * a keyword. */ - hasFollowingBindingIdentifier(): boolean { - const next = this.nextTokenStart(); + hasInLineFollowingBindingIdentifier(): boolean { + const next = this.nextTokenInLineStart(); const nextCh = this.codePointAtPos(next); return this.chStartsBindingIdentifier(nextCh, next); } @@ -485,9 +486,8 @@ export default abstract class StatementParser extends ExpressionParser { case tt._using: // using [no LineTerminator here][lookahead != `await`] BindingList[+Using] if ( - this.hasFollowingLineBreak() || this.state.containsEsc || - !this.hasFollowingBindingIdentifier() + !this.hasInLineFollowingBindingIdentifier() ) { break; } @@ -914,12 +914,11 @@ export default abstract class StatementParser extends ExpressionParser { const startsWithLet = this.isContextual(tt._let); const startsWithUsing = - this.isContextual(tt._using) && !this.hasFollowingLineBreak(); + this.isContextual(tt._using) && + this.hasInLineFollowingBindingIdentifier(); const isLetOrUsing = (startsWithLet && this.hasFollowingBindingAtom()) || - (startsWithUsing && - this.hasFollowingBindingIdentifier() && - this.startsUsingForOf()); + (startsWithUsing && this.startsUsingForOf()); if (this.match(tt._var) || this.match(tt._const) || isLetOrUsing) { const initNode = this.startNode(); const kind = this.state.value; diff --git a/packages/babel-parser/src/tokenizer/index.ts b/packages/babel-parser/src/tokenizer/index.ts index 4e5c9154c531..a5653239338f 100644 --- a/packages/babel-parser/src/tokenizer/index.ts +++ b/packages/babel-parser/src/tokenizer/index.ts @@ -29,6 +29,7 @@ import { isNewLine, isWhitespace, skipWhiteSpace, + skipWhiteSpaceInLine, } from "../util/whitespace"; import State from "./state"; import type { LookaheadState, DeferredStrictError } from "./state"; @@ -195,6 +196,34 @@ export default abstract class Tokenizer extends CommentsParser { return this.input.charCodeAt(this.nextTokenStart()); } + /** + * Similar to nextToken, but it will stop at line break when it is seen before the next token + * + * @returns {number} position of the next token start or line break, whichever is seen first. + * @memberof Tokenizer + */ + nextTokenInLineStart(): number { + return this.nextTokenInLineStartSince(this.state.pos); + } + + nextTokenInLineStartSince(pos: number): number { + skipWhiteSpaceInLine.lastIndex = pos; + return skipWhiteSpaceInLine.test(this.input) + ? skipWhiteSpaceInLine.lastIndex + : pos; + } + + /** + * Similar to lookaheadCharCode, but it will return the char code of line break if it is + * seen before the next token + * + * @returns {number} char code of the next token start or line break, whichever is seen first. + * @memberof Tokenizer + */ + lookaheadInLineCharCode(): number { + return this.input.charCodeAt(this.nextTokenInLineStart()); + } + codePointAtPos(pos: number): number { // The implementation is based on // https://source.chromium.org/chromium/chromium/src/+/master:v8/src/builtins/builtins-string-gen.cc;l=1455;drc=221e331b49dfefadbc6fa40b0c68e6f97606d0b3;bpv=0;bpt=1 diff --git a/packages/babel-parser/src/util/whitespace.ts b/packages/babel-parser/src/util/whitespace.ts index 095fa9322899..d0c4e2de6240 100644 --- a/packages/babel-parser/src/util/whitespace.ts +++ b/packages/babel-parser/src/util/whitespace.ts @@ -22,7 +22,7 @@ export function isNewLine(code: number): boolean { export const skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g; export const skipWhiteSpaceInLine = - /(?:[^\S\n\r\u2028\u2029]|\/\/.*|\/\*.*?\*\/)*/y; + /(?:[^\S\n\r\u2028\u2029]|\/\/.*|\/\*.*?\*\/)*/g; // Skip whitespace and single-line comments, including /* no newline here */. // After this RegExp matches, its lastIndex points to a line terminator, or From 0459e64378e58561a9acf8fb3aa7bb3456fcdbe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 23 Mar 2023 15:46:55 -0400 Subject: [PATCH 2/2] address review comments --- packages/babel-parser/src/parser/statement.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/babel-parser/src/parser/statement.ts b/packages/babel-parser/src/parser/statement.ts index d7b9b9c6023c..891d6bbe7312 100644 --- a/packages/babel-parser/src/parser/statement.ts +++ b/packages/babel-parser/src/parser/statement.ts @@ -339,11 +339,11 @@ export default abstract class StatementParser extends ExpressionParser { } startsUsingForOf(): boolean { - const lookahead = this.lookahead(); - if (lookahead.type === tt._of && !lookahead.containsEsc) { + const { type, containsEsc } = this.lookahead(); + if (type === tt._of && !containsEsc) { // `using of` must start a for-lhs-of statement return false; - } else { + } else if (tokenIsIdentifier(type) && !this.hasFollowingLineBreak()) { this.expectPlugin("explicitResourceManagement"); return true; } @@ -913,9 +913,7 @@ export default abstract class StatementParser extends ExpressionParser { } const startsWithLet = this.isContextual(tt._let); - const startsWithUsing = - this.isContextual(tt._using) && - this.hasInLineFollowingBindingIdentifier(); + const startsWithUsing = this.isContextual(tt._using); const isLetOrUsing = (startsWithLet && this.hasFollowingBindingAtom()) || (startsWithUsing && this.startsUsingForOf());