Skip to content

Commit

Permalink
fix(parser): fixed directive prologue edge cases
Browse files Browse the repository at this point in the history
  • Loading branch information
@KFlash
KFlash committed May 31, 2019
1 parent 9f85539 commit 9092515
Show file tree
Hide file tree
Showing 26 changed files with 277 additions and 219 deletions.
2 changes: 1 addition & 1 deletion src/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ export interface ParserState {
};
assignable: AssignmentKind | DestructuringKind;
destructible: AssignmentKind | DestructuringKind;
currentCodePoint: number;
nextCP: number;
}

/**
Expand Down
26 changes: 11 additions & 15 deletions src/lexer/comments.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { nextCodePoint, CharTypes, CharFlags, ScannerState, Seek } from './';
import { Chars } from '../chars';
import { Token } from '../token';
import { ParserState, Flags } from '../common';
import { report, Errors } from '../errors';

Expand All @@ -12,16 +11,16 @@ import { report, Errors } from '../errors';
export function skipHashBang(parser: ParserState): void {
let index = parser.index;
if (index === parser.end) return;
if (parser.currentCodePoint === Chars.ByteOrderMark) {
parser.currentCodePoint = parser.source.charCodeAt(++index);
if (parser.nextCP === Chars.ByteOrderMark) {
parser.nextCP = parser.source.charCodeAt(++index);
parser.index = index;
}

if (index < parser.end && parser.source.charCodeAt(index) === Chars.Hash) {
index++;
if (index < parser.end && parser.source.charCodeAt(index) === Chars.Exclamation) {
parser.index = index + 1;
parser.currentCodePoint = parser.source.charCodeAt(parser.index);
parser.nextCP = parser.source.charCodeAt(parser.index);
skipSingleLineComment(parser, ScannerState.None);
} else {
report(parser, Errors.IllegalCaracter, '#');
Expand All @@ -36,14 +35,11 @@ export function skipHashBang(parser: ParserState): void {
*/
export function skipSingleLineComment(parser: ParserState, state: ScannerState): ScannerState {
while (parser.index < parser.end) {
if (
CharTypes[parser.currentCodePoint] & CharFlags.LineTerminator ||
(parser.currentCodePoint ^ Chars.LineSeparator) <= 1
) {
if (CharTypes[parser.nextCP] & CharFlags.LineTerminator || (parser.nextCP ^ Chars.LineSeparator) <= 1) {
parser.flags |= Flags.NewLine;
parser.column = 0;
parser.line++;
parser.currentCodePoint = parser.source.charCodeAt(++parser.index);
parser.nextCP = parser.source.charCodeAt(++parser.index);
return state;
}
nextCodePoint(parser);
Expand All @@ -58,16 +54,16 @@ export function skipSingleLineComment(parser: ParserState, state: ScannerState):
*/
export function skipMultiLineComment(parser: ParserState, state: ScannerState): any {
while (parser.index < parser.end) {
while (CharTypes[parser.currentCodePoint] & CharFlags.Asterisk) {
while (CharTypes[parser.nextCP] & CharFlags.Asterisk) {
if (nextCodePoint(parser) === Chars.Slash) {
nextCodePoint(parser);
return state;
}
}

// ES 2020 11.3 Line Terminators
if (CharTypes[parser.currentCodePoint] & CharFlags.LineTerminator) {
if (CharTypes[parser.currentCodePoint] & CharFlags.CarriageReturn) {
if (CharTypes[parser.nextCP] & CharFlags.LineTerminator) {
if (CharTypes[parser.nextCP] & CharFlags.CarriageReturn) {
state |= ScannerState.NewLine | ScannerState.LastIsCR;
parser.column = 0;
parser.line++;
Expand All @@ -78,12 +74,12 @@ export function skipMultiLineComment(parser: ParserState, state: ScannerState):
}
state = (state & ~ScannerState.LastIsCR) | ScannerState.NewLine;
}
parser.currentCodePoint = parser.source.charCodeAt(++parser.index);
parser.nextCP = parser.source.charCodeAt(++parser.index);
parser.flags |= Flags.NewLine;
} else if ((parser.currentCodePoint ^ Chars.LineSeparator) <= 1) {
} else if ((parser.nextCP ^ Chars.LineSeparator) <= 1) {
state = (state & ~ScannerState.LastIsCR) | ScannerState.NewLine;
parser.column = 0;
parser.currentCodePoint = parser.source.charCodeAt(++parser.index);
parser.nextCP = parser.source.charCodeAt(++parser.index);
parser.line++;
parser.flags |= Flags.NewLine;
} else {
Expand Down
4 changes: 2 additions & 2 deletions src/lexer/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export const enum ScannerState {
*/
export function nextCodePoint(parser: ParserState): number {
parser.column++;
return (parser.currentCodePoint = parser.source.charCodeAt(++parser.index));
return (parser.nextCP = parser.source.charCodeAt(++parser.index));
}

export function consumeMultiUnitCodePoint(parser: ParserState, hi: number): boolean {
Expand All @@ -35,7 +35,7 @@ export function consumeMultiUnitCodePoint(parser: ParserState, hi: number): bool
}
parser.index++;
parser.column++;
parser.currentCodePoint = hi;
parser.nextCP = hi;
return true;
}

Expand Down
37 changes: 17 additions & 20 deletions src/lexer/identifier.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ import { unicodeLookup } from '../unicode';

export function scanIdentifier(parser: ParserState, context: Context): Token {
let hasEscape: 0 | 1 = 0;
let canBeKeyword: number = CharTypes[parser.currentCodePoint] & CharFlags.KeywordCandidate;
let canBeKeyword: number = CharTypes[parser.nextCP] & CharFlags.KeywordCandidate;
parser.tokenValue = '';
if (parser.currentCodePoint <= 0x7e) {
if ((CharTypes[parser.currentCodePoint] & CharFlags.BackSlash) === 0) {
if (parser.nextCP <= 0x7e) {
if ((CharTypes[parser.nextCP] & CharFlags.BackSlash) === 0) {
while ((CharTypes[nextCodePoint(parser)] & CharFlags.IdentifierPart) !== 0) {}
parser.tokenValue = parser.source.slice(parser.tokenIndex, parser.index);
if (parser.currentCodePoint > 0x7e) return scanIdentifierSlowCase(parser, context, hasEscape, canBeKeyword);
if (parser.nextCP > 0x7e) return scanIdentifierSlowCase(parser, context, hasEscape, canBeKeyword);

if ((CharTypes[parser.currentCodePoint] & CharFlags.BackSlash) === 0) {
if ((CharTypes[parser.nextCP] & CharFlags.BackSlash) === 0) {
return descKeywordTable[parser.tokenValue] || Token.Identifier;
}
} else {
Expand All @@ -39,18 +39,15 @@ export function scanIdentifierSlowCase(
): Token {
let start = parser.index;
while (parser.index < parser.end) {
if (CharTypes[parser.currentCodePoint] & CharFlags.BackSlash) {
if (CharTypes[parser.nextCP] & CharFlags.BackSlash) {
parser.tokenValue += parser.source.slice(start, parser.index);
hasEscape = 1;
const code = scanIdentifierUnicodeEscape(parser);
if (!isIdentifierPart(code)) report(parser, Errors.InvalidUnicodeEscapeSequence);
canBeKeyword = canBeKeyword && CharTypes[code] & CharFlags.KeywordCandidate;
parser.tokenValue += fromCodePoint(code);
start = parser.index;
} else if (
isIdentifierPart(parser.currentCodePoint) ||
consumeMultiUnitCodePoint(parser, parser.currentCodePoint)
) {
} else if (isIdentifierPart(parser.nextCP) || consumeMultiUnitCodePoint(parser, parser.nextCP)) {
nextCodePoint(parser);
} else {
break;
Expand Down Expand Up @@ -84,9 +81,9 @@ export function scanIdentifierSlowCase(
export function scanPrivateName(parser: ParserState): Token {
nextCodePoint(parser); // consumes '#'
if (
(CharTypes[parser.currentCodePoint] & CharFlags.Decimal) !== 0 ||
((CharTypes[parser.currentCodePoint] & CharFlags.IdentifierStart) === 0 &&
((unicodeLookup[(parser.currentCodePoint >>> 5) + 0] >>> parser.currentCodePoint) & 31 & 1) === 0)
(CharTypes[parser.nextCP] & CharFlags.Decimal) !== 0 ||
((CharTypes[parser.nextCP] & CharFlags.IdentifierStart) === 0 &&
((unicodeLookup[(parser.nextCP >>> 5) + 0] >>> parser.nextCP) & 31 & 1) === 0)
) {
report(parser, Errors.MissingPrivateName);
}
Expand All @@ -98,7 +95,7 @@ export function scanIdentifierUnicodeEscape(parser: ParserState): any {
// Check for Unicode escape of the form '\uXXXX'
// and return code point value if valid Unicode escape is found. Otherwise return -1.
if (parser.index + 5 < parser.end && parser.source.charCodeAt(parser.index + 1) === Chars.LowerU) {
parser.currentCodePoint = parser.source.charCodeAt((parser.index += 2));
parser.nextCP = parser.source.charCodeAt((parser.index += 2));
return scanUnicodeEscapeValue(parser);
}
report(parser, Errors.InvalidUnicodeEscapeSequence);
Expand All @@ -107,9 +104,9 @@ export function scanIdentifierUnicodeEscape(parser: ParserState): any {
export function scanUnicodeEscapeValue(parser: ParserState): number {
let codePoint = 0;
// First handle a delimited Unicode escape, e.g. \u{1F4A9}
if (parser.currentCodePoint === Chars.LeftBrace) {
if (parser.nextCP === Chars.LeftBrace) {
while (CharTypes[nextCodePoint(parser)] & CharFlags.Hex) {
codePoint = (codePoint << 4) | toHex(parser.currentCodePoint);
codePoint = (codePoint << 4) | toHex(parser.nextCP);
// Check this early to avoid `code` wrapping to a negative on overflow (which is
// reserved for abnormal conditions).
if (codePoint > Chars.NonBMPMax) {
Expand All @@ -118,14 +115,14 @@ export function scanUnicodeEscapeValue(parser: ParserState): number {
}

// At least 4 characters have to be read
if (codePoint < 1 || (parser.currentCodePoint as number) !== Chars.RightBrace) {
if (codePoint < 1 || (parser.nextCP as number) !== Chars.RightBrace) {
report(parser, Errors.InvalidHexEscapeSequence);
}
nextCodePoint(parser); // consumes '}'
return codePoint;
}

if ((CharTypes[parser.currentCodePoint] & CharFlags.Hex) === 0) report(parser, Errors.InvalidHexEscapeSequence); // first one is mandatory
if ((CharTypes[parser.nextCP] & CharFlags.Hex) === 0) report(parser, Errors.InvalidHexEscapeSequence); // first one is mandatory

const c2 = parser.source.charCodeAt(parser.index + 1);
if ((CharTypes[c2] & CharFlags.Hex) === 0) report(parser, Errors.Unexpected);
Expand All @@ -134,9 +131,9 @@ export function scanUnicodeEscapeValue(parser: ParserState): number {
const c4 = parser.source.charCodeAt(parser.index + 3);
if ((CharTypes[c4] & CharFlags.Hex) === 0) report(parser, Errors.Unexpected);

codePoint = (toHex(parser.currentCodePoint) << 12) | (toHex(c2) << 8) | (toHex(c3) << 4) | toHex(c4);
codePoint = (toHex(parser.nextCP) << 12) | (toHex(c2) << 8) | (toHex(c3) << 4) | toHex(c4);

parser.currentCodePoint = parser.source.charCodeAt((parser.index += 4));
parser.nextCP = parser.source.charCodeAt((parser.index += 4));

return codePoint;
}
46 changes: 23 additions & 23 deletions src/lexer/numeric.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,53 +18,53 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: boole
let value: number | string = 0;
let atStart = !isFloat;
if (isFloat) {
while (CharTypes[parser.currentCodePoint] & CharFlags.Decimal) {
while (CharTypes[parser.nextCP] & CharFlags.Decimal) {
nextCodePoint(parser);
}
} else {
if (parser.currentCodePoint === Chars.Zero) {
if (parser.nextCP === Chars.Zero) {
nextCodePoint(parser);

// Hex
if ((parser.currentCodePoint | 32) === Chars.LowerX) {
if ((parser.nextCP | 32) === Chars.LowerX) {
kind = NumberKind.Hex;
let digits = 0;
while (CharTypes[nextCodePoint(parser)] & CharFlags.Hex) {
value = value * 0x10 + toHex(parser.currentCodePoint);
value = value * 0x10 + toHex(parser.nextCP);
digits++;
}
if (digits < 1) report(parser, Errors.MissingHexDigits);
// Octal
} else if ((parser.currentCodePoint | 32) === Chars.LowerO) {
} else if ((parser.nextCP | 32) === Chars.LowerO) {
kind = NumberKind.Octal;
let digits = 0;
while (CharTypes[nextCodePoint(parser)] & CharFlags.Octal) {
value = value * 8 + (parser.currentCodePoint - Chars.Zero);
value = value * 8 + (parser.nextCP - Chars.Zero);
digits++;
}
if (digits < 1) report(parser, Errors.ExpectedNumberInRadix, `${8}`);
} else if ((parser.currentCodePoint | 32) === Chars.LowerB) {
} else if ((parser.nextCP | 32) === Chars.LowerB) {
kind = NumberKind.Binary;
let digits = 0;
while (CharTypes[nextCodePoint(parser)] & CharFlags.Binary) {
value = value * 2 + (parser.currentCodePoint - Chars.Zero);
value = value * 2 + (parser.nextCP - Chars.Zero);
digits++;
}
if (digits < 1) report(parser, Errors.ExpectedNumberInRadix, `${2}`);
} else if (CharTypes[parser.currentCodePoint] & CharFlags.Octal) {
} else if (CharTypes[parser.nextCP] & CharFlags.Octal) {
// Octal integer literals are not permitted in strict mode code
if (context & Context.Strict) report(parser, Errors.StrictOctalEscape);
else parser.flags = Flags.Octals;
kind = NumberKind.ImplicitOctal;
do {
if (CharTypes[parser.currentCodePoint] & CharFlags.ImplicitOctalDigits) {
if (CharTypes[parser.nextCP] & CharFlags.ImplicitOctalDigits) {
kind = NumberKind.DecimalWithLeadingZero;
atStart = false;
break;
}
value = value * 8 + (parser.currentCodePoint - Chars.Zero);
value = value * 8 + (parser.nextCP - Chars.Zero);
} while (CharTypes[nextCodePoint(parser)] & CharFlags.Decimal);
} else if (CharTypes[parser.currentCodePoint] & CharFlags.ImplicitOctalDigits) {
} else if (CharTypes[parser.nextCP] & CharFlags.ImplicitOctalDigits) {
if (context & Context.Strict) report(parser, Errors.StrictOctalEscape);
else parser.flags = Flags.Octals;
kind = NumberKind.DecimalWithLeadingZero;
Expand All @@ -76,28 +76,28 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: boole
if (atStart) {
// scan subsequent decimal digits
let digit = 9;
while (digit >= 0 && CharTypes[parser.currentCodePoint] & CharFlags.Decimal) {
value = 10 * value + (parser.currentCodePoint - Chars.Zero);
while (digit >= 0 && CharTypes[parser.nextCP] & CharFlags.Decimal) {
value = 10 * value + (parser.nextCP - Chars.Zero);
nextCodePoint(parser);
--digit;
}

if (digit >= 0 && parser.currentCodePoint !== Chars.Period && !isIdentifierStart(parser.currentCodePoint)) {
if (digit >= 0 && parser.nextCP !== Chars.Period && !isIdentifierStart(parser.nextCP)) {
if (context & Context.OptionsRaw) parser.tokenRaw = parser.source.slice(parser.tokenIndex, parser.index);
parser.tokenValue = value;
return Token.NumericLiteral;
}
}

while (CharTypes[parser.currentCodePoint] & CharFlags.Decimal) {
while (CharTypes[parser.nextCP] & CharFlags.Decimal) {
nextCodePoint(parser);
}

// Scan any decimal dot and fractional component
if (parser.currentCodePoint === Chars.Period) {
if (parser.nextCP === Chars.Period) {
isFloat = true;
nextCodePoint(parser); // consumes '.'
while (CharTypes[parser.currentCodePoint] & CharFlags.Decimal) {
while (CharTypes[parser.nextCP] & CharFlags.Decimal) {
nextCodePoint(parser);
}
}
Expand All @@ -106,28 +106,28 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: boole

let isBigInt = false;
if (
parser.currentCodePoint === Chars.LowerN &&
parser.nextCP === Chars.LowerN &&
(kind & (NumberKind.Decimal | NumberKind.Binary | NumberKind.Octal | NumberKind.Hex)) !== 0
) {
if (isFloat) report(parser, Errors.InvalidBigInt);
isBigInt = true;
nextCodePoint(parser);
// Scan any exponential notation
} else if ((parser.currentCodePoint | 32) === Chars.LowerE) {
} else if ((parser.nextCP | 32) === Chars.LowerE) {
if ((kind & (NumberKind.Decimal | NumberKind.DecimalWithLeadingZero)) === 0) {
report(parser, Errors.MissingExponent);
}

nextCodePoint(parser);

// '-', '+'
if (CharTypes[parser.currentCodePoint] & CharFlags.Exponent) {
if (CharTypes[parser.nextCP] & CharFlags.Exponent) {
nextCodePoint(parser);
}

let exponentDigits = 0;
// Consume exponential digits
while (CharTypes[parser.currentCodePoint] & CharFlags.Decimal) {
while (CharTypes[parser.nextCP] & CharFlags.Decimal) {
nextCodePoint(parser);
exponentDigits++;
}
Expand All @@ -139,7 +139,7 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: boole

// The source character immediately following a numeric literal must
// not be an identifier start or a decimal digit
if (CharTypes[parser.currentCodePoint] & CharFlags.Decimal || isIdentifierStart(parser.currentCodePoint)) {
if (CharTypes[parser.nextCP] & CharFlags.Decimal || isIdentifierStart(parser.nextCP)) {
report(parser, Errors.IDStartAfterNumber);
}
if (context & Context.OptionsRaw) parser.tokenRaw = parser.source.slice(parser.tokenIndex, parser.index);
Expand Down
6 changes: 3 additions & 3 deletions src/lexer/regexp.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export function scanRegularExpression(parser: ParserState, context: Context): To
let preparseState = RegexState.Empty;

loop: while (true) {
const ch = parser.currentCodePoint;
const ch = parser.nextCP;
nextCodePoint(parser);

if (preparseState & RegexState.Escape) {
Expand Down Expand Up @@ -72,7 +72,7 @@ export function scanRegularExpression(parser: ParserState, context: Context): To
const { index: flagStart } = parser;

loop: while (parser.index < parser.source.length) {
switch (parser.currentCodePoint) {
switch (parser.nextCP) {
case Chars.LowerG:
if (mask & RegexFlags.Global) report(parser, Errors.DuplicateRegExpFlag, 'g');
mask |= RegexFlags.Global;
Expand Down Expand Up @@ -104,7 +104,7 @@ export function scanRegularExpression(parser: ParserState, context: Context): To
break;

default:
if (!isIdentifierPart(parser.currentCodePoint)) break loop;
if (!isIdentifierPart(parser.nextCP)) break loop;
report(parser, Errors.UnexpectedTokenRegExpFlag);
}

Expand Down

0 comments on commit 9092515

Please sign in to comment.