Skip to content

Commit

Permalink
feat(parser): implemented support for v8 Intrinsic
Browse files Browse the repository at this point in the history
  • Loading branch information
@KFlash
KFlash committed Jul 29, 2019
1 parent 9fd6856 commit 5e41577
Show file tree
Hide file tree
Showing 7 changed files with 380 additions and 72 deletions.
11 changes: 9 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@

* Conforms to the standard ECMAScript® 2020 (ECMA-262 10th Edition) language specification
* Support TC39 proposals via option
* Support V8 experimental features via option
* Support for additional ECMAScript features for Web Browsers
* JSX support via option
* Optionally track syntactic node locations
* Emits an ESTree-compatible abstract syntax tree.
* No backtracking
* Reduced memory usage
* Very well tested (~83 000 unit tests with full code coverage)
* Lightweight - ~82 KB minified
* Very well tested (~85 000 unit tests with full code coverage)
* Lightweight - ~86 KB minified

## ESNext features

Expand All @@ -37,6 +38,12 @@

**Note:** These features need to be enabled with the `next` option.

## V8 features

* Intrinsic

**Note:** These features need to be enabled with the `next` option.

## Installation

```sh
Expand Down
3 changes: 2 additions & 1 deletion src/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ export const enum Context {
AllowNewTarget = 1 << 26,
DisallowIn = 1 << 27,
OptionsIdentifierPattern = 1 << 28,
OptionsSpecDeviation = 1 << 29
OptionsSpecDeviation = 1 << 29,
OptionsV8 = 1 << 30,
}

/**
Expand Down
2 changes: 1 addition & 1 deletion src/lexer/template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ export function scanTemplate(parser: ParserState, context: Context): Token {
parser.tokenValue = ret;
if (tail) {
parser.tokenRaw = parser.source.slice(start + 1, parser.index - 1);
return Token.TemplateTail;
return Token.TemplateSpan;
} else {
parser.tokenRaw = parser.source.slice(start + 1, parser.index - 2);
return Token.TemplateContinuation;
Expand Down
106 changes: 73 additions & 33 deletions src/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,8 @@ export interface Options {
jsx?: boolean;
// Allow edge cases that deviate from the spec
specDeviation?: boolean;
// Enabled V8 features
v8?: boolean;
// Allowes comment extraction. Accepts either a a callback function or an array
onComment?: OnComment;
}
Expand All @@ -221,6 +223,7 @@ export function parseSource(source: string, options: Options | void, context: Co
if (options.preserveParens) context |= Context.OptionsPreserveParens;
if (options.impliedStrict) context |= Context.Strict;
if (options.jsx) context |= Context.OptionsJSX;
if (options.v8) context |= Context.OptionsV8;
if (options.identifierPattern) context |= Context.OptionsIdentifierPattern;
if (options.specDeviation) context |= Context.OptionsSpecDeviation;
if (options.source) sourceFile = options.source;
Expand Down Expand Up @@ -251,9 +254,7 @@ export function parseSource(source: string, options: Options | void, context: Co

if (scope) {
for (const key in parser.exportedBindings) {
if (key !== '#default' && !(scope as any)[key]) {
report(parser, Errors.UndeclaredExportedBinding, key.slice(1));
}
if (key[0] === '#' && !(scope as any)[key]) report(parser, Errors.UndeclaredExportedBinding, key.slice(1));
}
}
} else {
Expand Down Expand Up @@ -3460,8 +3461,10 @@ export function parseFunctionBody(
const { tokenPos, linePos, colPos } = parser;

consume(parser, context | Context.AllowRegExp, Token.LeftBrace);

const body: ESTree.Statement[] = [];
const prevContext = context;

if (parser.token !== Token.RightBrace) {
while (parser.token === Token.StringLiteral) {
const { index, tokenPos, tokenValue, token } = parser;
Expand Down Expand Up @@ -3946,7 +3949,7 @@ export function parsePrimaryExpressionExtended(
return parseNullOrTrueOrFalseLiteral(parser, context, start, line, column);
case Token.SuperKeyword:
return parseSuperExpression(parser, context, start, line, column);
case Token.TemplateTail:
case Token.TemplateSpan:
return parseTemplateLiteral(parser, context, start, line, column);
case Token.TemplateContinuation:
return parseTemplate(parser, context, start, line, column);
Expand All @@ -3959,6 +3962,8 @@ export function parsePrimaryExpressionExtended(
case Token.LessThan:
if (context & Context.OptionsJSX)
return parseJSXRootElementOrFragment(parser, context, /*inJSXChild*/ 1, start, line, column);
case Token.Modulo:
if (context & Context.OptionsV8) return parseV8Intrinsic(parser, context, start, line, column);
default:
if (
context & Context.Strict
Expand All @@ -3974,13 +3979,59 @@ export function parsePrimaryExpressionExtended(
}
}

/**
* Parses V8 intrinsic
*
* @param parser Parser object
* @param context Context masks
* @param inGroup
* @param start
* @param line
* @param column
*/
export function parseV8Intrinsic(
parser: ParserState,
context: Context,
start: number,
line: number,
column: number
): any {
// CallRuntime ::
// '%' Identifier Arguments

nextToken(parser, context); // skips: '%'

const expr = v8IntrinsicIdentifier(parser, context);

if (parser.token !== Token.LeftParen) report(parser, Errors.Unexpected);

return parseMemberOrUpdateExpression(parser, context, expr, 0, start, line, column);
}

/**
* Parses V8 intrinsic identifier
*
* @param parser Parser object
* @param context Context masks
*/
export function v8IntrinsicIdentifier(parser: ParserState, context: Context): ESTree.Identifier {
const { tokenValue, tokenPos, linePos, colPos } = parser;
nextToken(parser, context);
return finishNode(parser, context, tokenPos, linePos, colPos, {
type: 'V8IntrinsicIdentifier',
name: tokenValue
} as any);
}

/**
* Parses Import call expression
*
* @param parser Parser object
* @param context Context masks
* @param inGroup
* @param start
* @param line
* @param column
*/
function parseImportCallExpression(
parser: ParserState,
Expand Down Expand Up @@ -4128,7 +4179,6 @@ export function parseTemplateLiteral(
quasis: [parseTemplateTail(parser, context, start, line, column)]
});
}

/**
* Parses template tail
*
Expand All @@ -4145,7 +4195,7 @@ export function parseTemplateTail(
): ESTree.TemplateElement {
const { tokenValue, tokenRaw } = parser;

consume(parser, context, Token.TemplateTail);
consume(parser, context, Token.TemplateSpan);

return finishNode(parser, context, start, line, column, {
type: 'TemplateElement',
Expand All @@ -4170,31 +4220,27 @@ export function parseTemplate(
line: number,
column: number
): ESTree.TemplateLiteral {
const quasis = [parseTemplateSpans(parser, context, /* tail */ false, start, line, column)];
context = (context | Context.DisallowIn) ^ Context.DisallowIn;

const quasis = [parseTemplateSpans(parser, context, /* tail */ false)];

consume(parser, context | Context.AllowRegExp, Token.TemplateContinuation);
const expressions = [
parseExpressions(
parser,
(context | Context.DisallowIn) ^ Context.DisallowIn,
0,
1,
parser.tokenPos,
parser.linePos,
parser.colPos
)
];

const expressions = [parseExpressions(parser, context, 0, 1, parser.tokenPos, parser.linePos, parser.colPos)];

if (parser.token !== Token.RightBrace) report(parser, Errors.InvalidTemplateContinuation);
while ((parser.token = scanTemplateTail(parser, context)) !== Token.TemplateTail) {

while ((parser.token = scanTemplateTail(parser, context)) !== Token.TemplateSpan) {
const { tokenPos, linePos, colPos } = parser;
quasis.push(parseTemplateSpans(parser, context, /* tail */ false, tokenPos, linePos, colPos));
quasis.push(parseTemplateSpans(parser, context, /* tail */ false));
consume(parser, context | Context.AllowRegExp, Token.TemplateContinuation);
expressions.push(parseExpressions(parser, context, 0, 1, tokenPos, linePos, colPos));
if (parser.token !== Token.RightBrace) report(parser, Errors.InvalidTemplateContinuation);
}

quasis.push(parseTemplateSpans(parser, context, /* tail */ true, parser.tokenPos, parser.linePos, parser.colPos));
quasis.push(parseTemplateSpans(parser, context, /* tail */ true));

nextToken(parser, context);
consume(parser, context, Token.TemplateSpan);

return finishNode(parser, context, start, line, column, {
type: 'TemplateLiteral',
Expand All @@ -4209,15 +4255,9 @@ export function parseTemplate(
* @param parser Parser object
* @param tail
*/
export function parseTemplateSpans(
parser: ParserState,
context: Context,
tail: boolean,
start: number,
line: number,
column: number
): ESTree.TemplateElement {
return finishNode(parser, context, start, line, column, {
export function parseTemplateSpans(parser: ParserState, context: Context, tail: boolean): ESTree.TemplateElement {
const { tokenPos, linePos, colPos } = parser;
return finishNode(parser, context, tokenPos, linePos, colPos, {
type: 'TemplateElement',
value: {
cooked: parser.tokenValue,
Expand Down Expand Up @@ -6744,7 +6784,7 @@ export function parseArrowFunctionExpression(
switch (parser.token) {
case Token.Period:
case Token.LeftBracket:
case Token.TemplateTail:
case Token.TemplateSpan:
case Token.QuestionMark:
report(parser, Errors.InvalidAccessedBlockBodyArrow);
case Token.LeftParen:
Expand Down Expand Up @@ -7003,7 +7043,7 @@ export function parseMembeExpressionNoCall(
column
);
/* Template */
} else if (token === Token.TemplateContinuation || token === Token.TemplateTail) {
} else if (token === Token.TemplateContinuation || token === Token.TemplateSpan) {
parser.assignable = AssignmentKind.CannotAssign;

return parseMembeExpressionNoCall(
Expand Down
2 changes: 1 addition & 1 deletion src/token.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export const enum Token {

/* Template nodes */
TemplateContinuation = 8 | IsExpressionStart | IsMemberOrCallExpression,
TemplateTail = 9 | IsExpressionStart | IsMemberOrCallExpression,
TemplateSpan = 9 | IsExpressionStart | IsMemberOrCallExpression,

/* Punctuators */
Arrow = 10, // =>
Expand Down
68 changes: 34 additions & 34 deletions test/lexer/template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,33 +7,33 @@ import { scanSingleToken } from '../../src/lexer/scan';
describe('Lexer - Template', () => {
describe('Lexer - Template Tail', () => {
const tokens: [Context, Token, string, string][] = [
[Context.None, Token.TemplateTail, '``', ''],
[Context.None, Token.TemplateTail, '`a`', 'a'],
[Context.None, Token.TemplateTail, '`foo `', 'foo '],
[Context.None, Token.TemplateTail, '`foo `', 'foo '],
[Context.None, Token.TemplateTail, '`f1o2o`', 'f1o2o'],
[Context.None, Token.TemplateTail, '`دیوانه`', 'دیوانه'],
[Context.None, Token.TemplateTail, '`a℮`', 'a℮'],
[Context.None, Token.TemplateTail, '`℘`', '℘'],
[Context.None, Token.TemplateTail, '`a᧚`', 'a᧚'],
[Context.None, Token.TemplateTail, '`foo\\tbar`', 'foo\tbar'],
// [Context.None, Token.TemplateTail, '`\\x55a`', 'U'],
[Context.None, Token.TemplateTail, '`a\\nb`', 'a\nb'],
[Context.None, Token.TemplateTail, '`;`', ';'],
[Context.None, Token.TemplateTail, '``', ''],
[Context.None, Token.TemplateTail, '`123`', '123'],
[Context.None, Token.TemplateTail, '`true`', 'true'],
[Context.None, Token.TemplateTail, '`\n\r`', '\n\r'],
[Context.None, Token.TemplateTail, '`\r\n`', '\r\n'],
[Context.None, Token.TemplateTail, '`$$$a}`', '$$$a}'],
[Context.None, Token.TemplateSpan, '``', ''],
[Context.None, Token.TemplateSpan, '`a`', 'a'],
[Context.None, Token.TemplateSpan, '`foo `', 'foo '],
[Context.None, Token.TemplateSpan, '`foo `', 'foo '],
[Context.None, Token.TemplateSpan, '`f1o2o`', 'f1o2o'],
[Context.None, Token.TemplateSpan, '`دیوانه`', 'دیوانه'],
[Context.None, Token.TemplateSpan, '`a℮`', 'a℮'],
[Context.None, Token.TemplateSpan, '`℘`', '℘'],
[Context.None, Token.TemplateSpan, '`a᧚`', 'a᧚'],
[Context.None, Token.TemplateSpan, '`foo\\tbar`', 'foo\tbar'],
// [Context.None, Token.TemplateSpan, '`\\x55a`', 'U'],
[Context.None, Token.TemplateSpan, '`a\\nb`', 'a\nb'],
[Context.None, Token.TemplateSpan, '`;`', ';'],
[Context.None, Token.TemplateSpan, '``', ''],
[Context.None, Token.TemplateSpan, '`123`', '123'],
[Context.None, Token.TemplateSpan, '`true`', 'true'],
[Context.None, Token.TemplateSpan, '`\n\r`', '\n\r'],
[Context.None, Token.TemplateSpan, '`\r\n`', '\r\n'],
[Context.None, Token.TemplateSpan, '`$$$a}`', '$$$a}'],

// Russian letters
[Context.None, Token.TemplateTail, '`\\б`', 'б']
[Context.None, Token.TemplateSpan, '`\\б`', 'б']

// Unicode escape sequence - classic

// [Context.None, Token.TemplateTail, '`\\u1000`', 'က'],
//[Context.None, Token.TemplateTail, '`\\u0041`', 'A'],
// [Context.None, Token.TemplateSpan, '`\\u1000`', 'က'],
//[Context.None, Token.TemplateSpan, '`\\u0041`', 'A'],
];

for (const [ctx, token, op, value] of tokens) {
Expand Down Expand Up @@ -107,34 +107,34 @@ describe('Lexer - Template', () => {

describe('Lexer - Tagged Template', () => {
const tokens: [Context, Token, string, string | void][] = [
//[Context.TaggedTemplate, Token.TemplateTail, '`\\u{70bc`', undefined],
//[Context.TaggedTemplate, Token.TemplateSpan, '`\\u{70bc`', undefined],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\7${', undefined],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\1${', undefined],
[Context.TaggedTemplate, Token.TemplateContinuation, "`'${", "'"],
[Context.TaggedTemplate, Token.TemplateContinuation, '`"${', '"'],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\`${', '`'],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\`${', '`'],
[Context.TaggedTemplate, Token.TemplateTail, '`\\r`', '\r'],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\r`', '\r'],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\f${', '\f'],
[Context.TaggedTemplate, Token.TemplateTail, '`\\f`', '\f'],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\f`', '\f'],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\v${', '\v'],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\n${', '\n'],
[Context.TaggedTemplate, Token.TemplateTail, '`\\n`', '\n'],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\n`', '\n'],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\b${', '\b'],
[Context.TaggedTemplate, Token.TemplateTail, '`\\t`', '\t'],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\t`', '\t'],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\u{11ffff}${', undefined],
[Context.TaggedTemplate, Token.TemplateTail, '`\\u{11ffff}`', undefined],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\u{11ffff}`', undefined],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\u{11ffff}${', undefined],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\u{110000}${', undefined],
[Context.TaggedTemplate, Token.TemplateTail, '`\\u{g0g}`', undefined],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\u{g0g}`', undefined],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\u{0g}${', undefined],
[Context.TaggedTemplate, Token.TemplateTail, '`\\u{g0}`', undefined],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\u{g0}`', undefined],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\u{g}${', undefined],
[Context.TaggedTemplate, Token.TemplateTail, '`\\u{g}`', undefined],
[Context.TaggedTemplate, Token.TemplateTail, '`\\u{g}`', undefined],
[Context.TaggedTemplate, Token.TemplateTail, '`\\x0g`', undefined],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\u{g}`', undefined],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\u{g}`', undefined],
[Context.TaggedTemplate, Token.TemplateSpan, '`\\x0g`', undefined],
[Context.TaggedTemplate, Token.TemplateContinuation, '`\\x0g${', undefined],
[Context.TaggedTemplate, Token.TemplateTail, '`\\xg0`', undefined]
[Context.TaggedTemplate, Token.TemplateSpan, '`\\xg0`', undefined]
];

for (const [ctx, token, op, value] of tokens) {
Expand Down

0 comments on commit 5e41577

Please sign in to comment.