diff --git a/JetStreamDriver.js b/JetStreamDriver.js index 735e9f0c..f3bde78a 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -2363,11 +2363,14 @@ let BENCHMARKS = [ ], tags: ["default", "js", "Generators"], }), - new DefaultBenchmark({ + new AsyncBenchmark({ name: "js-tokens", files: [ "./generators/js-tokens.js", ], + preload: { + SOURCE_CODE: "./generators/js-tokens-input.js", + }, tags: ["default", "js", "Generators"], }), new DefaultBenchmark({ diff --git a/generators/js-tokens-input.js b/generators/js-tokens-input.js new file mode 100644 index 00000000..96f114c2 --- /dev/null +++ b/generators/js-tokens-input.js @@ -0,0 +1,412 @@ +// The MIT License (MIT) + +// Copyright (c) 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Simon Lydell + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +"use strict"; + +function jsTokensWrapperFunction() { + var Identifier, JSXIdentifier, JSXPunctuator, JSXString, JSXText, KeywordsWithExpressionAfter, KeywordsWithNoLineTerminatorAfter, LineTerminatorSequence, MultiLineComment, Newline, NumericLiteral, Punctuator, RegularExpressionLiteral, SingleLineComment, StringLiteral, Template, TokensNotPrecedingObjectLiteral, TokensPrecedingExpression, WhiteSpace, jsTokens; + RegularExpressionLiteral = /\/(?![*\/])(?:\[(?:(?![\]\\]).|\\.)*\]|(?![\/\\]).|\\.)*(\/[$_\u200C\u200D\p{ID_Continue}]*|\\)?/yu; + Punctuator = /--|\+\+|=>|\.{3}|\??\.(?!\d)|(?:&&|\|\||\?\?|[+\-%&|^]|\*{1,2}|<{1,2}|>{1,3}|!=?|={1,2}|\/(?![\/*]))=?|[?~,:;[\](){}]/y; + Identifier = /(\x23?)(?=[$_\p{ID_Start}\\])(?:[$_\u200C\u200D\p{ID_Continue}]|\\u[\da-fA-F]{4}|\\u\{[\da-fA-F]+\})+/yu; + StringLiteral = /(['"])(?:(?!\1)[^\\\n\r]|\\(?:\r\n|[^]))*(\1)?/y; + NumericLiteral = /(?:0[xX][\da-fA-F](?:_?[\da-fA-F])*|0[oO][0-7](?:_?[0-7])*|0[bB][01](?:_?[01])*)n?|0n|[1-9](?:_?\d)*n|(?:(?:0(?!\d)|0\d*[89]\d*|[1-9](?:_?\d)*)(?:\.(?:\d(?:_?\d)*)?)?|\.\d(?:_?\d)*)(?:[eE][+-]?\d(?:_?\d)*)?|0[0-7]+/y; + Template = /[`}](?:[^`\\$]|\\[^]|\$(?!\{))*(`|\$\{)?/y; + WhiteSpace = /[\t\v\f\ufeff\p{Zs}]+/yu; + LineTerminatorSequence = /\r?\n|[\r\u2028\u2029]/y; + MultiLineComment = /\/\*(?:[^*]|\*(?!\/))*(\*\/)?/y; + SingleLineComment = /\/\/.*/y; + JSXPunctuator = /[<>.:={}]|\/(?![\/*])/y; + JSXIdentifier = /[$_\p{ID_Start}][$_\u200C\u200D\p{ID_Continue}-]*/yu; + JSXString = /(['"])(?:(?!\1)[^])*(\1)?/y; + JSXText = /[^<>{}]+/y; + TokensPrecedingExpression = /^(?:[\/+-]|\.{3}|\?(?:InterpolationIn(?:JSX|Template)|NoLineTerminatorHere|NonExpressionParenEnd|UnaryIncDec))?$|[{}([,;<>=*%&|^!~?:]$/; + TokensNotPrecedingObjectLiteral = /^(?:=>|[;\]){}]|else|\?(?:NoLineTerminatorHere|NonExpressionParenEnd))?$/; + KeywordsWithExpressionAfter = /^(?:await|case|default|delete|do|else|instanceof|new|return|throw|typeof|void|yield)$/; + KeywordsWithNoLineTerminatorAfter = /^(?:return|throw|yield)$/; + Newline = RegExp(LineTerminatorSequence.source); + return function*(input, {jsx = false} = {}) { + var braces, firstCodePoint, isExpression, lastIndex, lastSignificantToken, length, match, mode, nextLastIndex, nextLastSignificantToken, parenNesting, postfixIncDec, punctuator, stack; + ({length} = input); + lastIndex = 0; + lastSignificantToken = ""; + stack = [ + {tag: "JS"} + ]; + braces = []; + parenNesting = 0; + postfixIncDec = false; + while (lastIndex < length) { + mode = stack[stack.length - 1]; + switch (mode.tag) { + case "JS": + case "JSNonExpressionParen": + case "InterpolationInTemplate": + case "InterpolationInJSX": + if (input[lastIndex] === "/" && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) { + RegularExpressionLiteral.lastIndex = lastIndex; + if (match = RegularExpressionLiteral.exec(input)) { + lastIndex = RegularExpressionLiteral.lastIndex; + lastSignificantToken = match[0]; + postfixIncDec = true; + yield ({ + type: "RegularExpressionLiteral", + value: match[0], + closed: match[1] !== void 0 && match[1] !== "\\" + }); + continue; + } + } + Punctuator.lastIndex = lastIndex; + if (match = Punctuator.exec(input)) { + punctuator = match[0]; + nextLastIndex = Punctuator.lastIndex; + nextLastSignificantToken = punctuator; + switch (punctuator) { + case "(": + if (lastSignificantToken === "?NonExpressionParenKeyword") { + stack.push({ + tag: "JSNonExpressionParen", + nesting: parenNesting + }); + } + parenNesting++; + postfixIncDec = false; + break; + case ")": + parenNesting--; + postfixIncDec = true; + if (mode.tag === "JSNonExpressionParen" && parenNesting === mode.nesting) { + stack.pop(); + nextLastSignificantToken = "?NonExpressionParenEnd"; + postfixIncDec = false; + } + break; + case "{": + Punctuator.lastIndex = 0; + isExpression = !TokensNotPrecedingObjectLiteral.test(lastSignificantToken) && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken)); + braces.push(isExpression); + postfixIncDec = false; + break; + case "}": + switch (mode.tag) { + case "InterpolationInTemplate": + if (braces.length === mode.nesting) { + Template.lastIndex = lastIndex; + match = Template.exec(input); + lastIndex = Template.lastIndex; + lastSignificantToken = match[0]; + if (match[1] === "${") { + lastSignificantToken = "?InterpolationInTemplate"; + postfixIncDec = false; + yield ({ + type: "TemplateMiddle", + value: match[0] + }); + } else { + stack.pop(); + postfixIncDec = true; + yield ({ + type: "TemplateTail", + value: match[0], + closed: match[1] === "`" + }); + } + continue; + } + break; + case "InterpolationInJSX": + if (braces.length === mode.nesting) { + stack.pop(); + lastIndex += 1; + lastSignificantToken = "}"; + yield ({ + type: "JSXPunctuator", + value: "}" + }); + continue; + } + } + postfixIncDec = braces.pop(); + nextLastSignificantToken = postfixIncDec ? "?ExpressionBraceEnd" : "}"; + break; + case "]": + postfixIncDec = true; + break; + case "++": + case "--": + nextLastSignificantToken = postfixIncDec ? "?PostfixIncDec" : "?UnaryIncDec"; + break; + case "<": + if (jsx && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) { + stack.push({tag: "JSXTag"}); + lastIndex += 1; + lastSignificantToken = "<"; + yield ({ + type: "JSXPunctuator", + value: punctuator + }); + continue; + } + postfixIncDec = false; + break; + default: + postfixIncDec = false; + } + lastIndex = nextLastIndex; + lastSignificantToken = nextLastSignificantToken; + yield ({ + type: "Punctuator", + value: punctuator + }); + continue; + } + Identifier.lastIndex = lastIndex; + if (match = Identifier.exec(input)) { + lastIndex = Identifier.lastIndex; + nextLastSignificantToken = match[0]; + switch (match[0]) { + case "for": + case "if": + case "while": + case "with": + if (lastSignificantToken !== "." && lastSignificantToken !== "?.") { + nextLastSignificantToken = "?NonExpressionParenKeyword"; + } + } + lastSignificantToken = nextLastSignificantToken; + postfixIncDec = !KeywordsWithExpressionAfter.test(match[0]); + yield ({ + type: match[1] === "#" ? "PrivateIdentifier" : "IdentifierName", + value: match[0] + }); + continue; + } + StringLiteral.lastIndex = lastIndex; + if (match = StringLiteral.exec(input)) { + lastIndex = StringLiteral.lastIndex; + lastSignificantToken = match[0]; + postfixIncDec = true; + yield ({ + type: "StringLiteral", + value: match[0], + closed: match[2] !== void 0 + }); + continue; + } + NumericLiteral.lastIndex = lastIndex; + if (match = NumericLiteral.exec(input)) { + lastIndex = NumericLiteral.lastIndex; + lastSignificantToken = match[0]; + postfixIncDec = true; + yield ({ + type: "NumericLiteral", + value: match[0] + }); + continue; + } + Template.lastIndex = lastIndex; + if (match = Template.exec(input)) { + lastIndex = Template.lastIndex; + lastSignificantToken = match[0]; + if (match[1] === "${") { + lastSignificantToken = "?InterpolationInTemplate"; + stack.push({ + tag: "InterpolationInTemplate", + nesting: braces.length + }); + postfixIncDec = false; + yield ({ + type: "TemplateHead", + value: match[0] + }); + } else { + postfixIncDec = true; + yield ({ + type: "NoSubstitutionTemplate", + value: match[0], + closed: match[1] === "`" + }); + } + continue; + } + break; + case "JSXTag": + case "JSXTagEnd": + JSXPunctuator.lastIndex = lastIndex; + if (match = JSXPunctuator.exec(input)) { + lastIndex = JSXPunctuator.lastIndex; + nextLastSignificantToken = match[0]; + switch (match[0]) { + case "<": + stack.push({tag: "JSXTag"}); + break; + case ">": + stack.pop(); + if (lastSignificantToken === "/" || mode.tag === "JSXTagEnd") { + nextLastSignificantToken = "?JSX"; + postfixIncDec = true; + } else { + stack.push({tag: "JSXChildren"}); + } + break; + case "{": + stack.push({ + tag: "InterpolationInJSX", + nesting: braces.length + }); + nextLastSignificantToken = "?InterpolationInJSX"; + postfixIncDec = false; + break; + case "/": + if (lastSignificantToken === "<") { + stack.pop(); + if (stack[stack.length - 1].tag === "JSXChildren") { + stack.pop(); + } + stack.push({tag: "JSXTagEnd"}); + } + } + lastSignificantToken = nextLastSignificantToken; + yield ({ + type: "JSXPunctuator", + value: match[0] + }); + continue; + } + JSXIdentifier.lastIndex = lastIndex; + if (match = JSXIdentifier.exec(input)) { + lastIndex = JSXIdentifier.lastIndex; + lastSignificantToken = match[0]; + yield ({ + type: "JSXIdentifier", + value: match[0] + }); + continue; + } + JSXString.lastIndex = lastIndex; + if (match = JSXString.exec(input)) { + lastIndex = JSXString.lastIndex; + lastSignificantToken = match[0]; + yield ({ + type: "JSXString", + value: match[0], + closed: match[2] !== void 0 + }); + continue; + } + break; + case "JSXChildren": + JSXText.lastIndex = lastIndex; + if (match = JSXText.exec(input)) { + lastIndex = JSXText.lastIndex; + lastSignificantToken = match[0]; + yield ({ + type: "JSXText", + value: match[0] + }); + continue; + } + switch (input[lastIndex]) { + case "<": + stack.push({tag: "JSXTag"}); + lastIndex++; + lastSignificantToken = "<"; + yield ({ + type: "JSXPunctuator", + value: "<" + }); + continue; + case "{": + stack.push({ + tag: "InterpolationInJSX", + nesting: braces.length + }); + lastIndex++; + lastSignificantToken = "?InterpolationInJSX"; + postfixIncDec = false; + yield ({ + type: "JSXPunctuator", + value: "{" + }); + continue; + } + } + WhiteSpace.lastIndex = lastIndex; + if (match = WhiteSpace.exec(input)) { + lastIndex = WhiteSpace.lastIndex; + yield ({ + type: "WhiteSpace", + value: match[0] + }); + continue; + } + LineTerminatorSequence.lastIndex = lastIndex; + if (match = LineTerminatorSequence.exec(input)) { + lastIndex = LineTerminatorSequence.lastIndex; + postfixIncDec = false; + if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) { + lastSignificantToken = "?NoLineTerminatorHere"; + } + yield ({ + type: "LineTerminatorSequence", + value: match[0] + }); + continue; + } + MultiLineComment.lastIndex = lastIndex; + if (match = MultiLineComment.exec(input)) { + lastIndex = MultiLineComment.lastIndex; + if (Newline.test(match[0])) { + postfixIncDec = false; + if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) { + lastSignificantToken = "?NoLineTerminatorHere"; + } + } + yield ({ + type: "MultiLineComment", + value: match[0], + closed: match[1] !== void 0 + }); + continue; + } + SingleLineComment.lastIndex = lastIndex; + if (match = SingleLineComment.exec(input)) { + lastIndex = SingleLineComment.lastIndex; + postfixIncDec = false; + yield ({ + type: "SingleLineComment", + value: match[0] + }); + continue; + } + firstCodePoint = String.fromCodePoint(input.codePointAt(lastIndex)); + lastIndex += firstCodePoint.length; + lastSignificantToken = firstCodePoint; + postfixIncDec = false; + yield ({ + type: mode.tag.startsWith("JSX") ? "JSXInvalid" : "Invalid", + value: firstCodePoint + }); + + } + }; + } \ No newline at end of file diff --git a/generators/js-tokens.js b/generators/js-tokens.js index 8385109c..b09210a1 100644 --- a/generators/js-tokens.js +++ b/generators/js-tokens.js @@ -410,7 +410,6 @@ function jsTokensWrapperFunction() { }; }; -const jsTokensSourceCode = jsTokensWrapperFunction.toString(); const jsxSourceCode = ` function Comment(props) { @@ -437,11 +436,20 @@ function Comment(props) { const jsTokens = jsTokensWrapperFunction(); class Benchmark { + EXPECTED_TOKEN_COUNT = 115100; + + tokenCount = 0; + jsTokensSourceCode = ""; + + async init() { + this.jsTokensSourceCode = await JetStream.getString(JetStream.preload.SOURCE_CODE); + } + runIteration() { this.tokenCount = 0; for (var i = 0; i < 25; i++) { - for (const token of jsTokens(jsTokensSourceCode)) + for (const token of jsTokens(this.jsTokensSourceCode)) this.tokenCount++; } @@ -452,7 +460,7 @@ class Benchmark { } validate(iterations) { - if (this.tokenCount !== 113975) - throw new Error(`this.tokenCount of ${this.tokenCount} is invalid!`); + if (this.tokenCount !== this.EXPECTED_TOKEN_COUNT) + throw new Error(`Expected this.tokenCount of ${this.EXPECTED_TOKEN_COUNT}, but got ${this.tokenCount}!`); } }