From 2a65293cf08ff2d96b70a6eab35e22d4205874e4 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 16:09:27 +0100 Subject: [PATCH 1/7] inject source code from the outside --- JetStreamDriver.js | 5 ++++- generators/js-tokens.js | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/JetStreamDriver.js b/JetStreamDriver.js index e15905c7..a70b110a 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -2345,11 +2345,14 @@ let BENCHMARKS = [ ], tags: ["default", "js", "Generators"], }), - new DefaultBenchmark({ + new AsyncBenchmark({ name: "js-tokens", files: [ "./generators/js-tokens.js", ], + preload: { + jsTokensSourceCode: "./generators/js-tokens.js", + }, tags: ["default", "js", "Generators"], }), new DefaultBenchmark({ diff --git a/generators/js-tokens.js b/generators/js-tokens.js index 8385109c..fc7d8fc5 100644 --- a/generators/js-tokens.js +++ b/generators/js-tokens.js @@ -437,11 +437,20 @@ function Comment(props) { const jsTokens = jsTokensWrapperFunction(); class Benchmark { + EXPECTED_TOKEN_COUNT = 122200; + tokenCount = 0; + + jsTokensSourceCode; + + async init() { + this.jsTokensSourceCode = await JetStream.getString(JetStream.preload.jsTokensSourceCode); + } + runIteration() { this.tokenCount = 0; for (var i = 0; i < 25; i++) { - for (const token of jsTokens(jsTokensSourceCode)) + for (const token of jsTokens(this.jsTokensSourceCode)) this.tokenCount++; } @@ -452,7 +461,7 @@ class Benchmark { } validate(iterations) { - if (this.tokenCount !== 113975) - throw new Error(`this.tokenCount of ${this.tokenCount} is invalid!`); + if (this.tokenCount !== this.EXPECTED_TOKEN_COUNT) + throw new Error(`Expected this.tokenCount of ${this.EXPECTED_TOKEN_COUNT}, but got ${this.tokenCount}!`); } } From 2946337cd6406c057db4918c2b15ec076cfc4041 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 16:10:04 +0100 Subject: [PATCH 2/7] cleanup --- generators/js-tokens.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/generators/js-tokens.js b/generators/js-tokens.js index fc7d8fc5..197d771a 100644 --- a/generators/js-tokens.js +++ b/generators/js-tokens.js @@ -410,7 +410,6 @@ function jsTokensWrapperFunction() { }; }; -const jsTokensSourceCode = jsTokensWrapperFunction.toString(); const jsxSourceCode = ` function Comment(props) { @@ -438,9 +437,9 @@ const jsTokens = jsTokensWrapperFunction(); class Benchmark { EXPECTED_TOKEN_COUNT = 122200; - tokenCount = 0; - jsTokensSourceCode; + tokenCount = 0; + jsTokensSourceCode = ""; async init() { this.jsTokensSourceCode = await JetStream.getString(JetStream.preload.jsTokensSourceCode); From 8ad7b0f22b5a0d6c95fa4a41a0ddff4b7018fd76 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 17:42:36 +0100 Subject: [PATCH 3/7] fix token count --- generators/js-tokens.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generators/js-tokens.js b/generators/js-tokens.js index 197d771a..ba4c9a1a 100644 --- a/generators/js-tokens.js +++ b/generators/js-tokens.js @@ -436,7 +436,7 @@ function Comment(props) { const jsTokens = jsTokensWrapperFunction(); class Benchmark { - EXPECTED_TOKEN_COUNT = 122200; + EXPECTED_TOKEN_COUNT = 121975; tokenCount = 0; jsTokensSourceCode = ""; From 1b7dce560621809fbe99ae2e747580efabc41160 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 17:59:01 +0100 Subject: [PATCH 4/7] fix tokens --- JetStreamDriver.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JetStreamDriver.js b/JetStreamDriver.js index a70b110a..660288ee 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -2351,7 +2351,7 @@ let BENCHMARKS = [ "./generators/js-tokens.js", ], preload: { - jsTokensSourceCode: "./generators/js-tokens.js", + "jsTokensSourceCode": "./generators/js-tokens.js", }, tags: ["default", "js", "Generators"], }), From d2b3c889a5e41dcbf09d144fbc6852a2de4925bd Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 18:09:27 +0100 Subject: [PATCH 5/7] fix --- JetStreamDriver.js | 6 ++++-- generators/js-tokens.js | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/JetStreamDriver.js b/JetStreamDriver.js index 660288ee..b91704b9 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -905,8 +905,10 @@ class Benchmark { } if (this.plan.preload) { let preloadCode = ""; - for (let [ variableName, blobURLOrPath ] of this.preloads) + for (let [ variableName, blobURLOrPath ] of this.preloads) { + console.log(variableName, blobURLOrPath); preloadCode += `JetStream.preload[${JSON.stringify(variableName)}] = "${blobURLOrPath}";\n`; + } scripts.add(preloadCode); } @@ -2351,7 +2353,7 @@ let BENCHMARKS = [ "./generators/js-tokens.js", ], preload: { - "jsTokensSourceCode": "./generators/js-tokens.js", + SOURCE_CODE: "./generators/js-tokens-input.js", }, tags: ["default", "js", "Generators"], }), diff --git a/generators/js-tokens.js b/generators/js-tokens.js index ba4c9a1a..e0b9d327 100644 --- a/generators/js-tokens.js +++ b/generators/js-tokens.js @@ -442,7 +442,7 @@ class Benchmark { jsTokensSourceCode = ""; async init() { - this.jsTokensSourceCode = await JetStream.getString(JetStream.preload.jsTokensSourceCode); + this.jsTokensSourceCode = await JetStream.getString(JetStream.preload.SOURCE_CODE); } runIteration() { From b6d1cdfb82bbcd221a7639997008df5703037cd8 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 18:09:35 +0100 Subject: [PATCH 6/7] input --- generators/js-tokens-input.js | 412 ++++++++++++++++++++++++++++++++++ 1 file changed, 412 insertions(+) create mode 100644 generators/js-tokens-input.js diff --git a/generators/js-tokens-input.js b/generators/js-tokens-input.js new file mode 100644 index 00000000..96f114c2 --- /dev/null +++ b/generators/js-tokens-input.js @@ -0,0 +1,412 @@ +// The MIT License (MIT) + +// Copyright (c) 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Simon Lydell + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +"use strict"; + +function jsTokensWrapperFunction() { + var Identifier, JSXIdentifier, JSXPunctuator, JSXString, JSXText, KeywordsWithExpressionAfter, KeywordsWithNoLineTerminatorAfter, LineTerminatorSequence, MultiLineComment, Newline, NumericLiteral, Punctuator, RegularExpressionLiteral, SingleLineComment, StringLiteral, Template, TokensNotPrecedingObjectLiteral, TokensPrecedingExpression, WhiteSpace, jsTokens; + RegularExpressionLiteral = /\/(?![*\/])(?:\[(?:(?![\]\\]).|\\.)*\]|(?![\/\\]).|\\.)*(\/[$_\u200C\u200D\p{ID_Continue}]*|\\)?/yu; + Punctuator = /--|\+\+|=>|\.{3}|\??\.(?!\d)|(?:&&|\|\||\?\?|[+\-%&|^]|\*{1,2}|<{1,2}|>{1,3}|!=?|={1,2}|\/(?![\/*]))=?|[?~,:;[\](){}]/y; + Identifier = /(\x23?)(?=[$_\p{ID_Start}\\])(?:[$_\u200C\u200D\p{ID_Continue}]|\\u[\da-fA-F]{4}|\\u\{[\da-fA-F]+\})+/yu; + StringLiteral = /(['"])(?:(?!\1)[^\\\n\r]|\\(?:\r\n|[^]))*(\1)?/y; + NumericLiteral = /(?:0[xX][\da-fA-F](?:_?[\da-fA-F])*|0[oO][0-7](?:_?[0-7])*|0[bB][01](?:_?[01])*)n?|0n|[1-9](?:_?\d)*n|(?:(?:0(?!\d)|0\d*[89]\d*|[1-9](?:_?\d)*)(?:\.(?:\d(?:_?\d)*)?)?|\.\d(?:_?\d)*)(?:[eE][+-]?\d(?:_?\d)*)?|0[0-7]+/y; + Template = /[`}](?:[^`\\$]|\\[^]|\$(?!\{))*(`|\$\{)?/y; + WhiteSpace = /[\t\v\f\ufeff\p{Zs}]+/yu; + LineTerminatorSequence = /\r?\n|[\r\u2028\u2029]/y; + MultiLineComment = /\/\*(?:[^*]|\*(?!\/))*(\*\/)?/y; + SingleLineComment = /\/\/.*/y; + JSXPunctuator = /[<>.:={}]|\/(?![\/*])/y; + JSXIdentifier = /[$_\p{ID_Start}][$_\u200C\u200D\p{ID_Continue}-]*/yu; + JSXString = /(['"])(?:(?!\1)[^])*(\1)?/y; + JSXText = /[^<>{}]+/y; + TokensPrecedingExpression = /^(?:[\/+-]|\.{3}|\?(?:InterpolationIn(?:JSX|Template)|NoLineTerminatorHere|NonExpressionParenEnd|UnaryIncDec))?$|[{}([,;<>=*%&|^!~?:]$/; + TokensNotPrecedingObjectLiteral = /^(?:=>|[;\]){}]|else|\?(?:NoLineTerminatorHere|NonExpressionParenEnd))?$/; + KeywordsWithExpressionAfter = /^(?:await|case|default|delete|do|else|instanceof|new|return|throw|typeof|void|yield)$/; + KeywordsWithNoLineTerminatorAfter = /^(?:return|throw|yield)$/; + Newline = RegExp(LineTerminatorSequence.source); + return function*(input, {jsx = false} = {}) { + var braces, firstCodePoint, isExpression, lastIndex, lastSignificantToken, length, match, mode, nextLastIndex, nextLastSignificantToken, parenNesting, postfixIncDec, punctuator, stack; + ({length} = input); + lastIndex = 0; + lastSignificantToken = ""; + stack = [ + {tag: "JS"} + ]; + braces = []; + parenNesting = 0; + postfixIncDec = false; + while (lastIndex < length) { + mode = stack[stack.length - 1]; + switch (mode.tag) { + case "JS": + case "JSNonExpressionParen": + case "InterpolationInTemplate": + case "InterpolationInJSX": + if (input[lastIndex] === "/" && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) { + RegularExpressionLiteral.lastIndex = lastIndex; + if (match = RegularExpressionLiteral.exec(input)) { + lastIndex = RegularExpressionLiteral.lastIndex; + lastSignificantToken = match[0]; + postfixIncDec = true; + yield ({ + type: "RegularExpressionLiteral", + value: match[0], + closed: match[1] !== void 0 && match[1] !== "\\" + }); + continue; + } + } + Punctuator.lastIndex = lastIndex; + if (match = Punctuator.exec(input)) { + punctuator = match[0]; + nextLastIndex = Punctuator.lastIndex; + nextLastSignificantToken = punctuator; + switch (punctuator) { + case "(": + if (lastSignificantToken === "?NonExpressionParenKeyword") { + stack.push({ + tag: "JSNonExpressionParen", + nesting: parenNesting + }); + } + parenNesting++; + postfixIncDec = false; + break; + case ")": + parenNesting--; + postfixIncDec = true; + if (mode.tag === "JSNonExpressionParen" && parenNesting === mode.nesting) { + stack.pop(); + nextLastSignificantToken = "?NonExpressionParenEnd"; + postfixIncDec = false; + } + break; + case "{": + Punctuator.lastIndex = 0; + isExpression = !TokensNotPrecedingObjectLiteral.test(lastSignificantToken) && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken)); + braces.push(isExpression); + postfixIncDec = false; + break; + case "}": + switch (mode.tag) { + case "InterpolationInTemplate": + if (braces.length === mode.nesting) { + Template.lastIndex = lastIndex; + match = Template.exec(input); + lastIndex = Template.lastIndex; + lastSignificantToken = match[0]; + if (match[1] === "${") { + lastSignificantToken = "?InterpolationInTemplate"; + postfixIncDec = false; + yield ({ + type: "TemplateMiddle", + value: match[0] + }); + } else { + stack.pop(); + postfixIncDec = true; + yield ({ + type: "TemplateTail", + value: match[0], + closed: match[1] === "`" + }); + } + continue; + } + break; + case "InterpolationInJSX": + if (braces.length === mode.nesting) { + stack.pop(); + lastIndex += 1; + lastSignificantToken = "}"; + yield ({ + type: "JSXPunctuator", + value: "}" + }); + continue; + } + } + postfixIncDec = braces.pop(); + nextLastSignificantToken = postfixIncDec ? "?ExpressionBraceEnd" : "}"; + break; + case "]": + postfixIncDec = true; + break; + case "++": + case "--": + nextLastSignificantToken = postfixIncDec ? "?PostfixIncDec" : "?UnaryIncDec"; + break; + case "<": + if (jsx && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) { + stack.push({tag: "JSXTag"}); + lastIndex += 1; + lastSignificantToken = "<"; + yield ({ + type: "JSXPunctuator", + value: punctuator + }); + continue; + } + postfixIncDec = false; + break; + default: + postfixIncDec = false; + } + lastIndex = nextLastIndex; + lastSignificantToken = nextLastSignificantToken; + yield ({ + type: "Punctuator", + value: punctuator + }); + continue; + } + Identifier.lastIndex = lastIndex; + if (match = Identifier.exec(input)) { + lastIndex = Identifier.lastIndex; + nextLastSignificantToken = match[0]; + switch (match[0]) { + case "for": + case "if": + case "while": + case "with": + if (lastSignificantToken !== "." && lastSignificantToken !== "?.") { + nextLastSignificantToken = "?NonExpressionParenKeyword"; + } + } + lastSignificantToken = nextLastSignificantToken; + postfixIncDec = !KeywordsWithExpressionAfter.test(match[0]); + yield ({ + type: match[1] === "#" ? "PrivateIdentifier" : "IdentifierName", + value: match[0] + }); + continue; + } + StringLiteral.lastIndex = lastIndex; + if (match = StringLiteral.exec(input)) { + lastIndex = StringLiteral.lastIndex; + lastSignificantToken = match[0]; + postfixIncDec = true; + yield ({ + type: "StringLiteral", + value: match[0], + closed: match[2] !== void 0 + }); + continue; + } + NumericLiteral.lastIndex = lastIndex; + if (match = NumericLiteral.exec(input)) { + lastIndex = NumericLiteral.lastIndex; + lastSignificantToken = match[0]; + postfixIncDec = true; + yield ({ + type: "NumericLiteral", + value: match[0] + }); + continue; + } + Template.lastIndex = lastIndex; + if (match = Template.exec(input)) { + lastIndex = Template.lastIndex; + lastSignificantToken = match[0]; + if (match[1] === "${") { + lastSignificantToken = "?InterpolationInTemplate"; + stack.push({ + tag: "InterpolationInTemplate", + nesting: braces.length + }); + postfixIncDec = false; + yield ({ + type: "TemplateHead", + value: match[0] + }); + } else { + postfixIncDec = true; + yield ({ + type: "NoSubstitutionTemplate", + value: match[0], + closed: match[1] === "`" + }); + } + continue; + } + break; + case "JSXTag": + case "JSXTagEnd": + JSXPunctuator.lastIndex = lastIndex; + if (match = JSXPunctuator.exec(input)) { + lastIndex = JSXPunctuator.lastIndex; + nextLastSignificantToken = match[0]; + switch (match[0]) { + case "<": + stack.push({tag: "JSXTag"}); + break; + case ">": + stack.pop(); + if (lastSignificantToken === "/" || mode.tag === "JSXTagEnd") { + nextLastSignificantToken = "?JSX"; + postfixIncDec = true; + } else { + stack.push({tag: "JSXChildren"}); + } + break; + case "{": + stack.push({ + tag: "InterpolationInJSX", + nesting: braces.length + }); + nextLastSignificantToken = "?InterpolationInJSX"; + postfixIncDec = false; + break; + case "/": + if (lastSignificantToken === "<") { + stack.pop(); + if (stack[stack.length - 1].tag === "JSXChildren") { + stack.pop(); + } + stack.push({tag: "JSXTagEnd"}); + } + } + lastSignificantToken = nextLastSignificantToken; + yield ({ + type: "JSXPunctuator", + value: match[0] + }); + continue; + } + JSXIdentifier.lastIndex = lastIndex; + if (match = JSXIdentifier.exec(input)) { + lastIndex = JSXIdentifier.lastIndex; + lastSignificantToken = match[0]; + yield ({ + type: "JSXIdentifier", + value: match[0] + }); + continue; + } + JSXString.lastIndex = lastIndex; + if (match = JSXString.exec(input)) { + lastIndex = JSXString.lastIndex; + lastSignificantToken = match[0]; + yield ({ + type: "JSXString", + value: match[0], + closed: match[2] !== void 0 + }); + continue; + } + break; + case "JSXChildren": + JSXText.lastIndex = lastIndex; + if (match = JSXText.exec(input)) { + lastIndex = JSXText.lastIndex; + lastSignificantToken = match[0]; + yield ({ + type: "JSXText", + value: match[0] + }); + continue; + } + switch (input[lastIndex]) { + case "<": + stack.push({tag: "JSXTag"}); + lastIndex++; + lastSignificantToken = "<"; + yield ({ + type: "JSXPunctuator", + value: "<" + }); + continue; + case "{": + stack.push({ + tag: "InterpolationInJSX", + nesting: braces.length + }); + lastIndex++; + lastSignificantToken = "?InterpolationInJSX"; + postfixIncDec = false; + yield ({ + type: "JSXPunctuator", + value: "{" + }); + continue; + } + } + WhiteSpace.lastIndex = lastIndex; + if (match = WhiteSpace.exec(input)) { + lastIndex = WhiteSpace.lastIndex; + yield ({ + type: "WhiteSpace", + value: match[0] + }); + continue; + } + LineTerminatorSequence.lastIndex = lastIndex; + if (match = LineTerminatorSequence.exec(input)) { + lastIndex = LineTerminatorSequence.lastIndex; + postfixIncDec = false; + if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) { + lastSignificantToken = "?NoLineTerminatorHere"; + } + yield ({ + type: "LineTerminatorSequence", + value: match[0] + }); + continue; + } + MultiLineComment.lastIndex = lastIndex; + if (match = MultiLineComment.exec(input)) { + lastIndex = MultiLineComment.lastIndex; + if (Newline.test(match[0])) { + postfixIncDec = false; + if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) { + lastSignificantToken = "?NoLineTerminatorHere"; + } + } + yield ({ + type: "MultiLineComment", + value: match[0], + closed: match[1] !== void 0 + }); + continue; + } + SingleLineComment.lastIndex = lastIndex; + if (match = SingleLineComment.exec(input)) { + lastIndex = SingleLineComment.lastIndex; + postfixIncDec = false; + yield ({ + type: "SingleLineComment", + value: match[0] + }); + continue; + } + firstCodePoint = String.fromCodePoint(input.codePointAt(lastIndex)); + lastIndex += firstCodePoint.length; + lastSignificantToken = firstCodePoint; + postfixIncDec = false; + yield ({ + type: mode.tag.startsWith("JSX") ? "JSXInvalid" : "Invalid", + value: firstCodePoint + }); + + } + }; + } \ No newline at end of file From 6e82cba71b6f38dc233e4a17d05870c962ede1f4 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 28 Oct 2025 15:22:05 +0100 Subject: [PATCH 7/7] fix tokens inpu --- generators/js-tokens.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generators/js-tokens.js b/generators/js-tokens.js index e0b9d327..b09210a1 100644 --- a/generators/js-tokens.js +++ b/generators/js-tokens.js @@ -436,7 +436,7 @@ function Comment(props) { const jsTokens = jsTokensWrapperFunction(); class Benchmark { - EXPECTED_TOKEN_COUNT = 121975; + EXPECTED_TOKEN_COUNT = 115100; tokenCount = 0; jsTokensSourceCode = "";