diff --git a/JetStreamDriver.js b/JetStreamDriver.js index f248e11..98874f7 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -30,58 +30,7 @@ const measureTotalTimeAsSubtest = false; // Once we move to preloading all resou const defaultIterationCount = 120; const defaultWorstCaseCount = 4; -globalThis.performance ??= Date; -globalThis.RAMification ??= false; -globalThis.testIterationCount ??= undefined; -globalThis.testIterationCountMap ??= new Map(); -globalThis.testWorstCaseCount ??= undefined; -globalThis.testWorstCaseCountMap ??= new Map(); -globalThis.dumpJSONResults ??= false; -globalThis.testList ??= undefined; -globalThis.startDelay ??= undefined; -globalThis.shouldReport ??= false; -globalThis.prefetchResources ??= true; - -function getIntParam(urlParams, key) { - const rawValue = urlParams.get(key); - const value = parseInt(rawValue); - if (value <= 0) - throw new Error(`Expected positive value for ${key}, but got ${rawValue}`); - return value; -} - -function getBoolParam(urlParams, key) { - const rawValue = urlParams.get(key).toLowerCase() - return !(rawValue === "false" || rawValue === "0") - } - -function getTestListParam(urlParams, key) { - if (globalThis.testList?.length) - throw new Error(`Overriding previous testList=${globalThis.testList.join()} with ${key} url-parameter.`); - return urlParams.getAll(key); -} - -if (typeof(URLSearchParams) !== "undefined") { - const urlParameters = new URLSearchParams(window.location.search); - if (urlParameters.has("report")) - globalThis.shouldReport = urlParameters.get("report").toLowerCase() == "true"; - if (urlParameters.has("startDelay")) - globalThis.startDelay = getIntParam(urlParameters, "startDelay"); - if (globalThis.shouldReport && !globalThis.startDelay) - globalThis.startDelay = 4000; - if (urlParameters.has("tag")) - globalThis.testList = getTestListParam(urlParameters, "tag"); - if (urlParameters.has("test")) - globalThis.testList = getTestListParam(urlParameters, "test"); - if (urlParameters.has("iterationCount")) - globalThis.testIterationCount = getIntParam(urlParameters, "iterationCount"); - if (urlParameters.has("worstCaseCount")) - globalThis.testWorstCaseCount = getIntParam(urlParameters, "worstCaseCount"); - if (urlParameters.has("prefetchResources")) - globalThis.prefetchResources = getBoolParam(urlParameters, "prefetchResources"); -} - -if (!globalThis.prefetchResources) +if (!JetStreamParams.prefetchResources) console.warn("Disabling resource prefetching!"); // Used for the promise representing the current benchmark run. @@ -110,20 +59,20 @@ function displayCategoryScores() { } function getIterationCount(plan) { - if (testIterationCountMap.has(plan.name)) - return testIterationCountMap.get(plan.name); - if (globalThis.testIterationCount) - return globalThis.testIterationCount; + if (JetStreamParams.testIterationCountMap.has(plan.name)) + return JetStreamParams.testIterationCountMap.get(plan.name); + if (JetStreamParams.testIterationCount) + return JetStreamParams.testIterationCount; if (plan.iterations) return plan.iterations; return defaultIterationCount; } function getWorstCaseCount(plan) { - if (testWorstCaseCountMap.has(plan.name)) - return testWorstCaseCountMap.get(plan.name); - if (globalThis.testWorstCaseCount) - return globalThis.testWorstCaseCount; + if (JetStreamParams.testWorstCaseCountMap.has(plan.name)) + return JetStreamParams.testWorstCaseCountMap.get(plan.name); + if (JetStreamParams.testWorstCaseCount) + return JetStreamParams.testWorstCaseCount; if (plan.worstCaseCount) return plan.worstCaseCount; return defaultWorstCaseCount; @@ -201,7 +150,7 @@ class ShellFileLoader { // share common code. load(url) { console.assert(!isInBrowser); - if (!globalThis.prefetchResources) + if (!JetStreamParams.prefetchResources) return `load("${url}");` if (this.requests.has(url)) { @@ -240,7 +189,7 @@ class Driver { if (isInBrowser) { statusElement = document.getElementById("status"); statusElement.innerHTML = ``; - } else if (!dumpJSONResults) + } else if (!JetStreamParams.dumpJSONResults) console.log("Starting JetStream3"); performance.mark("update-ui-start"); @@ -260,7 +209,7 @@ class Driver { performance.mark("update-ui"); benchmark.updateUIAfterRun(); - if (isInBrowser && globalThis.prefetchResources) { + if (isInBrowser && JetStreamParams.prefetchResources) { const cache = JetStream.blobDataCache; for (const file of benchmark.files) { const blobData = cache[file]; @@ -276,7 +225,7 @@ class Driver { if (measureTotalTimeAsSubtest) { if (isInBrowser) document.getElementById("benchmark-total-time-score").innerHTML = uiFriendlyNumber(totalTime); - else if (!dumpJSONResults) + else if (!JetStreamParams.dumpJSONResults) console.log("Total time:", uiFriendlyNumber(totalTime)); allScores.push(totalTime); } @@ -314,7 +263,7 @@ class Driver { if (showScoreDetails) displayCategoryScores(); statusElement.innerHTML = ""; - } else if (!dumpJSONResults) { + } else if (!JetStreamParams.dumpJSONResults) { console.log("\n"); for (let [category, scores] of categoryScores) console.log(`${category}: ${uiFriendlyScore(geomeanScore(scores))}`); @@ -404,8 +353,8 @@ class Driver { this.isReady = true; if (isInBrowser) { globalThis.dispatchEvent(new Event("JetStreamReady")); - if (typeof(globalThis.startDelay) !== "undefined") { - setTimeout(() => this.start(), globalThis.startDelay); + if (typeof(JetStreamParams.startDelay) !== "undefined") { + setTimeout(() => this.start(), JetStreamParams.startDelay); } } } @@ -510,7 +459,7 @@ class Driver { dumpJSONResultsIfNeeded() { - if (dumpJSONResults) { + if (JetStreamParams.dumpJSONResults) { console.log("\n"); console.log(this.resultsJSON()); console.log("\n"); @@ -529,7 +478,7 @@ class Driver { if (!isInBrowser) return; - if (!globalThis.shouldReport) + if (!JetStreamParams.shouldReport) return; const content = this.resultsJSON(); @@ -798,8 +747,8 @@ class Benchmark { if (this.plan.deterministicRandom) code += `Math.random.__resetSeed();`; - if (globalThis.customPreIterationCode) - code += customPreIterationCode; + if (JetStreamParams.customPreIterationCode) + code += JetStreamParams.customPreIterationCode; return code; } @@ -807,8 +756,8 @@ class Benchmark { get postIterationCode() { let code = ""; - if (globalThis.customPostIterationCode) - code += customPostIterationCode; + if (JetStreamParams.customPostIterationCode) + code += JetStreamParams.customPostIterationCode; return code; } @@ -842,7 +791,7 @@ class Benchmark { } else { const cache = JetStream.blobDataCache; for (const file of this.plan.files) { - scripts.addWithURL(globalThis.prefetchResources ? cache[file].blobURL : file); + scripts.addWithURL(JetStreamParams.prefetchResources ? cache[file].blobURL : file); } } @@ -856,7 +805,7 @@ class Benchmark { performance.mark(this.name); this.startTime = performance.now(); - if (RAMification) + if (JetStreamParams.RAMification) resetMemoryPeak(); let magicFrame; @@ -876,7 +825,7 @@ class Benchmark { this.endTime = performance.now(); performance.measure(this.name, this.name); - if (RAMification) { + if (JetStreamParams.RAMification) { const memoryFootprint = MemoryFootprint(); this.currentFootprint = memoryFootprint.current; this.peakFootprint = memoryFootprint.peak; @@ -893,7 +842,7 @@ class Benchmark { async doLoadBlob(resource) { const blobData = JetStream.blobDataCache[resource]; - if (!globalThis.prefetchResources) { + if (!JetStreamParams.prefetchResources) { blobData.blobURL = resource; return blobData; } @@ -1061,7 +1010,7 @@ class Benchmark { } updateUIBeforeRun() { - if (!dumpJSONResults) + if (!JetStreamParams.dumpJSONResults) console.log(`Running ${this.name}:`); if (isInBrowser) this.updateUIBeforeRunInBrowser(); @@ -1080,7 +1029,7 @@ class Benchmark { const scoreEntries = Object.entries(this.allScores()); if (isInBrowser) this.updateUIAfterRunInBrowser(scoreEntries); - if (dumpJSONResults) + if (JetStreamParams.dumpJSONResults) return; this.updateConsoleAfterRun(scoreEntries); } @@ -1139,7 +1088,7 @@ class Benchmark { name = legacyScoreNameMap[name]; console.log(` ${name}:`, uiFriendlyScore(value)); } - if (RAMification) { + if (JetStreamParams.RAMification) { console.log(" Current Footprint:", uiFriendlyNumber(this.currentFootprint)); console.log(" Peak Footprint:", uiFriendlyNumber(this.peakFootprint)); } @@ -2701,8 +2650,8 @@ const defaultDisabledTags = []; if (!isInBrowser) defaultDisabledTags.push("WorkerTests"); -if (globalThis.testList?.length) { - benchmarks = processTestList(globalThis.testList); +if (JetStreamParams.testList.length) { + benchmarks = processTestList(JetStreamParams.testList); } else { benchmarks = findBenchmarksByTag("Default", defaultDisabledTags) } diff --git a/cli.js b/cli.js index 5429ccd..6c0ccb6 100644 --- a/cli.js +++ b/cli.js @@ -21,48 +21,108 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. -*/ + */ -load("./shell-config.js") +load("./shell-config.js"); -const cliFlags = { __proto__: null }; +const CLI_PARAMS = { + __proto__: null, + help: { help: "Print this help message.", param: "help" }, + "iteration-count": { + help: "Set the default iteration count.", + param: "testIterationCount", + }, + "worst-case-count": { + help: "Set the default worst-case count.", + param: "testWorstCaseCount", + }, + "dump-json-results": { + help: "Print summary json to the console.", + param: "dumpJSONResults", + }, + "dump-test-list": { + help: "Print test list instead of running.", + param: "dumpTestList", + }, + ramification: { + help: "Enable ramification support. See RAMification.py for more details.", + param: "RAMification", + }, + "no-prefetch": { + help: "Do not prefetch resources. Will add network overhead to measurements!", + param: "prefetchResources", + }, + test: { + help: "Run a specific test or comma-separated list of tests.", + param: "test", + }, + tag: { + help: "Run tests with a specific tag or comma-separated list of tags.", + param: "tag", + }, + "start-automatically": { + help: "Start the benchmark automatically.", + param: "startAutomatically", + }, + report: { help: "Report results to a server.", param: "shouldReport" }, + "start-delay": { + help: "Delay before starting the benchmark.", + param: "startDelay", + }, + "custom-pre-iteration-code": { + help: "Custom code to run before each iteration.", + param: "customPreIterationCode", + }, + "custom-post-iteration-code": { + help: "Custom code to run after each iteration.", + param: "customPostIterationCode", + }, +}; + +const cliParams = new Map(); const cliArgs = []; + if (globalThis.arguments?.length) { - for (const argument of globalThis.arguments) - if (argument.startsWith("--")) { - const parts = argument.split("="); - cliFlags[parts[0].toLowerCase()] = parts.slice(1).join("="); - } else - cliArgs.push(argument); + for (const argument of globalThis.arguments) { + if (argument.startsWith("--")) { + parseCliFlag(argument); + } else { + cliArgs.push(argument); + } + } } -function getIntFlag(flags, flag) { - if (!(flag in flags)) - return undefined; - const rawValue = flags[flag]; - const value = parseInt(rawValue); - if (value <= 0) - throw new Error(`Expected positive value for ${flag}, but got ${rawValue}`); - return value; +function parseCliFlag(argument) { + const parts = argument.slice(2).split("="); + const flagName = parts[0]; + if (!(flagName in CLI_PARAMS)) { + const message =`Unknown flag: '--${flagName}'`; + help(message); + throw Error(message); + } + let value = parts.slice(1).join("="); + if (flagName === "no-prefetch") value = "false"; + else if (value === "") value = "true"; + cliParams.set(CLI_PARAMS[flagName].param, value); +} + + +if (cliArgs.length) { + let tests = cliParams.has("test") ? cliParams.get("tests").split(",") : [] + tests = tests.concat(cliArgs); + cliParams.set("test", tests.join(",")); } -if ("--iteration-count" in cliFlags) - globalThis.testIterationCount = getIntFlag(cliFlags, "--iteration-count"); -if ("--worst-case-count" in cliFlags) - globalThis.testWorstCaseCount = getIntFlag(cliFlags, "--worst-case-count"); -if ("--dump-json-results" in cliFlags) - globalThis.dumpJSONResults = true; -if (typeof runMode !== "undefined" && runMode == "RAMification") - globalThis.RAMification = true; -if ("--ramification" in cliFlags) - globalThis.RAMification = true; -if ("--no-prefetch" in cliFlags) - globalThis.prefetchResources = false; -if (cliArgs.length) - globalThis.testList = cliArgs; +if (cliParams.size) + globalThis.JetStreamParamsSource = cliParams; + +load("./params.js"); async function runJetStream() { + if (!JetStreamParams.isDefault) { + console.warn(`Using non standard params: ${JSON.stringify(JetStreamParams, null, 2)}`) + } try { await JetStream.initialize(); await JetStream.start(); @@ -75,31 +135,39 @@ async function runJetStream() { load("./JetStreamDriver.js"); -if ("--help" in cliFlags) { - console.log("JetStream Driver Help"); +function help(message=undefined) { + if (message) + console.log(message); + else + console.log("JetStream Driver Help"); console.log(""); console.log("Options:"); - console.log(" --iteration-count: Set the default iteration count."); - console.log(" --worst-case-count: Set the default worst-case count"); - console.log(" --dump-json-results: Print summary json to the console."); - console.log(" --dump-test-list: Print test list instead of running."); - console.log(" --ramification: Enable ramification support. See RAMification.py for more details."); - console.log(" --no-prefetch: Do not prefetch resources. Will add network overhead to measurements!"); + for (const [flag, { help }] of Object.entries(CLI_PARAMS)) + console.log(` --${flag.padEnd(20)} ${help}`); console.log(""); - console.log("Available tags:"); - const tagNames = Array.from(benchmarksByTag.keys()).sort(); - for (const tagName of tagNames) - console.log(" ", tagName); - console.log(""); + // If we had an early bailout during flag parsing we won't have + // JetStreamDriver.js loaded yet and thus none of the helper globals here + // have been defined yet. + if (typeof benchmarksByTag !== "undefined") { + console.log("Available tags:"); + const tagNames = Array.from(benchmarksByTag.keys()).sort(); + for (const tagName of tagNames) console.log(" ", tagName); + console.log(""); + } + + if (typeof BENCHMARKS !== "undefined") { + console.log("Available tests:"); + const benchmarkNames = BENCHMARKS.map((b) => b.name).sort(); + for (const benchmark of benchmarkNames) console.log(" ", benchmark); + } +} - console.log("Available tests:"); - const benchmarkNames = BENCHMARKS.map(b => b.name).sort(); - for (const benchmark of benchmarkNames) - console.log(" ", benchmark); -} else if ("--dump-test-list" in cliFlags) { - JetStream.dumpTestList(); +if (cliParams.has("help")) { + help(); +} else if (cliParams.has("dumpTestList")) { + JetStream.dumpTestList(); } else { - runJetStream(); + runJetStream(); } diff --git a/index.html b/index.html index 98992b5..09bd92b 100644 --- a/index.html +++ b/index.html @@ -36,6 +36,7 @@ const isInBrowser = true; const isD8 = false; const isSpiderMonkey = false; + globalThis.JetStreamParamsSource = new URLSearchParams(globalThis?.location?.search); globalThis.allIsGood = true; window.onerror = function(e) { if (e == "Script error.") { @@ -65,6 +66,7 @@ } + diff --git a/params.js b/params.js new file mode 100644 index 0000000..37c22a6 --- /dev/null +++ b/params.js @@ -0,0 +1,146 @@ +"use strict"; + +/* + * Copyright (C) 2025 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. +*/ + + +class Params { + // Enable a detailed developer menu to change the current Params. + developerMode = false; + startAutomatically = false; + shouldReport = false; + startDelay = undefined; + + testList = []; + testIterationCount = undefined; + testWorstCaseCount = undefined; + prefetchResources = true; + + RAMification = false; + dumpJSONResults = false; + testIterationCountMap = new Map(); + testWorstCaseCountMap = new Map(); + + customPreIterationCode = undefined; + customPostIterationCode = undefined; + + constructor(sourceParams = undefined) { + if (sourceParams) + this._copyFromSearchParams(sourceParams); + if (!this.developerMode) + Object.freeze(this); + } + + _copyFromSearchParams(sourceParams) { + this.startAutomatically = this._parseBooleanParam(sourceParams, "startAutomatically"); + this.developerMode = this._parseBooleanParam(sourceParams, "developerMode"); + this.shouldReport = this._parseBooleanParam(sourceParams, "report"); + this.prefetchResources = this._parseBooleanParam(sourceParams, "prefetchResources"); + this.RAMification = this._parseBooleanParam(sourceParams, "RAMification"); + this.dumpJSONResults = this._parseBooleanParam(sourceParams, "dumpJSONResults"); + + this.customPreIterationCode = this._parseStringParam(sourceParams, "customPreIterationCode"); + this.customPostIterationCode = this._parseStringParam(sourceParams, "customPostIterationCode"); + + this.startDelay = this._parseIntParam(sourceParams, "startDelay", 0); + if (this.shouldReport && !this.startDelay) + this.startDelay = 4000; + + for (const paramKey of ["tag", "tags", "test", "tests"]) + this.testList = this._parseTestListParam(sourceParams, paramKey); + + this.testIterationCount = this._parseIntParam(sourceParams, "iterationCount", 1); + this.testWorstCaseCount = this._parseIntParam(sourceParams, "worstCaseCount", 1); + + const unused = Array.from(sourceParams.keys()); + if (unused.length > 0) + console.error("Got unused source params", unused); + } + + _parseTestListParam(sourceParams, key) { + if (!sourceParams.has(key)) + return this.testList; + let testList = []; + if (sourceParams?.getAll) { + testList = sourceParams?.getAll(key); + } else { + // fallback for cli sourceParams which is just a Map; + testList = sourceParams.get(key).split(","); + } + sourceParams.delete(key); + if (this.testList.length > 0 && testList.length > 0) + throw new Error(`Overriding previous testList='${this.testList.join()}' with ${key} url-parameter.`); + return testList; + } + + _parseStringParam(sourceParams, paramKey) { + if (!sourceParams.has(paramKey)) + return DefaultJetStreamParams[paramKey]; + const value = sourceParams.get(paramKey); + sourceParams.delete(paramKey); + return value; + } + + _parseBooleanParam(sourceParams, paramKey) { + if (!sourceParams.has(paramKey)) + return DefaultJetStreamParams[paramKey]; + const value = sourceParams.get(paramKey).toLowerCase(); + sourceParams.delete(paramKey); + return !(value === "false" || value === "0"); + } + + _parseIntParam(sourceParams, paramKey, minValue) { + if (!sourceParams.has(paramKey)) + return DefaultJetStreamParams[paramKey]; + + const parsedValue = this._parseInt(sourceParams.get(paramKey), paramKey); + if (parsedValue < minValue) + throw new Error(`Invalid ${paramKey} param: '${parsedValue}', value must be >= ${minValue}.`); + sourceParams.delete(paramKey); + return parsedValue; + } + + _parseInt(value, errorMessage) { + const number = Number(value); + if (!Number.isInteger(number) && errorMessage) + throw new Error(`Invalid ${errorMessage} param: '${value}', expected int.`); + return parseInt(number); + } + + get isDefault() { + return this === DefaultJetStreamParams; + } +} + +const DefaultJetStreamParams = new Params(); +let maybeCustomParams = DefaultJetStreamParams; +if (globalThis?.JetStreamParamsSource) { + try { + maybeCustomParams = new Params(globalThis?.JetStreamParamsSource); + } catch (e) { + console.error("Invalid Params", e, "\nUsing defaults as fallback:", maybeCustomParams); + } +} +const JetStreamParams = maybeCustomParams; diff --git a/tests/unit-tests.js b/tests/unit-tests.js index 63a1efe..507be43 100644 --- a/tests/unit-tests.js +++ b/tests/unit-tests.js @@ -1,4 +1,5 @@ load("shell-config.js"); +load("params.js"); load("startup-helper/StartupBenchmark.js"); load("JetStreamDriver.js");