From 1b15d82382040ebe9ad058316e25d16f8b3dffdf Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 10:19:22 +0100 Subject: [PATCH 1/8] use ubuntu for shell tests --- .github/workflows/test.yml | 40 ++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad3b23c8..c8fe5399 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,15 +10,15 @@ on: workflow_dispatch: jobs: - test: - name: Test + test-browser: + name: Test Browser runs-on: macos-latest env: GITHUB_ACTIONS_OUTPUT: "" strategy: fail-fast: false matrix: - browser: [chrome, firefox, jsc, safari, spidermonkey, v8] + browser: [chrome, firefox, safari] steps: - name: Extract Week Number run: echo "WEEK_NUMBER=$(date +%W)" >> $GITHUB_ENV @@ -39,17 +39,45 @@ jobs: - name: Install Node Packages run: npm ci + - name: Run Tests + run: | + echo "Running in $BROWSER" + npm run test:${{ matrix.browser }} + test-shell: + name: Test + runs-on: ubuntu-latest + env: + GITHUB_ACTIONS_OUTPUT: "" + strategy: + fail-fast: false + matrix: + shell: [jsc, spidermonkey, v8] + steps: + - name: Extract Week Number + run: echo "WEEK_NUMBER=$(date +%W)" >> $GITHUB_ENV + + - name: Checkout Branch + uses: actions/checkout@v5 + + - name: Setup Node + uses: actions/setup-node@v5 + with: + node-version-file: package.json + cache: npm + + - name: Install Node Packages + run: npm ci + - name: Cache jsvu Binaries uses: actions/cache@v4 with: path: ~/.jsvu - key: ${{ runner.os }}-jsvu-${{ matrix.browser }}-week-${{ env.WEEK_NUMBER }} + key: ${{ runner.os }}-jsvu-${{ matrix.shell }}-week-${{ env.WEEK_NUMBER }} - name: Run Tests run: | echo "Running in $BROWSER" - npm run test:${{ matrix.browser }} - + npm run test:${{ matrix.shell }} build: name: Build runs-on: ubuntu-latest From b00bb966bc1ec1de6f9fe98868db0cb6a86a5c88 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 10:43:54 +0100 Subject: [PATCH 2/8] wip shell --- .github/workflows/test.yml | 11 ++++++----- tests/run-shell.mjs | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c8fe5399..e3061716 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: jobs: - test-browser: + browser: name: Test Browser runs-on: macos-latest env: @@ -43,8 +43,8 @@ jobs: run: | echo "Running in $BROWSER" npm run test:${{ matrix.browser }} - test-shell: - name: Test + shell: + name: Test Shell runs-on: ubuntu-latest env: GITHUB_ACTIONS_OUTPUT: "" @@ -52,6 +52,7 @@ jobs: fail-fast: false matrix: shell: [jsc, spidermonkey, v8] + suite: [default, disabled, main] steps: - name: Extract Week Number run: echo "WEEK_NUMBER=$(date +%W)" >> $GITHUB_ENV @@ -77,9 +78,9 @@ jobs: - name: Run Tests run: | echo "Running in $BROWSER" - npm run test:${{ matrix.shell }} + npm run test:${{ matrix.shell }} -- --suite ${{ matrix.suite }} build: - name: Build + name: Test Build runs-on: ubuntu-latest steps: - name: Checkout Branch diff --git a/tests/run-shell.mjs b/tests/run-shell.mjs index 80a62248..b16b861c 100644 --- a/tests/run-shell.mjs +++ b/tests/run-shell.mjs @@ -33,6 +33,7 @@ import { logGroup, logInfo, printHelp, runTest, sh } from "./helper.mjs"; const optionDefinitions = [ { name: "shell", type: String, description: "Set the shell to test, choices are [jsc, v8, spidermonkey]." }, + { name: "suite", type: String, description: "Run a specific suite." }, { name: "help", alias: "h", description: "Print this help text." }, ]; @@ -78,12 +79,17 @@ function convertCliArgs(cli, ...cliArgs) { async function runTests() { const shellBinary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); let success = true; - success &&= await runTest("Run UnitTests", () => sh(shellBinary, UNIT_TEST_PATH)); - success &&= await runCLITest("Run Single Suite", shellBinary, "proxy-mobx"); - success &&= await runCLITest("Run Tag No Prefetch", shellBinary, "proxy", "argon2-wasm", "--no-prefetch"); - success &&= await runCLITest("Run Grouped with Details", shellBinary, "SunSpider", "--group-details"); - success &&= await runCLITest("Run Disabled Suite", shellBinary, "disabled"); - success &&= await runCLITest("Run Default Suite", shellBinary); + + if (options.suite === "disabled") { + success &&= await runCLITest("Run Disabled Suite", shellBinary, "disabled"); + } else if (options.suite === "default") { + success &&= await runCLITest("Run Default Suite", shellBinary); + } else if (options.suite === "main" || !options.suite) { + success &&= await runTest("Run UnitTests", () => sh(shellBinary, UNIT_TEST_PATH)); + success &&= await runCLITest("Run Single Suite", shellBinary, "proxy-mobx"); + success &&= await runCLITest("Run Tag No Prefetch", shellBinary, "proxy", "argon2-wasm", "--no-prefetch"); + success &&= await runCLITest("Run Grouped with Details", shellBinary, "SunSpider", "--group-details"); + } if (!success) process.exit(1); } From 817362c807ec4c38d61a965b1b9eb0034c42b9b6 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 13:52:30 +0100 Subject: [PATCH 3/8] cleanup --- .github/workflows/test.yml | 2 +- tests/helper.mjs | 4 +- tests/run-shell.mjs | 83 +++++++++++++++++++++++++++++--------- 3 files changed, 66 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3061716..1557d6da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: - name: Run Tests run: | echo "Running in $BROWSER" - npm run test:${{ matrix.shell }} -- --suite ${{ matrix.suite }} + npm run test:${{ matrix.shell }} -- ${{ matrix.suite }} build: name: Test Build runs-on: ubuntu-latest diff --git a/tests/helper.mjs b/tests/helper.mjs index dff7497f..bff320f1 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -96,8 +96,8 @@ export function printHelp(message = "", optionDefinitions) { process.exit(0); } else { console.error(message); - console.error(); - console.error(usage); + console.log(); + console.log(usage); process.exit(1); } } diff --git a/tests/run-shell.mjs b/tests/run-shell.mjs index b16b861c..6df78aab 100644 --- a/tests/run-shell.mjs +++ b/tests/run-shell.mjs @@ -31,20 +31,64 @@ import { fileURLToPath } from "url"; import { logGroup, logInfo, printHelp, runTest, sh } from "./helper.mjs"; +const FILE_PATH = fileURLToPath(import.meta.url); +const SRC_DIR = path.dirname(path.dirname(FILE_PATH)); +const CLI_PATH = path.join(SRC_DIR, "cli.js"); +const UNIT_TEST_PATH = path.join(SRC_DIR, "tests", "unit-tests.js"); +let SHELL_BINARY = ""; + +const TESTS = [ + { + name: "UnitTests", + tags: ["main"], + run: () => runTest("UnitTests", () => sh(SHELL_BINARY, UNIT_TEST_PATH)) + }, + { + name: "Single Suite", + tags: ["main"], + run: () => runCLITest("Single Suite", SHELL_BINARY, "proxy-mobx") + }, + { + name: "Tag No Prefetch", + tags: ["main"], + run: () => runCLITest("Tag No Prefetch", SHELL_BINARY, "proxy", "argon2-wasm", "--no-prefetch") + }, + { + name: "Grouped with Details", + tags: ["main"], + run: () => runCLITest("Grouped with Details", SHELL_BINARY, "SunSpider", "--group-details") + }, + { + name: "Disabled Suite", + tags: ["disabled"], + run: () => runCLITest("Disabled Suite", SHELL_BINARY, "disabled") + }, + { + name: "Default Suite", + tags: ["default"], + run: () => runCLITest("Default Suite", SHELL_BINARY) + } +]; + +const VALID_TAGS = Array.from(new Set(TESTS.map(each => each.tags).flat())); + const optionDefinitions = [ { name: "shell", type: String, description: "Set the shell to test, choices are [jsc, v8, spidermonkey]." }, - { name: "suite", type: String, description: "Run a specific suite." }, { name: "help", alias: "h", description: "Print this help text." }, + { name: "suite", type: String, defaultOption: true, typeLabel: `choices: ${VALID_TAGS.join(", ")}`, description: "Run a specific suite by name." } ]; const options = commandLineArgs(optionDefinitions); if ("help" in options) - printHelp(optionDefinitions); + printHelp("", optionDefinitions); + +if (options.suite && !VALID_TAGS.includes(options.suite)) + printHelp(`Invalid suite: ${options.suite}. Choices are: ${VALID_TAGS.join(", ")}`, optionDefinitions); const JS_SHELL= options?.shell; if (!JS_SHELL) - printHelp("No javascript shell specified, use --shell", optionDefinitions); + printHelp("No javascript shell specified, use --shell", optionDefinitions); const SHELL_NAME = (function() { switch (JS_SHELL) { @@ -59,15 +103,11 @@ const SHELL_NAME = (function() { return "v8"; } default: { - printHelp(`Invalid shell "${JS_SHELL}", choices are: "jsc", "spidermonkey" and "v8)`); + printHelp(`Invalid shell "${JS_SHELL}", choices are: "jsc", "spidermonkey" and "v8)`, optionDefinitions); } } })(); -const FILE_PATH = fileURLToPath(import.meta.url); -const SRC_DIR = path.dirname(path.dirname(FILE_PATH)); -const CLI_PATH = path.join(SRC_DIR, "cli.js"); -const UNIT_TEST_PATH = path.join(SRC_DIR, "tests", "unit-tests.js"); function convertCliArgs(cli, ...cliArgs) { if (SHELL_NAME == "spidermonkey") @@ -76,22 +116,25 @@ function convertCliArgs(cli, ...cliArgs) { } + async function runTests() { - const shellBinary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); + SHELL_BINARY = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); + const suiteFilter = options.suite || "main"; let success = true; + const testsToRun = TESTS.filter(test => test.tags.includes(suiteFilter)); + + if (testsToRun.length === 0) { + console.error(`No suite found for filter: ${suiteFilter}`); + process.exit(1); + } + + for (const test of testsToRun) { + success &&= await test.run(); + } - if (options.suite === "disabled") { - success &&= await runCLITest("Run Disabled Suite", shellBinary, "disabled"); - } else if (options.suite === "default") { - success &&= await runCLITest("Run Default Suite", shellBinary); - } else if (options.suite === "main" || !options.suite) { - success &&= await runTest("Run UnitTests", () => sh(shellBinary, UNIT_TEST_PATH)); - success &&= await runCLITest("Run Single Suite", shellBinary, "proxy-mobx"); - success &&= await runCLITest("Run Tag No Prefetch", shellBinary, "proxy", "argon2-wasm", "--no-prefetch"); - success &&= await runCLITest("Run Grouped with Details", shellBinary, "SunSpider", "--group-details"); + if (!success) { + process.exit(1); } - if (!success) - process.exit(1); } function jsvuOSName() { From eb0fa1d3cf6c0b9227fea5a1119ca4cec403c3ec Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 14:03:23 +0100 Subject: [PATCH 4/8] more refactoring --- tests/helper.mjs | 4 +- tests/run-browser.mjs | 65 ++++++++++++++-- tests/run-shell.mjs | 171 ++++++++++++++++++++++++------------------ 3 files changed, 159 insertions(+), 81 deletions(-) diff --git a/tests/helper.mjs b/tests/helper.mjs index bff320f1..eb2b6b23 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -81,7 +81,7 @@ export async function logGroup(name, body) { } -export function printHelp(message = "", optionDefinitions) { +export function printHelp(message, optionDefinitions) { const usage = commandLineUsage([ { header: "Run all tests", @@ -91,7 +91,7 @@ export function printHelp(message = "", optionDefinitions) { optionList: optionDefinitions, }, ]); - if (!message) { + if (!message || !message.length) { console.log(usage); process.exit(0); } else { diff --git a/tests/run-browser.mjs b/tests/run-browser.mjs index 0a89efb0..f015cd88 100644 --- a/tests/run-browser.mjs +++ b/tests/run-browser.mjs @@ -33,16 +33,60 @@ import os from "os"; import {logInfo, logError, printHelp, runTest} from "./helper.mjs"; +const TESTS = [ + { + name: "Run Single Suite", + tags: ["main"], + run() { + return runEnd2EndTest("Run Single Suite", { test: "proxy-mobx" }); + } + }, + { + name: "Run Multiple Suites", + tags: ["main"], + run() { + runEnd2EndTest("Run Multiple Suites", { test: "prismjs-startup-es6,postcss-wtb" }); + } + }, + { + name: "Run Tag No Prefetch", + tags: ["main"], + run() { + return runEnd2EndTest("Run Tag No Prefetch", { tag: "proxy", prefetchResources: "false" }); + } + }, + { + name: "Run Disabled Suite", + tags: ["disabled"], + run() { + return runEnd2EndTest("Run Disabled Suite", { tag: "disabled" }); + } + }, + { + name: "Run Default Suite", + tags: ["default"], + run() { + return runEnd2EndTest("Run Default Suite"); + } + } +]; + +const VALID_TAGS = Array.from(new Set(TESTS.map((each) => each.tags).flat())); + const optionDefinitions = [ { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome, edge]. By default the $BROWSER env variable is used." }, { name: "port", type: Number, defaultValue: 8010, description: "Set the test-server port, The default value is 8010." }, { name: "help", alias: "h", description: "Print this help text." }, + { name: "suite", type: String, defaultOption: true, typeLabel: `{underline choices}: ${VALID_TAGS.join(", ")}`, description: "Run a specific suite by name." } ]; const options = commandLineArgs(optionDefinitions); if ("help" in options) - printHelp(optionDefinitions); + printHelp("". optionDefinitions); + +if (options.suite && !VALID_TAGS.includes(options.suite)) + printHelp(`Invalid suite: ${options.suite}. Choices are: ${VALID_TAGS.join(", ")}`); const BROWSER = options?.browser; if (!BROWSER) @@ -68,7 +112,7 @@ switch (BROWSER) { break; } default: { - printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`); + printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`, optionDefinitions); } } @@ -86,12 +130,19 @@ const server = await serve(PORT); async function runTests() { let success = true; + const suiteFilter = options.suite || "main"; + + const testsToRun = TESTS.filter(test => test.tags.includes(suiteFilter)); + + if (testsToRun.length === 0) { + console.error(`No suite found for filter: ${suiteFilter}`); + process.exit(1); + } + try { - success &&= await runEnd2EndTest("Run Single Suite", { test: "proxy-mobx" }); - success &&= await runEnd2EndTest("Run Multiple Suites", { test: "prismjs-startup-es6,postcss-wtb" }); - success &&= await runEnd2EndTest("Run Tag No Prefetch", { tag: "proxy", prefetchResources: "false" }); - success &&= await runEnd2EndTest("Run Disabled Suite", { tag: "disabled" }); - success &&= await runEnd2EndTest("Run Default Suite"); + for (const test of testsToRun) { + success &&= await test.run(); + } } finally { server.close(); } diff --git a/tests/run-shell.mjs b/tests/run-shell.mjs index 6df78aab..20e534ea 100644 --- a/tests/run-shell.mjs +++ b/tests/run-shell.mjs @@ -35,93 +35,114 @@ const FILE_PATH = fileURLToPath(import.meta.url); const SRC_DIR = path.dirname(path.dirname(FILE_PATH)); const CLI_PATH = path.join(SRC_DIR, "cli.js"); const UNIT_TEST_PATH = path.join(SRC_DIR, "tests", "unit-tests.js"); -let SHELL_BINARY = ""; const TESTS = [ { name: "UnitTests", - tags: ["main"], - run: () => runTest("UnitTests", () => sh(SHELL_BINARY, UNIT_TEST_PATH)) + tags: ["main", "unit"], + run(shell_binary) { + return runTest("UnitTests", () => sh(shell_binary, UNIT_TEST_PATH)); + }, }, { name: "Single Suite", - tags: ["main"], - run: () => runCLITest("Single Suite", SHELL_BINARY, "proxy-mobx") + tags: ["main", "single"], + run(shell_binary) { + return runCLITest("Single Suite", shell_binary, "proxy-mobx"); + }, }, { name: "Tag No Prefetch", - tags: ["main"], - run: () => runCLITest("Tag No Prefetch", SHELL_BINARY, "proxy", "argon2-wasm", "--no-prefetch") + tags: ["main", "no-prefetch"], + run(shell_binary) { + return runCLITest("Tag No Prefetch", shell_binary, "proxy", "argon2-wasm", "--no-prefetch"); + }, }, { name: "Grouped with Details", - tags: ["main"], - run: () => runCLITest("Grouped with Details", SHELL_BINARY, "SunSpider", "--group-details") + tags: ["main", "group-details"], + run(shell_binary) { + return runCLITest("Grouped with Details", shell_binary, "SunSpider", "--group-details"); + }, }, { name: "Disabled Suite", tags: ["disabled"], - run: () => runCLITest("Disabled Suite", SHELL_BINARY, "disabled") + run(shell_binary) { + return runCLITest("Disabled Suite", shell_binary, "disabled"); + }, }, { name: "Default Suite", tags: ["default"], - run: () => runCLITest("Default Suite", SHELL_BINARY) - } + run(shell_binary) { + return runCLITest("Default Suite", shell_binary); + }, + }, ]; -const VALID_TAGS = Array.from(new Set(TESTS.map(each => each.tags).flat())); +const VALID_TAGS = Array.from(new Set(TESTS.map((each) => each.tags).flat())); const optionDefinitions = [ - { name: "shell", type: String, description: "Set the shell to test, choices are [jsc, v8, spidermonkey]." }, - { name: "help", alias: "h", description: "Print this help text." }, - { name: "suite", type: String, defaultOption: true, typeLabel: `choices: ${VALID_TAGS.join(", ")}`, description: "Run a specific suite by name." } + { + name: "shell", + type: String, + description: "Set the shell to test, choices are [jsc, v8, spidermonkey].", + }, + { name: "help", alias: "h", description: "Print this help text." }, + { + name: "suite", + type: String, + defaultOption: true, + typeLabel: `choices: ${VALID_TAGS.join(", ")}`, + description: "Run a specific suite by name.", + }, ]; const options = commandLineArgs(optionDefinitions); -if ("help" in options) - printHelp("", optionDefinitions); +if ("help" in options) printHelp("", optionDefinitions); if (options.suite && !VALID_TAGS.includes(options.suite)) - printHelp(`Invalid suite: ${options.suite}. Choices are: ${VALID_TAGS.join(", ")}`, optionDefinitions); - -const JS_SHELL= options?.shell; -if (!JS_SHELL) - printHelp("No javascript shell specified, use --shell", optionDefinitions); - -const SHELL_NAME = (function() { - switch (JS_SHELL) { - case "javascriptcore": - case "jsc": { - return "javascriptcore"; + printHelp( + `Invalid suite: ${options.suite}. Choices are: ${VALID_TAGS.join(", ")}`, + optionDefinitions + ); + +const JS_SHELL = options?.shell; +if (!JS_SHELL) printHelp("No javascript shell specified, use --shell", optionDefinitions); + +const SHELL_NAME = (function () { + switch (JS_SHELL) { + case "javascriptcore": + case "jsc": { + return "javascriptcore"; + } + case "spidermonkey": { + return "spidermonkey"; + } + case "v8": { + return "v8"; + } + default: { + printHelp( + `Invalid shell "${JS_SHELL}", choices are: "jsc", "spidermonkey" and "v8)`, + optionDefinitions + ); + } } - case "spidermonkey": { - return "spidermonkey"; - } - case "v8": { - return "v8"; - } - default: { - printHelp(`Invalid shell "${JS_SHELL}", choices are: "jsc", "spidermonkey" and "v8)`, optionDefinitions); - } - } })(); - function convertCliArgs(cli, ...cliArgs) { - if (SHELL_NAME == "spidermonkey") - return [cli, ...cliArgs]; - return [cli, "--", ...cliArgs]; + if (SHELL_NAME == "spidermonkey") return [cli, ...cliArgs]; + return [cli, "--", ...cliArgs]; } - - async function runTests() { - SHELL_BINARY = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); + const shell_binary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); const suiteFilter = options.suite || "main"; let success = true; - const testsToRun = TESTS.filter(test => test.tags.includes(suiteFilter)); + const testsToRun = TESTS.filter((test) => test.tags.includes(suiteFilter)); if (testsToRun.length === 0) { console.error(`No suite found for filter: ${suiteFilter}`); @@ -129,7 +150,7 @@ async function runTests() { } for (const test of testsToRun) { - success &&= await test.run(); + success &&= await test.run(shell_binary); } if (!success) { @@ -138,45 +159,51 @@ async function runTests() { } function jsvuOSName() { - const osName = () => { - switch (os.platform()) { - case "win32": return "win"; - case "darwin": return "mac"; - case "linux": return "linux"; - default: throw new Error("Unsupported OS"); - } - }; - const osArch = () => { - switch (os.arch()) { - case "x64": return "64"; - case "arm64": return "64arm"; - default: throw new Error("Unsupported architecture"); - } - }; - return `${osName()}${osArch()}`; + const osName = () => { + switch (os.platform()) { + case "win32": + return "win"; + case "darwin": + return "mac"; + case "linux": + return "linux"; + default: + throw new Error("Unsupported OS"); + } + }; + const osArch = () => { + switch (os.arch()) { + case "x64": + return "64"; + case "arm64": + return "64arm"; + default: + throw new Error("Unsupported architecture"); + } + }; + return `${osName()}${osArch()}`; } -const DEFAULT_JSC_LOCATION = "/System/Library/Frameworks/JavaScriptCore.framework/Versions/Current/Helpers/jsc" +const DEFAULT_JSC_LOCATION = + "/System/Library/Frameworks/JavaScriptCore.framework/Versions/Current/Helpers/jsc"; async function testSetup() { await sh("jsvu", `--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`); let shellBinary = path.join(os.homedir(), ".jsvu/bin", SHELL_NAME); if (!fs.existsSync(shellBinary) && SHELL_NAME == "javascriptcore") - shellBinary = DEFAULT_JSC_LOCATION; - if (!fs.existsSync(shellBinary)) - throw new Error(`Could not find shell binary: ${shellBinary}`); + shellBinary = DEFAULT_JSC_LOCATION; + if (!fs.existsSync(shellBinary)) throw new Error(`Could not find shell binary: ${shellBinary}`); logInfo(`Installed JavaScript Shell: ${shellBinary}`); return shellBinary; } function runCLITest(name, shellBinary, ...args) { - return runTest(name, () => runShell(shellBinary, ...convertCliArgs(CLI_PATH, ...args))); + return runTest(name, () => runShell(shellBinary, ...convertCliArgs(CLI_PATH, ...args))); } async function runShell(shellBinary, ...args) { - const result = await sh(shellBinary, ...args); - if (result.stdoutString.includes("JetStream3 failed")) - throw new Error("test failed") + const result = await sh(shellBinary, ...args); + if (result.stdoutString.includes("JetStream3 failed")) throw new Error("test failed"); } setImmediate(runTests); From d5faa2a2945c0339c5db6b0338e26bd7bc42413d Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 14:03:56 +0100 Subject: [PATCH 5/8] run suite --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1557d6da..12e5f35a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,7 @@ jobs: fail-fast: false matrix: browser: [chrome, firefox, safari] + suite: [default, disabled, main] steps: - name: Extract Week Number run: echo "WEEK_NUMBER=$(date +%W)" >> $GITHUB_ENV @@ -42,7 +43,7 @@ jobs: - name: Run Tests run: | echo "Running in $BROWSER" - npm run test:${{ matrix.browser }} + npm run test:${{ matrix.browser }} -- ${{ matrix.suite }} shell: name: Test Shell runs-on: ubuntu-latest From a29305caa6d8b5cecce4606f5e95dd39889fbf0b Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 27 Oct 2025 14:15:20 +0100 Subject: [PATCH 6/8] fixes --- tests/helper.mjs | 2 +- tests/run-browser.mjs | 17 ++++++++++------- tests/run-shell.mjs | 41 ++++++++++++++++++++++++++--------------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/tests/helper.mjs b/tests/helper.mjs index eb2b6b23..e8287a06 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -91,7 +91,7 @@ export function printHelp(message, optionDefinitions) { optionList: optionDefinitions, }, ]); - if (!message || !message.length) { + if (!message?.length) { console.log(usage); process.exit(0); } else { diff --git a/tests/run-browser.mjs b/tests/run-browser.mjs index f015cd88..bbb4bace 100644 --- a/tests/run-browser.mjs +++ b/tests/run-browser.mjs @@ -36,35 +36,35 @@ import {logInfo, logError, printHelp, runTest} from "./helper.mjs"; const TESTS = [ { name: "Run Single Suite", - tags: ["main"], + tags: ["all", "main"], run() { return runEnd2EndTest("Run Single Suite", { test: "proxy-mobx" }); } }, { name: "Run Multiple Suites", - tags: ["main"], + tags: ["all", "main"], run() { - runEnd2EndTest("Run Multiple Suites", { test: "prismjs-startup-es6,postcss-wtb" }); + return runEnd2EndTest("Run Multiple Suites", { test: "prismjs-startup-es6,postcss-wtb" }); } }, { name: "Run Tag No Prefetch", - tags: ["main"], + tags: ["all", "all", "main"], run() { return runEnd2EndTest("Run Tag No Prefetch", { tag: "proxy", prefetchResources: "false" }); } }, { name: "Run Disabled Suite", - tags: ["disabled"], + tags: ["all", "disabled"], run() { return runEnd2EndTest("Run Disabled Suite", { tag: "disabled" }); } }, { name: "Run Default Suite", - tags: ["default"], + tags: ["all", "default"], run() { return runEnd2EndTest("Run Default Suite"); } @@ -130,10 +130,13 @@ const server = await serve(PORT); async function runTests() { let success = true; - const suiteFilter = options.suite || "main"; + const suiteFilter = options.suite || "all"; const testsToRun = TESTS.filter(test => test.tags.includes(suiteFilter)); + console.log(testsToRun) + console.log(testsToRun.length) + console.log(suiteFilter) if (testsToRun.length === 0) { console.error(`No suite found for filter: ${suiteFilter}`); process.exit(1); diff --git a/tests/run-shell.mjs b/tests/run-shell.mjs index 20e534ea..1a3a0e6b 100644 --- a/tests/run-shell.mjs +++ b/tests/run-shell.mjs @@ -39,44 +39,50 @@ const UNIT_TEST_PATH = path.join(SRC_DIR, "tests", "unit-tests.js"); const TESTS = [ { name: "UnitTests", - tags: ["main", "unit"], + tags: ["all", "main", "unit"], run(shell_binary) { - return runTest("UnitTests", () => sh(shell_binary, UNIT_TEST_PATH)); + return runTest("UnitTests", () => sh(shell_binary, UNIT_TEST_PATH)); }, }, { name: "Single Suite", - tags: ["main", "single"], + tags: ["all", "main", "single"], run(shell_binary) { - return runCLITest("Single Suite", shell_binary, "proxy-mobx"); + return runCLITest("Single Suite", shell_binary, "proxy-mobx"); }, }, { name: "Tag No Prefetch", - tags: ["main", "no-prefetch"], + tags: ["all", "main", "no-prefetch"], run(shell_binary) { - return runCLITest("Tag No Prefetch", shell_binary, "proxy", "argon2-wasm", "--no-prefetch"); + return runCLITest( + "Tag No Prefetch", + shell_binary, + "proxy", + "argon2-wasm", + "--no-prefetch" + ); }, }, { name: "Grouped with Details", - tags: ["main", "group-details"], + tags: ["all", "main", "group-details"], run(shell_binary) { return runCLITest("Grouped with Details", shell_binary, "SunSpider", "--group-details"); }, }, { name: "Disabled Suite", - tags: ["disabled"], + tags: ["all", "disabled"], run(shell_binary) { - return runCLITest("Disabled Suite", shell_binary, "disabled"); + return runCLITest("Disabled Suite", shell_binary, "disabled"); }, }, { name: "Default Suite", - tags: ["default"], + tags: ["all", "default"], run(shell_binary) { - return runCLITest("Default Suite", shell_binary); + return runCLITest("Default Suite", shell_binary); }, }, ]; @@ -101,16 +107,21 @@ const optionDefinitions = [ const options = commandLineArgs(optionDefinitions); -if ("help" in options) printHelp("", optionDefinitions); +if ("help" in options) { + printHelp("", optionDefinitions); +} -if (options.suite && !VALID_TAGS.includes(options.suite)) +if (options.suite && !VALID_TAGS.includes(options.suite)) { printHelp( `Invalid suite: ${options.suite}. Choices are: ${VALID_TAGS.join(", ")}`, optionDefinitions ); +} const JS_SHELL = options?.shell; -if (!JS_SHELL) printHelp("No javascript shell specified, use --shell", optionDefinitions); +if (!JS_SHELL) { + printHelp("No javascript shell specified, use --shell", optionDefinitions); +} const SHELL_NAME = (function () { switch (JS_SHELL) { @@ -140,7 +151,7 @@ function convertCliArgs(cli, ...cliArgs) { async function runTests() { const shell_binary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); - const suiteFilter = options.suite || "main"; + const suiteFilter = options.suite || "all"; let success = true; const testsToRun = TESTS.filter((test) => test.tags.includes(suiteFilter)); From 1aecdae091c5f85c41494e120d7a9384f17d438b Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Thu, 30 Oct 2025 14:58:06 +0100 Subject: [PATCH 7/8] address comments --- tests/run-browser.mjs | 5 +---- tests/run-shell.mjs | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/run-browser.mjs b/tests/run-browser.mjs index 9f5cbc2a..92caadb0 100644 --- a/tests/run-browser.mjs +++ b/tests/run-browser.mjs @@ -50,7 +50,7 @@ const TESTS = [ }, { name: "Run Tag No Prefetch", - tags: ["all", "all", "main"], + tags: ["all", "main"], run() { return runEnd2EndTest("Run Tag No Prefetch", { tag: "proxy", prefetchResources: "false" }); } @@ -138,9 +138,6 @@ async function runTests() { const testsToRun = TESTS.filter(test => test.tags.includes(suiteFilter)); - console.log(testsToRun) - console.log(testsToRun.length) - console.log(suiteFilter) if (testsToRun.length === 0) { console.error(`No suite found for filter: ${suiteFilter}`); process.exit(1); diff --git a/tests/run-shell.mjs b/tests/run-shell.mjs index 1a3a0e6b..a91a55b4 100644 --- a/tests/run-shell.mjs +++ b/tests/run-shell.mjs @@ -214,7 +214,10 @@ function runCLITest(name, shellBinary, ...args) { async function runShell(shellBinary, ...args) { const result = await sh(shellBinary, ...args); - if (result.stdoutString.includes("JetStream3 failed")) throw new Error("test failed"); + // JSC does not set a non-0 exit status on async exceptions. + if (SHELL_NAME == "javascriptcore" && "JetStream3 failed") { + throw new Error("test failed"); + } } setImmediate(runTests); From d1236f1595499df42ee2558d81a651ab66bbfe25 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Thu, 30 Oct 2025 15:01:18 +0100 Subject: [PATCH 8/8] fix check --- tests/run-shell.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run-shell.mjs b/tests/run-shell.mjs index a91a55b4..7944dce0 100644 --- a/tests/run-shell.mjs +++ b/tests/run-shell.mjs @@ -215,7 +215,7 @@ function runCLITest(name, shellBinary, ...args) { async function runShell(shellBinary, ...args) { const result = await sh(shellBinary, ...args); // JSC does not set a non-0 exit status on async exceptions. - if (SHELL_NAME == "javascriptcore" && "JetStream3 failed") { + if (SHELL_NAME == "javascriptcore" && result.stdoutString.includes("JetStream3 failed")) { throw new Error("test failed"); } }