diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad3b23c8..12e5f35a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,15 +10,16 @@ on: workflow_dispatch: jobs: - test: - name: Test + browser: + name: Test Browser runs-on: macos-latest env: GITHUB_ACTIONS_OUTPUT: "" strategy: fail-fast: false matrix: - browser: [chrome, firefox, jsc, safari, spidermonkey, v8] + browser: [chrome, firefox, safari] + suite: [default, disabled, main] steps: - name: Extract Week Number run: echo "WEEK_NUMBER=$(date +%W)" >> $GITHUB_ENV @@ -39,19 +40,48 @@ jobs: - name: Install Node Packages run: npm ci + - name: Run Tests + run: | + echo "Running in $BROWSER" + npm run test:${{ matrix.browser }} -- ${{ matrix.suite }} + shell: + name: Test Shell + runs-on: ubuntu-latest + env: + GITHUB_ACTIONS_OUTPUT: "" + strategy: + fail-fast: false + matrix: + shell: [jsc, spidermonkey, v8] + suite: [default, disabled, main] + steps: + - name: Extract Week Number + run: echo "WEEK_NUMBER=$(date +%W)" >> $GITHUB_ENV + + - name: Checkout Branch + uses: actions/checkout@v5 + + - name: Setup Node + uses: actions/setup-node@v5 + with: + node-version-file: package.json + cache: npm + + - name: Install Node Packages + run: npm ci + - name: Cache jsvu Binaries uses: actions/cache@v4 with: path: ~/.jsvu - key: ${{ runner.os }}-jsvu-${{ matrix.browser }}-week-${{ env.WEEK_NUMBER }} + key: ${{ runner.os }}-jsvu-${{ matrix.shell }}-week-${{ env.WEEK_NUMBER }} - name: Run Tests run: | echo "Running in $BROWSER" - npm run test:${{ matrix.browser }} - + npm run test:${{ matrix.shell }} -- ${{ matrix.suite }} build: - name: Build + name: Test Build runs-on: ubuntu-latest steps: - name: Checkout Branch diff --git a/tests/helper.mjs b/tests/helper.mjs index dff7497f..e8287a06 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -81,7 +81,7 @@ export async function logGroup(name, body) { } -export function printHelp(message = "", optionDefinitions) { +export function printHelp(message, optionDefinitions) { const usage = commandLineUsage([ { header: "Run all tests", @@ -91,13 +91,13 @@ export function printHelp(message = "", optionDefinitions) { optionList: optionDefinitions, }, ]); - if (!message) { + if (!message?.length) { console.log(usage); process.exit(0); } else { console.error(message); - console.error(); - console.error(usage); + console.log(); + console.log(usage); process.exit(1); } } diff --git a/tests/run-browser.mjs b/tests/run-browser.mjs index 05c8be99..92caadb0 100644 --- a/tests/run-browser.mjs +++ b/tests/run-browser.mjs @@ -33,6 +33,45 @@ import os from "os"; import {logInfo, logError, printHelp, runTest} from "./helper.mjs"; +const TESTS = [ + { + name: "Run Single Suite", + tags: ["all", "main"], + run() { + return runEnd2EndTest("Run Single Suite", { test: "proxy-mobx" }); + } + }, + { + name: "Run Multiple Suites", + tags: ["all", "main"], + run() { + return runEnd2EndTest("Run Multiple Suites", { test: "prismjs-startup-es6,postcss-wtb" }); + } + }, + { + name: "Run Tag No Prefetch", + tags: ["all", "main"], + run() { + return runEnd2EndTest("Run Tag No Prefetch", { tag: "proxy", prefetchResources: "false" }); + } + }, + { + name: "Run Disabled Suite", + tags: ["all", "disabled"], + run() { + return runEnd2EndTest("Run Disabled Suite", { tag: "disabled" }); + } + }, + { + name: "Run Default Suite", + tags: ["all", "default"], + run() { + return runEnd2EndTest("Run Default Suite"); + } + } +]; + +const VALID_TAGS = Array.from(new Set(TESTS.map((each) => each.tags).flat())); function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); @@ -42,12 +81,16 @@ const optionDefinitions = [ { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome, edge]. By default the $BROWSER env variable is used." }, { name: "port", type: Number, defaultValue: 8010, description: "Set the test-server port, The default value is 8010." }, { name: "help", alias: "h", description: "Print this help text." }, + { name: "suite", type: String, defaultOption: true, typeLabel: `{underline choices}: ${VALID_TAGS.join(", ")}`, description: "Run a specific suite by name." } ]; const options = commandLineArgs(optionDefinitions); if ("help" in options) - printHelp(optionDefinitions); + printHelp("". optionDefinitions); + +if (options.suite && !VALID_TAGS.includes(options.suite)) + printHelp(`Invalid suite: ${options.suite}. Choices are: ${VALID_TAGS.join(", ")}`); const BROWSER = options?.browser; if (!BROWSER) @@ -73,7 +116,7 @@ switch (BROWSER) { break; } default: { - printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`); + printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`, optionDefinitions); } } @@ -91,12 +134,19 @@ const server = await serve(PORT); async function runTests() { let success = true; + const suiteFilter = options.suite || "all"; + + const testsToRun = TESTS.filter(test => test.tags.includes(suiteFilter)); + + if (testsToRun.length === 0) { + console.error(`No suite found for filter: ${suiteFilter}`); + process.exit(1); + } + try { - success &&= await runEnd2EndTest("Run Single Suite", { test: "proxy-mobx" }); - success &&= await runEnd2EndTest("Run Multiple Suites", { test: "prismjs-startup-es6,postcss-wtb" }); - success &&= await runEnd2EndTest("Run Tag No Prefetch", { tag: "proxy", prefetchResources: "false" }); - success &&= await runEnd2EndTest("Run Disabled Suite", { tag: "disabled" }); - success &&= await runEnd2EndTest("Run Default Suite"); + for (const test of testsToRun) { + success &&= await test.run(); + } } finally { server.close(); } diff --git a/tests/run-shell.mjs b/tests/run-shell.mjs index 80a62248..7944dce0 100644 --- a/tests/run-shell.mjs +++ b/tests/run-shell.mjs @@ -31,103 +31,193 @@ import { fileURLToPath } from "url"; import { logGroup, logInfo, printHelp, runTest, sh } from "./helper.mjs"; +const FILE_PATH = fileURLToPath(import.meta.url); +const SRC_DIR = path.dirname(path.dirname(FILE_PATH)); +const CLI_PATH = path.join(SRC_DIR, "cli.js"); +const UNIT_TEST_PATH = path.join(SRC_DIR, "tests", "unit-tests.js"); + +const TESTS = [ + { + name: "UnitTests", + tags: ["all", "main", "unit"], + run(shell_binary) { + return runTest("UnitTests", () => sh(shell_binary, UNIT_TEST_PATH)); + }, + }, + { + name: "Single Suite", + tags: ["all", "main", "single"], + run(shell_binary) { + return runCLITest("Single Suite", shell_binary, "proxy-mobx"); + }, + }, + { + name: "Tag No Prefetch", + tags: ["all", "main", "no-prefetch"], + run(shell_binary) { + return runCLITest( + "Tag No Prefetch", + shell_binary, + "proxy", + "argon2-wasm", + "--no-prefetch" + ); + }, + }, + { + name: "Grouped with Details", + tags: ["all", "main", "group-details"], + run(shell_binary) { + return runCLITest("Grouped with Details", shell_binary, "SunSpider", "--group-details"); + }, + }, + { + name: "Disabled Suite", + tags: ["all", "disabled"], + run(shell_binary) { + return runCLITest("Disabled Suite", shell_binary, "disabled"); + }, + }, + { + name: "Default Suite", + tags: ["all", "default"], + run(shell_binary) { + return runCLITest("Default Suite", shell_binary); + }, + }, +]; + +const VALID_TAGS = Array.from(new Set(TESTS.map((each) => each.tags).flat())); + const optionDefinitions = [ - { name: "shell", type: String, description: "Set the shell to test, choices are [jsc, v8, spidermonkey]." }, - { name: "help", alias: "h", description: "Print this help text." }, + { + name: "shell", + type: String, + description: "Set the shell to test, choices are [jsc, v8, spidermonkey].", + }, + { name: "help", alias: "h", description: "Print this help text." }, + { + name: "suite", + type: String, + defaultOption: true, + typeLabel: `choices: ${VALID_TAGS.join(", ")}`, + description: "Run a specific suite by name.", + }, ]; const options = commandLineArgs(optionDefinitions); -if ("help" in options) - printHelp(optionDefinitions); +if ("help" in options) { + printHelp("", optionDefinitions); +} + +if (options.suite && !VALID_TAGS.includes(options.suite)) { + printHelp( + `Invalid suite: ${options.suite}. Choices are: ${VALID_TAGS.join(", ")}`, + optionDefinitions + ); +} -const JS_SHELL= options?.shell; -if (!JS_SHELL) - printHelp("No javascript shell specified, use --shell", optionDefinitions); +const JS_SHELL = options?.shell; +if (!JS_SHELL) { + printHelp("No javascript shell specified, use --shell", optionDefinitions); +} -const SHELL_NAME = (function() { - switch (JS_SHELL) { - case "javascriptcore": - case "jsc": { - return "javascriptcore"; +const SHELL_NAME = (function () { + switch (JS_SHELL) { + case "javascriptcore": + case "jsc": { + return "javascriptcore"; + } + case "spidermonkey": { + return "spidermonkey"; + } + case "v8": { + return "v8"; + } + default: { + printHelp( + `Invalid shell "${JS_SHELL}", choices are: "jsc", "spidermonkey" and "v8)`, + optionDefinitions + ); + } } - case "spidermonkey": { - return "spidermonkey"; - } - case "v8": { - return "v8"; - } - default: { - printHelp(`Invalid shell "${JS_SHELL}", choices are: "jsc", "spidermonkey" and "v8)`); - } - } })(); -const FILE_PATH = fileURLToPath(import.meta.url); -const SRC_DIR = path.dirname(path.dirname(FILE_PATH)); -const CLI_PATH = path.join(SRC_DIR, "cli.js"); -const UNIT_TEST_PATH = path.join(SRC_DIR, "tests", "unit-tests.js"); - function convertCliArgs(cli, ...cliArgs) { - if (SHELL_NAME == "spidermonkey") - return [cli, ...cliArgs]; - return [cli, "--", ...cliArgs]; + if (SHELL_NAME == "spidermonkey") return [cli, ...cliArgs]; + return [cli, "--", ...cliArgs]; } - async function runTests() { - const shellBinary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); + const shell_binary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup); + const suiteFilter = options.suite || "all"; let success = true; - success &&= await runTest("Run UnitTests", () => sh(shellBinary, UNIT_TEST_PATH)); - success &&= await runCLITest("Run Single Suite", shellBinary, "proxy-mobx"); - success &&= await runCLITest("Run Tag No Prefetch", shellBinary, "proxy", "argon2-wasm", "--no-prefetch"); - success &&= await runCLITest("Run Grouped with Details", shellBinary, "SunSpider", "--group-details"); - success &&= await runCLITest("Run Disabled Suite", shellBinary, "disabled"); - success &&= await runCLITest("Run Default Suite", shellBinary); - if (!success) - process.exit(1); + const testsToRun = TESTS.filter((test) => test.tags.includes(suiteFilter)); + + if (testsToRun.length === 0) { + console.error(`No suite found for filter: ${suiteFilter}`); + process.exit(1); + } + + for (const test of testsToRun) { + success &&= await test.run(shell_binary); + } + + if (!success) { + process.exit(1); + } } function jsvuOSName() { - const osName = () => { - switch (os.platform()) { - case "win32": return "win"; - case "darwin": return "mac"; - case "linux": return "linux"; - default: throw new Error("Unsupported OS"); - } - }; - const osArch = () => { - switch (os.arch()) { - case "x64": return "64"; - case "arm64": return "64arm"; - default: throw new Error("Unsupported architecture"); - } - }; - return `${osName()}${osArch()}`; + const osName = () => { + switch (os.platform()) { + case "win32": + return "win"; + case "darwin": + return "mac"; + case "linux": + return "linux"; + default: + throw new Error("Unsupported OS"); + } + }; + const osArch = () => { + switch (os.arch()) { + case "x64": + return "64"; + case "arm64": + return "64arm"; + default: + throw new Error("Unsupported architecture"); + } + }; + return `${osName()}${osArch()}`; } -const DEFAULT_JSC_LOCATION = "/System/Library/Frameworks/JavaScriptCore.framework/Versions/Current/Helpers/jsc" +const DEFAULT_JSC_LOCATION = + "/System/Library/Frameworks/JavaScriptCore.framework/Versions/Current/Helpers/jsc"; async function testSetup() { await sh("jsvu", `--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`); let shellBinary = path.join(os.homedir(), ".jsvu/bin", SHELL_NAME); if (!fs.existsSync(shellBinary) && SHELL_NAME == "javascriptcore") - shellBinary = DEFAULT_JSC_LOCATION; - if (!fs.existsSync(shellBinary)) - throw new Error(`Could not find shell binary: ${shellBinary}`); + shellBinary = DEFAULT_JSC_LOCATION; + if (!fs.existsSync(shellBinary)) throw new Error(`Could not find shell binary: ${shellBinary}`); logInfo(`Installed JavaScript Shell: ${shellBinary}`); return shellBinary; } function runCLITest(name, shellBinary, ...args) { - return runTest(name, () => runShell(shellBinary, ...convertCliArgs(CLI_PATH, ...args))); + return runTest(name, () => runShell(shellBinary, ...convertCliArgs(CLI_PATH, ...args))); } async function runShell(shellBinary, ...args) { - const result = await sh(shellBinary, ...args); - if (result.stdoutString.includes("JetStream3 failed")) - throw new Error("test failed") + const result = await sh(shellBinary, ...args); + // JSC does not set a non-0 exit status on async exceptions. + if (SHELL_NAME == "javascriptcore" && result.stdoutString.includes("JetStream3 failed")) { + throw new Error("test failed"); + } } setImmediate(runTests);