From c5c08d2476db146bae827190011d434816c247f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Sat, 6 Dec 2025 20:55:11 +0100 Subject: [PATCH 1/6] Add a failing test --- .gitignore | 1 + implementors/node/run-tests.ts | 94 ++++++++++++++++++++++++++++++++++ package-lock.json | 32 ++++++++++++ package.json | 12 +++++ tests/harness/assert.js | 33 ++++++++++++ tests/js-native-api/tumbleweed | 0 tests/node-api/tumbleweed | 0 7 files changed, 172 insertions(+) create mode 100644 .gitignore create mode 100644 implementors/node/run-tests.ts create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 tests/harness/assert.js create mode 100644 tests/js-native-api/tumbleweed create mode 100644 tests/node-api/tumbleweed diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/implementors/node/run-tests.ts b/implementors/node/run-tests.ts new file mode 100644 index 0000000..d5e993f --- /dev/null +++ b/implementors/node/run-tests.ts @@ -0,0 +1,94 @@ +import { spawn } from "node:child_process"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { test, type TestContext } from "node:test"; + +const ROOT_PATH = path.resolve(import.meta.dirname, "..", ".."); +const TESTS_ROOT_PATH = path.join(ROOT_PATH, "tests"); + +async function listDirectoryEntries(dir: string) { + const entries = await fs.readdir(dir, { withFileTypes: true }); + const directories: string[] = []; + const files: string[] = []; + + for (const entry of entries) { + if (entry.isDirectory()) { + directories.push(entry.name); + } else if (entry.isFile() && entry.name.endsWith(".js")) { + files.push(entry.name); + } + } + + directories.sort(); + files.sort(); + + return { directories, files }; +} + +function runFileInSubprocess(filePath: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [filePath]); + + let stderrOutput = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk) => { + stderrOutput += chunk; + }); + + child.stdout.pipe(process.stdout); + + child.on("error", reject); + + child.on("close", (code, signal) => { + if (code === 0) { + resolve(); + return; + } + + const reason = + code !== null ? `exit code ${code}` : `signal ${signal ?? "unknown"}`; + const trimmedStderr = stderrOutput.trim(); + const stderrSuffix = trimmedStderr + ? `\n--- stderr ---\n${trimmedStderr}\n--- end stderr ---` + : ""; + reject( + new Error( + `Test file ${path.relative( + TESTS_ROOT_PATH, + filePath + )} failed (${reason})${stderrSuffix}` + ) + ); + }); + }); +} + +async function populateSuite( + testContext: TestContext, + dir: string +): Promise { + const { directories, files } = await listDirectoryEntries(dir); + + for (const file of files) { + const filePath = path.join(dir, file); + await testContext.test(file, () => runFileInSubprocess(filePath)); + } + + for (const directory of directories) { + await testContext.test(directory, async (subTest) => { + await populateSuite(subTest, path.join(dir, directory)); + }); + } +} + +test("harness", async (t) => { + await populateSuite(t, path.join(TESTS_ROOT_PATH, "harness")); +}); + +test("js-native-api", async (t) => { + await populateSuite(t, path.join(TESTS_ROOT_PATH, "js-native-api")); +}); + +test("node-api", async (t) => { + await populateSuite(t, path.join(TESTS_ROOT_PATH, "node-api")); +}); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..dddcb54 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,32 @@ +{ + "name": "node-api-cts", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "node-api-cts", + "version": "0.1.0", + "devDependencies": { + "@types/node": "^24.10.1" + } + }, + "node_modules/@types/node": { + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..c2483a1 --- /dev/null +++ b/package.json @@ -0,0 +1,12 @@ +{ + "name": "node-api-cts", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "node:test": "node --test ./implementors/node/run-tests.ts" + }, + "devDependencies": { + "@types/node": "^24.10.1" + } +} diff --git a/tests/harness/assert.js b/tests/harness/assert.js new file mode 100644 index 0000000..47b89f3 --- /dev/null +++ b/tests/harness/assert.js @@ -0,0 +1,33 @@ +if (typeof assert !== 'function') { + throw new Error('Expected a global assert function'); +} + +try { + assert(true, 'assert(true, message) should not throw'); +} catch (error) { + throw new Error(`Global assert(true, message) must not throw: ${String(error)}`); +} + +const failureMessage = 'assert(false, message) should throw this message'; +let threw = false; + +try { + assert(false, failureMessage); +} catch (error) { + threw = true; + + if (!(error instanceof Error)) { + throw new Error(`Global assert(false, message) must throw an Error instance but got: ${String(error)}`); + } + + const actualMessage = error.message; + if (actualMessage !== failureMessage) { + throw new Error( + `Global assert(false, message) must throw message "${failureMessage}" but got "${actualMessage}"`, + ); + } +} + +if (!threw) { + throw new Error('Global assert(false, message) must throw'); +} diff --git a/tests/js-native-api/tumbleweed b/tests/js-native-api/tumbleweed new file mode 100644 index 0000000..e69de29 diff --git a/tests/node-api/tumbleweed b/tests/node-api/tumbleweed new file mode 100644 index 0000000..e69de29 From f9435d8666f3eaf896c5417e3b91dd628fe4db12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Sat, 6 Dec 2025 20:55:49 +0100 Subject: [PATCH 2/6] Add assert implementation --- implementors/node/assert.js | 8 ++++++++ implementors/node/run-tests.ts | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 implementors/node/assert.js diff --git a/implementors/node/assert.js b/implementors/node/assert.js new file mode 100644 index 0000000..8a5039f --- /dev/null +++ b/implementors/node/assert.js @@ -0,0 +1,8 @@ + +import { ok } from "node:assert/strict"; + +const assert = (value, message) => { + ok(value, message); +}; + +Object.assign(globalThis, { assert }); diff --git a/implementors/node/run-tests.ts b/implementors/node/run-tests.ts index d5e993f..a0fa138 100644 --- a/implementors/node/run-tests.ts +++ b/implementors/node/run-tests.ts @@ -5,6 +5,12 @@ import { test, type TestContext } from "node:test"; const ROOT_PATH = path.resolve(import.meta.dirname, "..", ".."); const TESTS_ROOT_PATH = path.join(ROOT_PATH, "tests"); +const ASSERT_MODULE_PATH = path.join( + ROOT_PATH, + "implementors", + "node", + "assert.js" +); async function listDirectoryEntries(dir: string) { const entries = await fs.readdir(dir, { withFileTypes: true }); @@ -27,7 +33,11 @@ async function listDirectoryEntries(dir: string) { function runFileInSubprocess(filePath: string): Promise { return new Promise((resolve, reject) => { - const child = spawn(process.execPath, [filePath]); + const child = spawn(process.execPath, [ + "--import", + ASSERT_MODULE_PATH, + filePath, + ]); let stderrOutput = ""; child.stderr.setEncoding("utf8"); From 73df7c8d0bc9e7fc332173b028ad2d932fe1d630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Sat, 6 Dec 2025 20:56:06 +0100 Subject: [PATCH 3/6] Update copilot instructions --- .github/copilot-instructions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index e9388be..85de207 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -7,7 +7,7 @@ Node-API Conformance Test Suite: A pure ECMAScript test suite for Node-API imple ## General Principles - **Keep it minimal**: Avoid unnecessary dependencies, configuration, or complexity - **Pure ECMAScript tests**: To lower the barrier for implementors to run the tests, all tests are written as single files of pure ECMAScript, with no import / export statements and no use of Node.js runtime APIs outside of the language standard (such as `process`, `require`, `node:*` modules). -- **TypeScript tooling**: The tooling around the tests are written in TypeScript and expects to be ran in a Node.js compatible runtime with type-stripping enabled. +- **TypeScript tooling**: The tooling around the tests are written in TypeScript and expects to be ran in a Node.js compatible runtime with type-stripping enabled. Never rely on `ts-node`, `node --loader ts-node/esm`, or `--experimental-strip-types`; use only stable, built-in Node.js capabilities. - **Implementor Flexibility**: Native code building and loading is delegated to implementors, with the test-suite providing defaults that work for Node.js. - **Extra convenience**: Extra (generated) code is provided to make it easier for implementors to load and run tests, such as extra package exports exposing test functions that implementors can integrate with their preferred test frameworks. - **Process Isolation**: The built-in runner for Node.js, run each test in isolation to prevent crashes from aborting entire test suite. From 3a519755dfd8ce8c53c3bf26874edfe20089490f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Sat, 6 Dec 2025 20:56:22 +0100 Subject: [PATCH 4/6] Add docs on the implementors directory --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 326acdb..a8b60f7 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,19 @@ Written in ECMAScript & C/C++ with an implementor customizable harness. ## Overview -The tests are divided into two buckets, based on the two header files declaring the Node-API functions: +### Tests + +The tests are divided into three buckets: + +- `tests/harness/*` exercising the implementor's test harness. + +and two parts based on the two header files declaring the Node-API functions: - `tests/js-native-api/*` testing the engine-specific part of Node-API defined in the [`js_native_api.h`](https://github.com/nodejs/node-api-headers/blob/main/include/js_native_api.h) header. - `tests/node-api/*` testing the runtime-specific part of Node-API defined in the [`node_api.h`](https://github.com/nodejs/node-api-headers/blob/main/include/node_api.h) header. + +### Implementors + +This repository offers an opportunity for implementors of Node-API to maintain (parts of) their implementor-specific harness inside this repository, in a sub-directory of the `implementors` directory. We do this in hope of increased velocity from faster iteration and potentials for reuse of code across the harnesses. + +We maintain a list of other runtimes implementing Node-API in [doc/node-api-engine-bindings.md](https://github.com/nodejs/abi-stable-node/blob/doc/node-api-engine-bindings.md#node-api-bindings-in-other-runtimes) of the `nodejs/abi-stable-node` repository. From 50000688d93716c45dbbbb6ef987a4e59bb1c6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Sat, 6 Dec 2025 21:11:06 +0100 Subject: [PATCH 5/6] Add simple GHA action running the Node.js tests --- .github/workflows/test.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..dc46ac3 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,34 @@ +name: Test Node.js implementation + +on: [push, pull_request] + +jobs: + test: + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + node-version: + - 20.x + - 22.x + - 24.x + - 25.x + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + with: + egress-policy: audit + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + with: + node-version: ${{ matrix.node-version }} + - name: Check Node.js installation + run: | + node --version + npm --version + - name: Install dependencies + run: npm install + - name: npm test + run: npm run node:test From a6f2fa099630ef296a9c4bbed83b3b80476ac861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Tue, 9 Dec 2025 23:24:41 +0100 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: Chengzhong Wu --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc46ac3..3a7f774 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,6 +29,6 @@ jobs: node --version npm --version - name: Install dependencies - run: npm install + run: npm ci - name: npm test run: npm run node:test