From 75b5c32743e7b0ce35f56247ea2002122a3163fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ari=20Perkki=C3=B6?= Date: Wed, 8 May 2024 21:59:25 +0300 Subject: [PATCH] test: convert benchmarks to vitest --- .github/workflows/benchmark.yml | 32 +++++++ benchmark/fixtures/add-process.mjs | 5 + .../fixtures/{wrap-add.mjs => add-worker.mjs} | 0 benchmark/isolate-benchmark.bench.ts | 91 +++++++++++++++++++ benchmark/isolate-benchmark.mjs | 65 ------------- benchmark/simple-benchmark.mjs | 28 ------ benchmark/simple.bench.ts | 21 +++++ package.json | 1 + vitest.config.ts | 4 + 9 files changed, 154 insertions(+), 93 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/fixtures/add-process.mjs rename benchmark/fixtures/{wrap-add.mjs => add-worker.mjs} (100%) create mode 100644 benchmark/isolate-benchmark.bench.ts delete mode 100644 benchmark/isolate-benchmark.mjs delete mode 100644 benchmark/simple-benchmark.mjs create mode 100644 benchmark/simple.bench.ts diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..cf9cf5f --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,32 @@ +on: [workflow_dispatch] + +name: Benchmark + +jobs: + test: + name: Test + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + node-version: [18.x, 20.x] + + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v2 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + + - uses: pnpm/action-setup@v2 + + - name: Install Dependencies + run: pnpm install + + - name: Build + run: pnpm build + + - name: Benchmark + run: pnpm bench diff --git a/benchmark/fixtures/add-process.mjs b/benchmark/fixtures/add-process.mjs new file mode 100644 index 0000000..0240fde --- /dev/null +++ b/benchmark/fixtures/add-process.mjs @@ -0,0 +1,5 @@ +import add from './add.mjs' + +process.on('message', (message) => { + process.send(add(message)) +}) diff --git a/benchmark/fixtures/wrap-add.mjs b/benchmark/fixtures/add-worker.mjs similarity index 100% rename from benchmark/fixtures/wrap-add.mjs rename to benchmark/fixtures/add-worker.mjs diff --git a/benchmark/isolate-benchmark.bench.ts b/benchmark/isolate-benchmark.bench.ts new file mode 100644 index 0000000..80a0adf --- /dev/null +++ b/benchmark/isolate-benchmark.bench.ts @@ -0,0 +1,91 @@ +import { bench } from 'vitest' +import { cpus } from 'node:os' +import { Worker } from 'node:worker_threads' +import { fork } from 'node:child_process' +import Tinypool, { Options } from '../dist/index' + +const THREADS = cpus().length - 1 +const ROUNDS = THREADS * 10 +const ITERATIONS = 100 + +for (const runtime of [ + 'worker_threads', + 'child_process', +] as Options['runtime'][]) { + bench( + `Tinypool { runtime: '${runtime}' }`, + async () => { + const pool = new Tinypool({ + runtime, + filename: './benchmark/fixtures/add.mjs', + isolateWorkers: true, + minThreads: THREADS, + maxThreads: THREADS, + }) + + await Promise.all( + Array(ROUNDS) + .fill(0) + .map(() => pool.run({ a: 1, b: 2 })) + ) + + await pool.destroy() + }, + { iterations: ITERATIONS } + ) +} + +for (const { task, name } of [ + { name: 'worker_threads', task: workerThreadTask }, + { name: 'child_process', task: childProcessTask }, +] as const) { + bench( + `node:${name}`, + async () => { + const pool = Array(ROUNDS).fill(task) + + await Promise.all( + Array(THREADS) + .fill(execute) + .map((_task) => _task()) + ) + + async function execute() { + const _task = pool.shift() + + if (_task) { + await _task() + return execute() + } + } + }, + { iterations: ITERATIONS } + ) +} + +async function workerThreadTask() { + const worker = new Worker('./benchmark/fixtures/add-worker.mjs') + const onMessage = new Promise((resolve, reject) => + worker.on('message', (sum) => (sum === 3 ? resolve() : reject('Not 3'))) + ) + + worker.postMessage({ a: 1, b: 2 }) + await onMessage + + await worker.terminate() +} + +async function childProcessTask() { + const subprocess = fork('./benchmark/fixtures/add-process.mjs') + + const onExit = new Promise((resolve) => subprocess.on('exit', resolve)) + const onMessage = new Promise((resolve, reject) => + subprocess.on('message', (sum) => (sum === 3 ? resolve() : reject('Not 3'))) + ) + + subprocess.send({ a: 1, b: 2 }) + await onMessage + + subprocess.kill() + await onExit +} diff --git a/benchmark/isolate-benchmark.mjs b/benchmark/isolate-benchmark.mjs deleted file mode 100644 index d1a3767..0000000 --- a/benchmark/isolate-benchmark.mjs +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Benchmark for testing whether Tinypool's worker creation and teardown is expensive. - */ -import { cpus } from 'node:os' -import { Worker } from 'node:worker_threads' - -import Tinypool from '../dist/index.js' - -const THREADS = cpus().length - 1 -const ROUNDS = 5_000 - -await logTime('Tinypool', runTinypool) -await logTime('Worker threads', runWorkerThreads) - -async function runTinypool() { - const pool = new Tinypool({ - filename: new URL('./fixtures/add.mjs', import.meta.url).href, - isolateWorkers: true, - minThreads: THREADS, - maxThreads: THREADS, - }) - - await Promise.all( - Array(ROUNDS) - .fill() - .map(() => pool.run({ a: 1, b: 2 })) - ) -} - -async function runWorkerThreads() { - async function task() { - const worker = new Worker('./fixtures/wrap-add.mjs') - worker.postMessage({ a: 1, b: 2 }) - - await new Promise((resolve, reject) => - worker.on('message', (sum) => (sum === 3 ? resolve() : reject('Not 3'))) - ) - - await worker.terminate() - } - - const pool = Array(ROUNDS).fill(task) - - async function execute() { - const task = pool.shift() - - if (task) { - await task() - return execute() - } - } - - await Promise.all( - Array(THREADS) - .fill(execute) - .map((task) => task()) - ) -} - -async function logTime(label, method) { - const start = process.hrtime.bigint() - await method() - const end = process.hrtime.bigint() - console.log(label, 'took', ((end - start) / 1_000_000n).toString(), 'ms') -} diff --git a/benchmark/simple-benchmark.mjs b/benchmark/simple-benchmark.mjs deleted file mode 100644 index 48c2478..0000000 --- a/benchmark/simple-benchmark.mjs +++ /dev/null @@ -1,28 +0,0 @@ -import Tinypool from '../dist/index.js' - -async function simpleBenchmark({ duration = 10000 } = {}) { - const pool = new Tinypool({ - filename: new URL('./fixtures/add.mjs', import.meta.url).href, - }) - let done = 0 - - const results = [] - const start = process.hrtime.bigint() - while (pool.queueSize === 0) { - results.push(scheduleTasks()) - } - - async function scheduleTasks() { - while ((process.hrtime.bigint() - start) / 1_000_000n < duration) { - await pool.run({ a: 4, b: 6 }) - done++ - } - } - - await Promise.all(results) - - return (done / duration) * 1e3 -} - -const opsPerSecond = await simpleBenchmark() -console.log(`opsPerSecond: ${opsPerSecond}`) diff --git a/benchmark/simple.bench.ts b/benchmark/simple.bench.ts new file mode 100644 index 0000000..94d7805 --- /dev/null +++ b/benchmark/simple.bench.ts @@ -0,0 +1,21 @@ +import { bench } from 'vitest' +import Tinypool from '../dist/index' + +bench( + 'simple', + async () => { + const pool = new Tinypool({ + filename: './benchmark/fixtures/add.mjs', + }) + + const tasks: Promise[] = [] + + while (pool.queueSize === 0) { + tasks.push(pool.run({ a: 4, b: 6 })) + } + + await Promise.all(tasks) + await pool.destroy() + }, + { time: 10_000 } +) diff --git a/package.json b/package.json index d014885..d372d85 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ }, "scripts": { "test": "vitest", + "bench": "vitest bench", "dev": "tsup --watch", "build": "tsup", "publish": "clean-publish", diff --git a/vitest.config.ts b/vitest.config.ts index 08e0041..5202d69 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -16,5 +16,9 @@ export default defineConfig({ // simple.test.ts expects to be run in main thread poolMatchGlobs: [['**/simple.test.ts', 'forks']], + + benchmark: { + include: ['**/**.bench.ts'], + }, }, })