From 5d92cec12d1252dfbac8b59f0c744250e5dc0b1e Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 12 Feb 2020 19:33:33 +0100 Subject: [PATCH] benchmark: add `test` and `all` options and improve errors" This reverts commit 4671d551cf9210434bdadf65ee5654606d24da70 and contains a fix to the issue raised for the revert. PR-URL: https://github.com/nodejs/node/pull/31755 Reviewed-By: Anna Henningsen Reviewed-By: Vladimir de Turckheim Reviewed-By: Rich Trott --- benchmark/_cli.js | 22 ++- benchmark/_http-benchmarkers.js | 32 ++-- benchmark/_test-double-benchmarker.js | 10 +- .../async_hooks/async-resource-vs-destroy.js | 13 +- benchmark/async_hooks/http-server.js | 6 +- benchmark/buffers/buffer-base64-encode.js | 2 + benchmark/buffers/buffer-swap.js | 2 + benchmark/common.js | 177 +++++++++++------- benchmark/compare.js | 2 +- benchmark/crypto/cipher-stream.js | 2 + benchmark/fs/read-stream-throughput.js | 75 ++++---- benchmark/fs/readfile.js | 6 +- benchmark/http/chunked.js | 8 +- benchmark/http/cluster.js | 8 +- benchmark/http/end-vs-write-end.js | 8 +- benchmark/http/headers.js | 6 +- benchmark/http/incoming_headers.js | 16 +- benchmark/http/set-header.js | 10 +- benchmark/http/simple.js | 10 +- benchmark/http2/compat.js | 8 +- benchmark/http2/respond-with-fd.js | 6 +- benchmark/http2/simple.js | 6 +- benchmark/http2/write.js | 6 +- benchmark/misc/trace.js | 6 +- benchmark/net/net-c2s.js | 2 + benchmark/net/net-pipe.js | 2 + benchmark/net/net-s2c.js | 2 + .../net/net-wrap-js-stream-passthrough.js | 1 + benchmark/net/tcp-raw-c2s.js | 5 +- benchmark/net/tcp-raw-pipe.js | 1 + benchmark/net/tcp-raw-s2c.js | 1 + benchmark/run.js | 17 +- benchmark/scatter.js | 2 +- benchmark/tls/secure-pair.js | 2 + benchmark/util/type-check.js | 2 +- benchmark/zlib/pipe.js | 5 + doc/guides/writing-and-running-benchmarks.md | 7 +- test/common/benchmark.js | 9 +- 38 files changed, 305 insertions(+), 200 deletions(-) diff --git a/benchmark/_cli.js b/benchmark/_cli.js index 771cc72bff1964..eb6c4add9799a4 100644 --- a/benchmark/_cli.js +++ b/benchmark/_cli.js @@ -6,15 +6,16 @@ const path = require('path'); // Create an object of all benchmark scripts const benchmarks = {}; fs.readdirSync(__dirname) - .filter((name) => fs.statSync(path.resolve(__dirname, name)).isDirectory()) + .filter((name) => { + return name !== 'fixtures' && + fs.statSync(path.resolve(__dirname, name)).isDirectory(); + }) .forEach((category) => { benchmarks[category] = fs.readdirSync(path.resolve(__dirname, category)) .filter((filename) => filename[0] !== '.' && filename[0] !== '_'); }); function CLI(usage, settings) { - if (!(this instanceof CLI)) return new CLI(usage, settings); - if (process.argv.length < 3) { this.abort(usage); // Abort will exit the process } @@ -22,6 +23,7 @@ function CLI(usage, settings) { this.usage = usage; this.optional = {}; this.items = []; + this.test = false; for (const argName of settings.arrayArgs) { this.optional[argName] = []; @@ -34,7 +36,7 @@ function CLI(usage, settings) { if (arg === '--') { // Only items can follow -- mode = 'item'; - } else if ('both' === mode && arg[0] === '-') { + } else if (mode === 'both' && arg[0] === '-') { // Optional arguments declaration if (arg[1] === '-') { @@ -61,6 +63,8 @@ function CLI(usage, settings) { // The next value can be either an option or an item mode = 'both'; + } else if (arg === 'test') { + this.test = true; } else if (['both', 'item'].includes(mode)) { // item arguments this.items.push(arg); @@ -83,9 +87,15 @@ CLI.prototype.abort = function(msg) { CLI.prototype.benchmarks = function() { const paths = []; + if (this.items.includes('all')) { + this.items = Object.keys(benchmarks); + } + for (const category of this.items) { - if (benchmarks[category] === undefined) - continue; + if (benchmarks[category] === undefined) { + console.error(`The "${category}" category does not exist.`); + process.exit(1); + } for (const scripts of benchmarks[category]) { if (this.shouldSkip(scripts)) continue; diff --git a/benchmark/_http-benchmarkers.js b/benchmark/_http-benchmarkers.js index 821dab2d55e683..d0f192e75948b6 100644 --- a/benchmark/_http-benchmarkers.js +++ b/benchmark/_http-benchmarkers.js @@ -43,9 +43,8 @@ class AutocannonBenchmarker { } if (!result || !result.requests || !result.requests.average) { return undefined; - } else { - return result.requests.average; } + return result.requests.average; } } @@ -58,10 +57,13 @@ class WrkBenchmarker { } create(options) { + const duration = typeof options.duration === 'number' ? + Math.max(options.duration, 1) : + options.duration; const args = [ - '-d', options.duration, + '-d', duration, '-c', options.connections, - '-t', 8, + '-t', Math.min(options.connections, require('os').cpus().length || 8), `http://127.0.0.1:${options.port}${options.path}`, ]; for (const field in options.headers) { @@ -77,9 +79,8 @@ class WrkBenchmarker { const throughput = match && +match[1]; if (!isFinite(throughput)) { return undefined; - } else { - return throughput; } + return throughput; } } @@ -89,7 +90,8 @@ class WrkBenchmarker { */ class TestDoubleBenchmarker { constructor(type) { - // `type` is the type ofbenchmarker. Possible values are 'http' and 'http2'. + // `type` is the type of benchmarker. Possible values are 'http' and + // 'http2'. this.name = `test-double-${type}`; this.executable = path.resolve(__dirname, '_test-double-benchmarker.js'); this.present = fs.existsSync(this.executable); @@ -97,10 +99,12 @@ class TestDoubleBenchmarker { } create(options) { - const env = Object.assign({ - duration: options.duration, + process.env.duration = process.env.duration || options.duration || 5; + + const env = { test_url: `http://127.0.0.1:${options.port}${options.path}`, - }, process.env); + ...process.env + }; const child = child_process.fork(this.executable, [this.type], @@ -189,13 +193,14 @@ http_benchmarkers.forEach((benchmarker) => { }); exports.run = function(options, callback) { - options = Object.assign({ + options = { port: exports.PORT, path: '/', connections: 100, duration: 5, benchmarker: exports.default_http_benchmarker, - }, options); + ...options + }; if (!options.benchmarker) { callback(new Error('Could not locate required http benchmarker. See ' + `${requirementsURL} for further instructions.`)); @@ -220,7 +225,8 @@ exports.run = function(options, callback) { child.stderr.pipe(process.stderr); let stdout = ''; - child.stdout.on('data', (chunk) => stdout += chunk.toString()); + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => stdout += chunk); child.once('close', (code) => { const elapsed = process.hrtime(benchmarker_start); diff --git a/benchmark/_test-double-benchmarker.js b/benchmark/_test-double-benchmarker.js index b9379b907ffa07..60264dfd46a606 100644 --- a/benchmark/_test-double-benchmarker.js +++ b/benchmark/_test-double-benchmarker.js @@ -7,7 +7,7 @@ if (!['http', 'http2'].includes(myModule)) { const http = require(myModule); -const duration = process.env.duration || 0; +const duration = +process.env.duration; const url = process.env.test_url; const start = process.hrtime(); @@ -18,13 +18,15 @@ function request(res, client) { res.on('error', () => {}); res.on('end', () => { throughput++; - const diff = process.hrtime(start); - if (duration > 0 && diff[0] < duration) { + const [sec, nanosec] = process.hrtime(start); + const ms = sec * 1000 + nanosec / 1e6; + if (ms < duration * 1000) { run(); } else { console.log(JSON.stringify({ throughput })); if (client) { client.destroy(); + process.exit(0); } } }); @@ -33,7 +35,7 @@ function request(res, client) { function run() { if (http.get) { // HTTP http.get(url, request); - } else { // HTTP/2 + } else { // HTTP/2 const client = http.connect(url); client.on('error', (e) => { throw e; }); request(client.request(), client); diff --git a/benchmark/async_hooks/async-resource-vs-destroy.js b/benchmark/async_hooks/async-resource-vs-destroy.js index c9b9a81c5b7c7f..da0b52afa0ec40 100644 --- a/benchmark/async_hooks/async-resource-vs-destroy.js +++ b/benchmark/async_hooks/async-resource-vs-destroy.js @@ -13,14 +13,12 @@ const { } = require('async_hooks'); const { createServer } = require('http'); -// Configuration for the http server -// there is no need for parameters in this test -const connections = 500; -const path = '/'; - const bench = common.createBenchmark(main, { type: ['async-resource', 'destroy', 'async-local-storage'], asyncMethod: ['callbacks', 'async'], + path: '/', + connections: 500, + duration: 5, n: [1e6] }); @@ -165,7 +163,7 @@ const asyncMethods = { 'async': getServeAwait }; -function main({ type, asyncMethod }) { +function main({ type, asyncMethod, connections, duration, path }) { const { server, close } = types[type](asyncMethods[asyncMethod]); server @@ -174,7 +172,8 @@ function main({ type, asyncMethod }) { bench.http({ path, - connections + connections, + duration }, () => { close(); }); diff --git a/benchmark/async_hooks/http-server.js b/benchmark/async_hooks/http-server.js index 9e1c1214240eaa..c8e44849b7466f 100644 --- a/benchmark/async_hooks/http-server.js +++ b/benchmark/async_hooks/http-server.js @@ -3,10 +3,11 @@ const common = require('../common.js'); const bench = common.createBenchmark(main, { asyncHooks: ['init', 'before', 'after', 'all', 'disabled', 'none'], - connections: [50, 500] + connections: [50, 500], + duration: 5 }); -function main({ asyncHooks, connections }) { +function main({ asyncHooks, connections, duration }) { if (asyncHooks !== 'none') { let hooks = { init() {}, @@ -33,6 +34,7 @@ function main({ asyncHooks, connections }) { bench.http({ connections, path, + duration }, () => { server.close(); }); diff --git a/benchmark/buffers/buffer-base64-encode.js b/benchmark/buffers/buffer-base64-encode.js index d8b601bbd181f4..9837828a353c2d 100644 --- a/benchmark/buffers/buffer-base64-encode.js +++ b/benchmark/buffers/buffer-base64-encode.js @@ -25,6 +25,8 @@ const common = require('../common.js'); const bench = common.createBenchmark(main, { len: [64 * 1024 * 1024], n: [32] +}, { + test: { len: 256 } }); function main({ n, len }) { diff --git a/benchmark/buffers/buffer-swap.js b/benchmark/buffers/buffer-swap.js index a33bac4ae3ed78..5c31d86f7e2ab4 100644 --- a/benchmark/buffers/buffer-swap.js +++ b/benchmark/buffers/buffer-swap.js @@ -7,6 +7,8 @@ const bench = common.createBenchmark(main, { method: ['swap16', 'swap32', 'swap64'/* , 'htons', 'htonl', 'htonll' */], len: [64, 256, 768, 1024, 2056, 8192], n: [1e6] +}, { + test: { len: 16 } }); // The htons and htonl methods below are used to benchmark the diff --git a/benchmark/common.js b/benchmark/common.js index 62cd4023c1d860..d2103704ab2838 100644 --- a/benchmark/common.js +++ b/benchmark/common.js @@ -4,43 +4,83 @@ const child_process = require('child_process'); const http_benchmarkers = require('./_http-benchmarkers.js'); class Benchmark { - constructor(fn, configs, options) { - // Use the file name as the name of the benchmark - this.name = require.main.filename.slice(__dirname.length + 1); + // Used to make sure a benchmark only start a timer once + #started = false; + + // Indicate that the benchmark ended + #ended = false; + + // Holds process.hrtime value + #time = [0, 0]; + + // Use the file name as the name of the benchmark + name = require.main.filename.slice(__dirname.length + 1); + + // Execution arguments i.e. flags used to run the jobs + flags = process.env.NODE_BENCHMARK_FLAGS ? + process.env.NODE_BENCHMARK_FLAGS.split(/\s+/) : + []; + + constructor(fn, configs, options = {}) { // Parse job-specific configuration from the command line arguments - const parsed_args = this._parseArgs(process.argv.slice(2), configs); + const argv = process.argv.slice(2); + const parsed_args = this._parseArgs(argv, configs, options); this.options = parsed_args.cli; this.extra_options = parsed_args.extra; + if (options.flags) { + this.flags = this.flags.concat(options.flags); + } + // The configuration list as a queue of jobs this.queue = this._queue(this.options); + // The configuration of the current job, head of the queue this.config = this.queue[0]; - // Execution arguments i.e. flags used to run the jobs - this.flags = []; - if (options && options.flags) { - this.flags = this.flags.concat(options.flags); - } - if (process.env.NODE_BENCHMARK_FLAGS) { - const flags = process.env.NODE_BENCHMARK_FLAGS.split(/\s+/); - this.flags = this.flags.concat(flags); - } - // Holds process.hrtime value - this._time = [0, 0]; - // Used to make sure a benchmark only start a timer once - this._started = false; - this._ended = false; - - // this._run will use fork() to create a new process for each configuration - // combination. - if (process.env.hasOwnProperty('NODE_RUN_BENCHMARK_FN')) { - process.nextTick(() => fn(this.config)); - } else { - process.nextTick(() => this._run()); - } + + process.nextTick(() => { + if (process.env.hasOwnProperty('NODE_RUN_BENCHMARK_FN')) { + fn(this.config); + } else { + // _run will use fork() to create a new process for each configuration + // combination. + this._run(); + } + }); } - _parseArgs(argv, configs) { + _parseArgs(argv, configs, options) { const cliOptions = {}; + + // Check for the test mode first. + const testIndex = argv.indexOf('--test'); + if (testIndex !== -1) { + for (const [key, rawValue] of Object.entries(configs)) { + let value = Array.isArray(rawValue) ? rawValue[0] : rawValue; + // Set numbers to one by default to reduce the runtime. + if (typeof value === 'number') { + if (key === 'dur' || key === 'duration') { + value = 0.05; + } else if (value > 1) { + value = 1; + } + } + cliOptions[key] = [value]; + } + // Override specific test options. + if (options.test) { + for (const [key, value] of Object.entries(options.test)) { + cliOptions[key] = Array.isArray(value) ? value : [value]; + } + } + argv.splice(testIndex, 1); + } else { + // Accept single values instead of arrays. + for (const [key, value] of Object.entries(configs)) { + if (!Array.isArray(value)) + configs[key] = [value]; + } + } + const extraOptions = {}; const validArgRE = /^(.+?)=([\s\S]*)$/; // Parse configuration arguments @@ -50,45 +90,43 @@ class Benchmark { console.error(`bad argument: ${arg}`); process.exit(1); } - const config = match[1]; - - if (configs[config]) { - // Infer the type from the config object and parse accordingly - const isNumber = typeof configs[config][0] === 'number'; - const value = isNumber ? +match[2] : match[2]; - if (!cliOptions[config]) - cliOptions[config] = []; - cliOptions[config].push(value); + const [, key, value] = match; + if (Object.prototype.hasOwnProperty.call(configs, key)) { + if (!cliOptions[key]) + cliOptions[key] = []; + cliOptions[key].push( + // Infer the type from the config object and parse accordingly + typeof configs[key][0] === 'number' ? +value : value + ); } else { - extraOptions[config] = match[2]; + extraOptions[key] = value; } } - return { cli: Object.assign({}, configs, cliOptions), extra: extraOptions }; + return { cli: { ...configs, ...cliOptions }, extra: extraOptions }; } _queue(options) { const queue = []; const keys = Object.keys(options); - // Perform a depth-first walk though all options to generate a + // Perform a depth-first walk through all options to generate a // configuration list that contains all combinations. function recursive(keyIndex, prevConfig) { const key = keys[keyIndex]; const values = options[key]; - const type = typeof values[0]; for (const value of values) { if (typeof value !== 'number' && typeof value !== 'string') { throw new TypeError( `configuration "${key}" had type ${typeof value}`); } - if (typeof value !== type) { + if (typeof value !== typeof values[0]) { // This is a requirement for being able to consistently and // predictably parse CLI provided configuration values. throw new TypeError(`configuration "${key}" has mixed types`); } - const currConfig = Object.assign({ [key]: value }, prevConfig); + const currConfig = { [key]: value, ...prevConfig }; if (keyIndex + 1 < keys.length) { recursive(keyIndex + 1, currConfig); @@ -108,12 +146,11 @@ class Benchmark { } http(options, cb) { - const self = this; - const http_options = Object.assign({ }, options); + const http_options = { ...options }; http_options.benchmarker = http_options.benchmarker || - self.config.benchmarker || - self.extra_options.benchmarker || - exports.default_http_benchmarker; + this.config.benchmarker || + this.extra_options.benchmarker || + http_benchmarkers.default_http_benchmarker; http_benchmarkers.run( http_options, (error, code, used_benchmarker, result, elapsed) => { if (cb) { @@ -123,14 +160,13 @@ class Benchmark { console.error(error); process.exit(code || 1); } - self.config.benchmarker = used_benchmarker; - self.report(result, elapsed); + this.config.benchmarker = used_benchmarker; + this.report(result, elapsed); } ); } _run() { - const self = this; // If forked, report to the parent. if (process.send) { process.send({ @@ -140,27 +176,27 @@ class Benchmark { }); } - (function recursive(queueIndex) { - const config = self.queue[queueIndex]; + const recursive = (queueIndex) => { + const config = this.queue[queueIndex]; // Set NODE_RUN_BENCHMARK_FN to indicate that the child shouldn't // construct a configuration queue, but just execute the benchmark // function. - const childEnv = Object.assign({}, process.env); + const childEnv = { ...process.env }; childEnv.NODE_RUN_BENCHMARK_FN = ''; // Create configuration arguments const childArgs = []; - for (const key of Object.keys(config)) { - childArgs.push(`${key}=${config[key]}`); + for (const [key, value] of Object.entries(config)) { + childArgs.push(`${key}=${value}`); } - for (const key of Object.keys(self.extra_options)) { - childArgs.push(`${key}=${self.extra_options[key]}`); + for (const [key, value] of Object.entries(this.extra_options)) { + childArgs.push(`${key}=${value}`); } const child = child_process.fork(require.main.filename, childArgs, { env: childEnv, - execArgv: self.flags.concat(process.execArgv), + execArgv: this.flags.concat(process.execArgv), }); child.on('message', sendResult); child.on('close', (code) => { @@ -168,29 +204,31 @@ class Benchmark { process.exit(code); } - if (queueIndex + 1 < self.queue.length) { + if (queueIndex + 1 < this.queue.length) { recursive(queueIndex + 1); } }); - })(0); + }; + + recursive(0); } start() { - if (this._started) { + if (this.#started) { throw new Error('Called start more than once in a single benchmark'); } - this._started = true; - this._time = process.hrtime(); + this.#started = true; + this.#time = process.hrtime(); } end(operations) { // Get elapsed time now and do error checking later for accuracy. - const elapsed = process.hrtime(this._time); + const elapsed = process.hrtime(this.#time); - if (!this._started) { + if (!this.#started) { throw new Error('called end without start'); } - if (this._ended) { + if (this.#ended) { throw new Error('called end multiple times'); } if (typeof operations !== 'number') { @@ -206,7 +244,7 @@ class Benchmark { elapsed[1] = 1; } - this._ended = true; + this.#ended = true; const time = elapsed[0] + elapsed[1] / 1e9; const rate = operations / time; this.report(rate, elapsed); @@ -216,7 +254,7 @@ class Benchmark { sendResult({ name: this.name, conf: this.config, - rate: rate, + rate, time: elapsed[0] + elapsed[1] / 1e9, type: 'report', }); @@ -334,6 +372,7 @@ function bakeUrlData(type, e = 0, withBase = false, asUrl = false) { } module.exports = { + Benchmark, PORT: http_benchmarkers.PORT, bakeUrlData, binding(bindingName) { @@ -349,8 +388,6 @@ module.exports = { createBenchmark(fn, configs, options) { return new Benchmark(fn, configs, options); }, - // Benchmark an http server. - default_http_benchmarker: http_benchmarkers.default_http_benchmarker, sendResult, searchParams, urlDataTypes: Object.keys(urls).concat(['wpt']), diff --git a/benchmark/compare.js b/benchmark/compare.js index 53f82bb4b9f1b9..5c9cd03be3fdee 100644 --- a/benchmark/compare.js +++ b/benchmark/compare.js @@ -9,7 +9,7 @@ const BenchmarkProgress = require('./_benchmark_progress.js'); // // Parse arguments // -const cli = CLI(`usage: ./node compare.js [options] [--] ... +const cli = new CLI(`usage: ./node compare.js [options] [--] ... Run each benchmark in the directory many times using two different node versions. More than one directory can be specified. The output is formatted as csv, which can be processed using for diff --git a/benchmark/crypto/cipher-stream.js b/benchmark/crypto/cipher-stream.js index 4bb1695e2d20cc..9f4a1bb4fa3d40 100644 --- a/benchmark/crypto/cipher-stream.js +++ b/benchmark/crypto/cipher-stream.js @@ -7,6 +7,8 @@ const bench = common.createBenchmark(main, { type: ['asc', 'utf', 'buf'], len: [2, 1024, 102400, 1024 * 1024], api: ['legacy', 'stream'] +}, { + flags: ['--no-warnings'] }); function main({ api, cipher, type, len, writes }) { diff --git a/benchmark/fs/read-stream-throughput.js b/benchmark/fs/read-stream-throughput.js index 34c25760ea9c38..5984317ff91743 100644 --- a/benchmark/fs/read-stream-throughput.js +++ b/benchmark/fs/read-stream-throughput.js @@ -11,19 +11,18 @@ tmpdir.refresh(); const filename = path.resolve(tmpdir.path, `.removeme-benchmark-garbage-${process.pid}`); -let encodingType, encoding, size, filesize; - const bench = common.createBenchmark(main, { encodingType: ['buf', 'asc', 'utf'], - filesize: [1000 * 1024 * 1024], - size: [1024, 4096, 65535, 1024 * 1024] + filesize: [1000 * 1024], + highWaterMark: [1024, 4096, 65535, 1024 * 1024], + n: 1024 }); function main(conf) { - encodingType = conf.encodingType; - size = conf.size; - filesize = conf.filesize; + const { encodingType, highWaterMark, filesize } = conf; + let { n } = conf; + let encoding = ''; switch (encodingType) { case 'buf': encoding = null; @@ -38,34 +37,8 @@ function main(conf) { throw new Error(`invalid encodingType: ${encodingType}`); } - makeFile(); -} - -function runTest() { - assert(fs.statSync(filename).size === filesize); - const rs = fs.createReadStream(filename, { - highWaterMark: size, - encoding: encoding - }); - - rs.on('open', () => { - bench.start(); - }); - - let bytes = 0; - rs.on('data', (chunk) => { - bytes += chunk.length; - }); - - rs.on('end', () => { - try { fs.unlinkSync(filename); } catch {} - // MB/sec - bench.end(bytes / (1024 * 1024)); - }); -} - -function makeFile() { - const buf = Buffer.allocUnsafe(filesize / 1024); + // Make file + const buf = Buffer.allocUnsafe(filesize); if (encoding === 'utf8') { // ΓΌ for (let i = 0; i < buf.length; i++) { @@ -78,16 +51,38 @@ function makeFile() { } try { fs.unlinkSync(filename); } catch {} - let w = 1024; const ws = fs.createWriteStream(filename); - ws.on('close', runTest); + ws.on('close', runTest.bind(null, filesize, highWaterMark, encoding, n)); ws.on('drain', write); write(); function write() { do { - w--; - } while (false !== ws.write(buf) && w > 0); - if (w === 0) + n--; + } while (false !== ws.write(buf) && n > 0); + if (n === 0) ws.end(); } } + +function runTest(filesize, highWaterMark, encoding, n) { + assert(fs.statSync(filename).size === filesize * n); + const rs = fs.createReadStream(filename, { + highWaterMark, + encoding + }); + + rs.on('open', () => { + bench.start(); + }); + + let bytes = 0; + rs.on('data', (chunk) => { + bytes += chunk.length; + }); + + rs.on('end', () => { + try { fs.unlinkSync(filename); } catch {} + // MB/sec + bench.end(bytes / (1024 * 1024)); + }); +} diff --git a/benchmark/fs/readfile.js b/benchmark/fs/readfile.js index 361ffbff597d6e..3f996e02ede876 100644 --- a/benchmark/fs/readfile.js +++ b/benchmark/fs/readfile.js @@ -14,12 +14,12 @@ const filename = path.resolve(tmpdir.path, `.removeme-benchmark-garbage-${process.pid}`); const bench = common.createBenchmark(main, { - dur: [5], + duration: [5], len: [1024, 16 * 1024 * 1024], concurrent: [1, 10] }); -function main({ len, dur, concurrent }) { +function main({ len, duration, concurrent }) { try { fs.unlinkSync(filename); } catch {} let data = Buffer.alloc(len, 'x'); fs.writeFileSync(filename, data); @@ -33,7 +33,7 @@ function main({ len, dur, concurrent }) { bench.end(reads); try { fs.unlinkSync(filename); } catch {} process.exit(0); - }, dur * 1000); + }, duration * 1000); function read() { fs.readFile(filename, afterRead); diff --git a/benchmark/http/chunked.js b/benchmark/http/chunked.js index 52b4605715c322..9ae7bb7495f29a 100644 --- a/benchmark/http/chunked.js +++ b/benchmark/http/chunked.js @@ -13,10 +13,11 @@ const common = require('../common.js'); const bench = common.createBenchmark(main, { n: [1, 4, 8, 16], len: [1, 64, 256], - c: [100] + c: [100], + duration: 5 }); -function main({ len, n, c }) { +function main({ len, n, c, duration }) { const http = require('http'); const chunk = Buffer.alloc(len, '8'); @@ -33,7 +34,8 @@ function main({ len, n, c }) { server.listen(common.PORT, () => { bench.http({ - connections: c + connections: c, + duration }, () => { server.close(); }); diff --git a/benchmark/http/cluster.js b/benchmark/http/cluster.js index 3bcd061a0894c5..0d97b516ec506b 100644 --- a/benchmark/http/cluster.js +++ b/benchmark/http/cluster.js @@ -9,14 +9,15 @@ if (cluster.isMaster) { // Unicode confuses ab on os x. type: ['bytes', 'buffer'], len: [4, 1024, 102400], - c: [50, 500] + c: [50, 500], + duration: 5, }); } else { const port = parseInt(process.env.PORT || PORT); require('../fixtures/simple-http-server.js').listen(port); } -function main({ type, len, c }) { +function main({ type, len, c, duration }) { process.env.PORT = PORT; let workers = 0; const w1 = cluster.fork(); @@ -32,7 +33,8 @@ function main({ type, len, c }) { bench.http({ path: path, - connections: c + connections: c, + duration }, () => { w1.destroy(); w2.destroy(); diff --git a/benchmark/http/end-vs-write-end.js b/benchmark/http/end-vs-write-end.js index 38e9b89a97a6b4..60174ef3adf4f2 100644 --- a/benchmark/http/end-vs-write-end.js +++ b/benchmark/http/end-vs-write-end.js @@ -14,10 +14,11 @@ const bench = common.createBenchmark(main, { type: ['asc', 'utf', 'buf'], len: [64 * 1024, 128 * 1024, 256 * 1024, 1024 * 1024], c: [100], - method: ['write', 'end'] + method: ['write', 'end'], + duration: 5 }); -function main({ len, type, method, c }) { +function main({ len, type, method, c, duration }) { const http = require('http'); let chunk; switch (type) { @@ -49,7 +50,8 @@ function main({ len, type, method, c }) { server.listen(common.PORT, () => { bench.http({ - connections: c + connections: c, + duration }, () => { server.close(); }); diff --git a/benchmark/http/headers.js b/benchmark/http/headers.js index f8014a6a085d19..b83ac17e742a2e 100644 --- a/benchmark/http/headers.js +++ b/benchmark/http/headers.js @@ -6,9 +6,10 @@ const http = require('http'); const bench = common.createBenchmark(main, { n: [10, 1000], len: [1, 100], + duration: 5 }); -function main({ len, n }) { +function main({ len, n, duration }) { const headers = { 'Connection': 'keep-alive', 'Transfer-Encoding': 'chunked', @@ -29,7 +30,8 @@ function main({ len, n }) { server.listen(common.PORT, () => { bench.http({ path: '/', - connections: 10 + connections: 10, + duration }, () => { server.close(); }); diff --git a/benchmark/http/incoming_headers.js b/benchmark/http/incoming_headers.js index 810c92687bd981..983bd5632fcb7d 100644 --- a/benchmark/http/incoming_headers.js +++ b/benchmark/http/incoming_headers.js @@ -3,12 +3,13 @@ const common = require('../common.js'); const http = require('http'); const bench = common.createBenchmark(main, { - c: [50], // Concurrent connections - n: [20], // Number of header lines to append after the common headers - w: [0, 6], // Amount of trailing whitespace + connections: [50], // Concurrent connections + headers: [20], // Number of header lines to append after the common headers + w: [0, 6], // Amount of trailing whitespace + duration: 5 }); -function main({ c, n, w }) { +function main({ connections, headers, w, duration }) { const server = http.createServer((req, res) => { res.end(); }); @@ -21,7 +22,7 @@ function main({ c, n, w }) { 'Date': new Date().toString(), 'Cache-Control': 'no-cache' }; - for (let i = 0; i < n; i++) { + for (let i = 0; i < headers; i++) { // Note: // - autocannon does not send header values with OWS // - wrk can only send trailing OWS. This is a side-effect of wrk @@ -31,8 +32,9 @@ function main({ c, n, w }) { } bench.http({ path: '/', - connections: c, - headers + connections, + headers, + duration }, () => { server.close(); }); diff --git a/benchmark/http/set-header.js b/benchmark/http/set-header.js index 1909c0991dfc71..48e0163a6ced10 100644 --- a/benchmark/http/set-header.js +++ b/benchmark/http/set-header.js @@ -3,7 +3,8 @@ const common = require('../common.js'); const PORT = common.PORT; const bench = common.createBenchmark(main, { - res: ['normal', 'setHeader', 'setHeaderWH'] + res: ['normal', 'setHeader', 'setHeaderWH'], + duration: 5 }); const type = 'bytes'; @@ -15,16 +16,17 @@ const c = 50; // normal: writeHead(status, {...}) // setHeader: statusCode = status, setHeader(...) x2 // setHeaderWH: setHeader(...), writeHead(status, ...) -function main({ res }) { +function main({ res, duration }) { process.env.PORT = PORT; const server = require('../fixtures/simple-http-server.js') .listen(PORT) .on('listening', () => { - const path = `/${type}/${len}/${chunks}/normal/${chunkedEnc}`; + const path = `/${type}/${len}/${chunks}/${res}/${chunkedEnc}`; bench.http({ path: path, - connections: c + connections: c, + duration }, () => { server.close(); }); diff --git a/benchmark/http/simple.js b/benchmark/http/simple.js index 95409faa9a869c..095b15ca4465fb 100644 --- a/benchmark/http/simple.js +++ b/benchmark/http/simple.js @@ -7,18 +7,20 @@ const bench = common.createBenchmark(main, { len: [4, 1024, 102400], chunks: [1, 4], c: [50, 500], - chunkedEnc: [1, 0] + chunkedEnc: [1, 0], + duration: 5 }); -function main({ type, len, chunks, c, chunkedEnc, res }) { +function main({ type, len, chunks, c, chunkedEnc, duration }) { const server = require('../fixtures/simple-http-server.js') .listen(common.PORT) .on('listening', () => { const path = `/${type}/${len}/${chunks}/normal/${chunkedEnc}`; bench.http({ - path: path, - connections: c + path, + connections: c, + duration }, () => { server.close(); }); diff --git a/benchmark/http2/compat.js b/benchmark/http2/compat.js index 5d06ccf3178257..2c7e732b07f0a5 100644 --- a/benchmark/http2/compat.js +++ b/benchmark/http2/compat.js @@ -9,10 +9,11 @@ const bench = common.createBenchmark(main, { requests: [100, 1000, 5000], streams: [1, 10, 20, 40, 100, 200], clients: [2], - benchmarker: ['h2load'] + benchmarker: ['test-double-http2'], + duration: 5 }, { flags: ['--no-warnings'] }); -function main({ requests, streams, clients }) { +function main({ requests, streams, clients, duration }) { const http2 = require('http2'); const server = http2.createServer(); server.on('request', (req, res) => { @@ -29,7 +30,8 @@ function main({ requests, streams, clients }) { requests, maxConcurrentStreams: streams, clients, - threads: clients + threads: clients, + duration }, () => { server.close(); }); }); } diff --git a/benchmark/http2/respond-with-fd.js b/benchmark/http2/respond-with-fd.js index 35856490f7e4a2..5bf5988d16a64c 100644 --- a/benchmark/http2/respond-with-fd.js +++ b/benchmark/http2/respond-with-fd.js @@ -10,10 +10,11 @@ const bench = common.createBenchmark(main, { requests: [100, 1000, 5000], streams: [1, 10, 20, 40, 100, 200], clients: [2], - benchmarker: ['h2load'] + benchmarker: ['test-double-http2'], + duration: 5 }, { flags: ['--no-warnings'] }); -function main({ requests, streams, clients }) { +function main({ requests, streams, clients, duration }) { fs.open(file, 'r', (err, fd) => { if (err) throw err; @@ -30,6 +31,7 @@ function main({ requests, streams, clients }) { requests, maxConcurrentStreams: streams, clients, + duration, threads: clients }, () => server.close()); }); diff --git a/benchmark/http2/simple.js b/benchmark/http2/simple.js index aab7c6b609b715..929c4c655e1295 100644 --- a/benchmark/http2/simple.js +++ b/benchmark/http2/simple.js @@ -9,10 +9,11 @@ const bench = common.createBenchmark(main, { requests: [100, 1000, 5000], streams: [1, 10, 20, 40, 100, 200], clients: [2], - benchmarker: ['h2load'] + benchmarker: ['test-double-http2'], + duration: 5 }, { flags: ['--no-warnings'] }); -function main({ requests, streams, clients }) { +function main({ requests, streams, clients, duration }) { const http2 = require('http2'); const server = http2.createServer(); server.on('stream', (stream) => { @@ -27,6 +28,7 @@ function main({ requests, streams, clients }) { requests, maxConcurrentStreams: streams, clients, + duration, threads: clients }, () => { server.close(); }); }); diff --git a/benchmark/http2/write.js b/benchmark/http2/write.js index fc3203c6e55451..7ea8b2c02da650 100644 --- a/benchmark/http2/write.js +++ b/benchmark/http2/write.js @@ -6,10 +6,11 @@ const bench = common.createBenchmark(main, { streams: [100, 200, 1000], length: [64 * 1024, 128 * 1024, 256 * 1024, 1024 * 1024], size: [100000], - benchmarker: ['h2load'] + benchmarker: ['test-double-http2'], + duration: 5 }, { flags: ['--no-warnings'] }); -function main({ streams, length, size }) { +function main({ streams, length, size, duration }) { const http2 = require('http2'); const server = http2.createServer(); server.on('stream', (stream) => { @@ -29,6 +30,7 @@ function main({ streams, length, size }) { bench.http({ path: '/', requests: 10000, + duration, maxConcurrentStreams: streams, }, () => { server.close(); }); }); diff --git a/benchmark/misc/trace.js b/benchmark/misc/trace.js index bdbf547007e72e..f06e8597cc30e9 100644 --- a/benchmark/misc/trace.js +++ b/benchmark/misc/trace.js @@ -6,7 +6,11 @@ const bench = common.createBenchmark(main, { n: [100000], method: ['trace', 'isTraceCategoryEnabled'] }, { - flags: ['--expose-internals', '--trace-event-categories', 'foo'] + flags: [ + '--expose-internals', + '--no-warnings', + '--trace-event-categories', 'foo', + ] }); const { diff --git a/benchmark/net/net-c2s.js b/benchmark/net/net-c2s.js index cacd6815630b2e..424c8f6dd072b3 100644 --- a/benchmark/net/net-c2s.js +++ b/benchmark/net/net-c2s.js @@ -9,6 +9,8 @@ const bench = common.createBenchmark(main, { len: [64, 102400, 1024 * 1024 * 16], type: ['utf', 'asc', 'buf'], dur: [5], +}, { + test: { len: 1024 } }); let chunk; diff --git a/benchmark/net/net-pipe.js b/benchmark/net/net-pipe.js index d86ff73041d845..32e1085299a2ce 100644 --- a/benchmark/net/net-pipe.js +++ b/benchmark/net/net-pipe.js @@ -9,6 +9,8 @@ const bench = common.createBenchmark(main, { len: [2, 64, 102400, 1024 * 1024 * 16], type: ['utf', 'asc', 'buf'], dur: [5], +}, { + test: { len: 1024 } }); let chunk; diff --git a/benchmark/net/net-s2c.js b/benchmark/net/net-s2c.js index 789eadf0a18dba..835cc67567bcf2 100644 --- a/benchmark/net/net-s2c.js +++ b/benchmark/net/net-s2c.js @@ -10,6 +10,8 @@ const bench = common.createBenchmark(main, { recvbuflen: [0, 64 * 1024, 1024 * 1024], recvbufgenfn: ['true', 'false'], dur: [5] +}, { + test: { sendchunklen: 256 } }); let chunk; diff --git a/benchmark/net/net-wrap-js-stream-passthrough.js b/benchmark/net/net-wrap-js-stream-passthrough.js index 0d7be36c6aa545..3824cfb9c0e03c 100644 --- a/benchmark/net/net-wrap-js-stream-passthrough.js +++ b/benchmark/net/net-wrap-js-stream-passthrough.js @@ -9,6 +9,7 @@ const bench = common.createBenchmark(main, { type: ['utf', 'asc', 'buf'], dur: [5], }, { + test: { len: 64 }, flags: ['--expose-internals'] }); diff --git a/benchmark/net/tcp-raw-c2s.js b/benchmark/net/tcp-raw-c2s.js index b8af124a7f40fc..9547c01f38bc32 100644 --- a/benchmark/net/tcp-raw-c2s.js +++ b/benchmark/net/tcp-raw-c2s.js @@ -12,7 +12,10 @@ const bench = common.createBenchmark(main, { len: [102400, 1024 * 1024 * 16], type: ['utf', 'asc', 'buf'], dur: [5] -}, { flags: [ '--expose-internals', '--no-warnings' ] }); +}, { + test: { len: 1024 }, + flags: [ '--expose-internals', '--no-warnings' ] +}); function main({ dur, len, type }) { const { diff --git a/benchmark/net/tcp-raw-pipe.js b/benchmark/net/tcp-raw-pipe.js index 249b61046a84cf..e422ff749fd545 100644 --- a/benchmark/net/tcp-raw-pipe.js +++ b/benchmark/net/tcp-raw-pipe.js @@ -13,6 +13,7 @@ const bench = common.createBenchmark(main, { type: ['utf', 'asc', 'buf'], dur: [5] }, { + test: { len: 1024 }, flags: [ '--expose-internals', '--no-warnings' ] }); diff --git a/benchmark/net/tcp-raw-s2c.js b/benchmark/net/tcp-raw-s2c.js index 393cf060489cb8..be7279ca0c315c 100644 --- a/benchmark/net/tcp-raw-s2c.js +++ b/benchmark/net/tcp-raw-s2c.js @@ -13,6 +13,7 @@ const bench = common.createBenchmark(main, { type: ['utf', 'asc', 'buf'], dur: [5] }, { + test: { len: 1024 }, flags: [ '--expose-internals', '--no-warnings' ] }); diff --git a/benchmark/run.js b/benchmark/run.js index 8e81a2c5e16ab7..c2e38ce96d7895 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -4,7 +4,7 @@ const path = require('path'); const fork = require('child_process').fork; const CLI = require('./_cli.js'); -const cli = CLI(`usage: ./node run.js [options] [--] ... +const cli = new CLI(`usage: ./node run.js [options] [--] ... Run each benchmark in the directory a single time, more than one directory can be specified. @@ -14,6 +14,9 @@ const cli = CLI(`usage: ./node run.js [options] [--] ... repeated) --set variable=value set benchmark variable (can be repeated) --format [simple|csv] optional value that specifies the output format + test only run a single configuration from the options + matrix + all each benchmark category is run one after the other `, { arrayArgs: ['set', 'filter', 'exclude'] }); const benchmarks = cli.benchmarks(); @@ -37,7 +40,11 @@ if (format === 'csv') { (function recursive(i) { const filename = benchmarks[i]; - const child = fork(path.resolve(__dirname, filename), cli.optional.set); + const child = fork( + path.resolve(__dirname, filename), + cli.test ? ['--test'] : [], + cli.optional.set + ); if (format !== 'csv') { console.log(); @@ -51,10 +58,10 @@ if (format === 'csv') { // Construct configuration string, " A=a, B=b, ..." let conf = ''; for (const key of Object.keys(data.conf)) { - conf += ` ${key}=${JSON.stringify(data.conf[key])}`; + if (conf !== '') + conf += ' '; + conf += `${key}=${JSON.stringify(data.conf[key])}`; } - // Delete first space of the configuration - conf = conf.slice(1); if (format === 'csv') { // Escape quotes (") for correct csv formatting conf = conf.replace(/"/g, '""'); diff --git a/benchmark/scatter.js b/benchmark/scatter.js index 10649e6bb51e97..ecbf8e0041c837 100644 --- a/benchmark/scatter.js +++ b/benchmark/scatter.js @@ -7,7 +7,7 @@ const CLI = require('./_cli.js'); // // Parse arguments // -const cli = CLI(`usage: ./node scatter.js [options] [--] +const cli = new CLI(`usage: ./node scatter.js [options] [--] Run the benchmark script many times and output the rate (ops/s) together with the benchmark variables as a csv. diff --git a/benchmark/tls/secure-pair.js b/benchmark/tls/secure-pair.js index bb7933d837f999..76658fc3c42ad7 100644 --- a/benchmark/tls/secure-pair.js +++ b/benchmark/tls/secure-pair.js @@ -4,6 +4,8 @@ const bench = common.createBenchmark(main, { dur: [5], securing: ['SecurePair', 'TLSSocket', 'clear'], size: [100, 1024, 1024 * 1024] +}, { + flags: ['--no-warnings'] }); const fixtures = require('../../test/common/fixtures'); diff --git a/benchmark/util/type-check.js b/benchmark/util/type-check.js index 5b992e729e63ae..8d57bc000afcd1 100644 --- a/benchmark/util/type-check.js +++ b/benchmark/util/type-check.js @@ -31,7 +31,7 @@ const bench = common.createBenchmark(main, { argument: ['true', 'false-primitive', 'false-object'], n: [1e5] }, { - flags: ['--expose-internals'] + flags: ['--expose-internals', '--no-warnings'] }); function main({ type, argument, version, n }) { diff --git a/benchmark/zlib/pipe.js b/benchmark/zlib/pipe.js index 6a1c427bc8380b..76b0ddc6c65a25 100644 --- a/benchmark/zlib/pipe.js +++ b/benchmark/zlib/pipe.js @@ -8,6 +8,11 @@ const bench = common.createBenchmark(main, { duration: [5], type: ['string', 'buffer'], algorithm: ['gzip', 'brotli'] +}, { + test: { + inputLen: 1024, + duration: 0.2 + } }); function main({ inputLen, duration, type, algorithm }) { diff --git a/doc/guides/writing-and-running-benchmarks.md b/doc/guides/writing-and-running-benchmarks.md index 1db72d22de5324..0075023ce80fa8 100644 --- a/doc/guides/writing-and-running-benchmarks.md +++ b/doc/guides/writing-and-running-benchmarks.md @@ -519,7 +519,8 @@ const common = require('../common.js'); const bench = common.createBenchmark(main, { kb: [64, 128, 256, 1024], - connections: [100, 500] + connections: [100, 500], + duration: 5 }); function main(conf) { @@ -546,8 +547,8 @@ Supported options keys are: * `path` - defaults to `/` * `connections` - number of concurrent connections to use, defaults to 100 * `duration` - duration of the benchmark in seconds, defaults to 10 -* `benchmarker` - benchmarker to use, defaults to -`common.default_http_benchmarker` +* `benchmarker` - benchmarker to use, defaults to the first available http + benchmarker [autocannon]: https://github.com/mcollina/autocannon [wrk]: https://github.com/wg/wrk diff --git a/test/common/benchmark.js b/test/common/benchmark.js index f630bb9d0e6fd8..56351c92505efa 100644 --- a/test/common/benchmark.js +++ b/test/common/benchmark.js @@ -8,13 +8,8 @@ const path = require('path'); const runjs = path.join(__dirname, '..', '..', 'benchmark', 'run.js'); -function runBenchmark(name, args, env) { - const argv = []; - - for (let i = 0; i < args.length; i++) { - argv.push('--set'); - argv.push(args[i]); - } +function runBenchmark(name, env) { + const argv = ['test']; argv.push(name);