test: expand test coverage of node:bench

Signed-off-by: James M Snell <jasnell@gmail.com>
Assisted-by: Opencode
PR-URL: #65631
Reviewed-By: Filip Skokan <panva.ip@gmail.com>

Author: jasnell

test/fixtures/bench-runner/malformed-record.cjs

@@ -41,12 +41,24 @@ const record = kind === 'summary' ? {
     runId: process.env.NODE_BENCH_RUN_ID,
     success: true,
   },
+} : kind === 'name-path' ? {
+  ...plan,
+  data: {
+    ...plan.data,
+    namePath: [1],
+  },
 } : kind === 'plan' ? {
   ...plan,
   data: {
     ...plan.data,
     samples: 0,
   },
+} : kind === 'timeout' ? {
+  ...plan,
+  data: {
+    ...plan.data,
+    timeout: -1,
+  },
 } : kind === 'identity' ? {
   type: 'bench:complete',
   data: {

test/fixtures/bench-runner/run-file.cjs

@@ -7,6 +7,7 @@ bench('run file', {
   params: {
     context: process.env.NODE_BENCH_CONTEXT,
     exposed: typeof globalThis.gc === 'function',
+    omitted: process.env.NODE_BENCH_OMITTED === undefined,
     value: process.env.NODE_BENCH_RUN_FILE ?? 'unset',
   },
 }, (b) => {

test/parallel/test-bench-cli.js

@@ -488,7 +488,9 @@ for (const { kind, message } of [
   { kind: 'sequence', message: /valid record sequence/ },
   { kind: 'record', message: /not a valid benchmark record/ },
   { kind: 'identity', message: /not a valid benchmark record/ },
+  { kind: 'name-path', message: /not a valid benchmark record/ },
   { kind: 'plan', message: /not a valid benchmark plan/ },
+  { kind: 'timeout', message: /not a valid benchmark plan/ },
   { kind: 'diagnostic', message: /not a valid benchmark diagnostic/ },
   { kind: 'diagnostic-order', message: /valid lifecycle sequence/ },
   { kind: 'summary', message: /not a valid benchmark summary/ },
@@ -525,6 +527,22 @@ for (const { kind, message } of [
   assert.strictEqual(records.at(-1).data.success, false);
 }
 
+{
+  const result = spawnBench([
+    '--bench-isolation=none',
+    '--bench-reporter=json',
+    fixtures.path('bench-runner/a.cjs'),
+    fixtures.path('bench-runner/exit-code.cjs'),
+  ]);
+  assert.strictEqual(result.status, 1);
+  const records = parseRecords(result);
+  const diagnostic = records.find(({ type, data }) =>
+    type === 'bench:diagnostic' && /set exit code/.test(data.message)).data;
+  assert.strictEqual(diagnostic.entryFile, null);
+  assert.strictEqual(diagnostic.fileRunId, null);
+  assert.strictEqual(diagnostic.file, null);
+}
+
 for (const { mode, message } of [
   { mode: 'code', message: /failed with exit code 2/ },
   { mode: 'late', message: /failed with exit code 2/ },

test/parallel/test-bench-errors.js

@@ -3,7 +3,7 @@
 
 const common = require('../common');
 const assert = require('assert');
-const { bench, run } = require('node:bench');
+const { bench, run, suite } = require('node:bench');
 const { setTimeout } = require('timers/promises');
 
 const options = { samples: 1 };
@@ -58,8 +58,11 @@ bench('continues', options, complete);
 bench('timeout', { samples: 1, timeout: 10 }, async () => {
   await new Promise(() => {});
 });
-bench('after unsettled timeout', options, common.mustNotCall());
-bench.skip('skipped after unsettled timeout', options, common.mustNotCall());
+const suiteCompletion = suite('after unsettled timeout suite', () => {
+  bench('after unsettled timeout', options, common.mustNotCall());
+  bench.skip('skipped after unsettled timeout', options, common.mustNotCall());
+});
+suiteCompletion.then(common.mustCall());
 
 const completions = [];
 const sampleNames = [];

test/parallel/test-bench-harness-errors.js

@@ -62,17 +62,20 @@ async function testRunSignal() {
 async function testRunSignalAfterSample() {
   const runner = createRunner({ yieldBetweenSamples: false });
   const controller = new AbortController();
+  const reason = new Error('sample aborted');
   const completion = runner.bench('aborted after sample', {
-    samples: 1,
-  }, (b) => {
-    complete(b);
-    controller.abort(new Error('sample aborted'));
-  });
-  await runner.run({ signal: controller.signal }).toArray();
+    samples: 2,
+  }, complete);
+  const stream = runner.run({ signal: controller.signal });
+  stream.once('bench:sample', common.mustCall(() => {
+    controller.abort(reason);
+  }));
+  const records = await stream.toArray();
   const result = await completion;
-  await setImmediate();
   assert.strictEqual(result.error.code, 'ABORT_ERR');
-  assert.strictEqual(result.error.cause.message, 'sample aborted');
+  assert.strictEqual(result.error.cause, reason);
+  assert.strictEqual(result.samples.length, 1);
+  assert.strictEqual(records.at(-1).data.success, false);
 }
 
 async function testStringNamePattern() {
@@ -93,12 +96,15 @@ async function testStringNamePattern() {
 
 async function testTopLevelRecovery() {
   const runner = createRunner({ yieldBetweenSamples: false });
-  runner.bench('listener failure', { samples: 1 }, complete);
+  const suiteCompletion = runner.suite('nested', () => {
+    runner.bench('listener failure', { samples: 1 }, complete);
+  });
   const stream = runner.run();
   const failure = new Error();
   failure.message = undefined;
   stream.on('bench:start', common.mustCall(() => { throw failure; }));
   const records = await stream.toArray();
+  await suiteCompletion;
   const diagnostic = records.find(
     ({ type }) => type === 'bench:diagnostic').data;
   const summary = records.find(({ type }) => type === 'bench:summary').data;
@@ -110,10 +116,31 @@ async function testTopLevelRecovery() {
   assert.strictEqual(summary.success, false);
 }
 
+async function testRepeatedReportingFailure() {
+  const runner = createRunner({ yieldBetweenSamples: false });
+  const original = new Error('start listener failed');
+  const diagnostic = new Error('diagnostic listener failed');
+  const summary = new Error('summary listener failed');
+  const completion = runner.bench('reporting failures', {
+    samples: 1,
+  }, complete);
+  const stream = runner.run();
+  stream.on('bench:start', common.mustCall(() => { throw original; }));
+  stream.on('bench:diagnostic', common.mustCall(() => {
+    throw diagnostic;
+  }, 2));
+  stream.on('bench:summary', common.mustCall(() => { throw summary; }));
+  const ended = new Promise((resolve) => stream.once('end', resolve));
+  stream.resume();
+  await ended;
+  assert.strictEqual((await completion).error, original);
+}
+
 (async () => {
   await testSynchronousSuiteFailure();
   await testRunSignal();
   await testRunSignalAfterSample();
   await testStringNamePattern();
   await testTopLevelRecovery();
+  await testRepeatedReportingFailure();
 })().then(common.mustCall());

test/parallel/test-bench-run-file.js

@@ -29,9 +29,15 @@ assert.throws(() => runFile(fixture, { execArgv: [fixture] }), {
 assert.throws(() => runFile(fixture, { execArgv: ['-e', '0'] }), {
   code: 'ERR_INVALID_ARG_VALUE',
 });
+assert.throws(() => runFile(fixture, { execArgv: ['-e0'] }), {
+  code: 'ERR_INVALID_ARG_VALUE',
+});
 assert.throws(() => runFile(fixture, { execArgv: ['--require'] }), {
   code: 'ERR_INVALID_ARG_VALUE',
 });
+assert.throws(() => runFile(fixture, { execArgv: ['--require', ''] }), {
+  code: 'ERR_INVALID_ARG_VALUE',
+});
 assert.throws(() => runFile(fixture, { execArgv: ['--require='] }), {
   code: 'ERR_INVALID_ARG_VALUE',
 });
@@ -73,11 +79,12 @@ assert.throws(() => runFile(fixture, { signal: { aborted: false } }), {
 });
 
 async function testRunFile() {
-  const execArgv = ['--expose-gc', '-r', 'fs'];
+  const execArgv = ['--expose-gc', '--no-warnings', '-r', 'fs'];
   const env = {
     __proto__: null,
     ...process.env,
     NODE_BENCH_CONTEXT: 'not-a-child',
+    NODE_BENCH_OMITTED: undefined,
     NODE_BENCH_RUN_FILE: 'original',
     NODE_CHANNEL_FD: '999',
     NODE_CHANNEL_SERIALIZATION_MODE: 'json',
@@ -93,12 +100,14 @@ async function testRunFile() {
   assert.strictEqual(records.at(-1).type, 'bench:summary');
   assert.strictEqual(plan.params.context, 'child');
   assert.strictEqual(plan.params.exposed, true);
+  assert.strictEqual(plan.params.omitted, true);
   assert.strictEqual(plan.params.value, 'original');
   assert.strictEqual(result.error, undefined);
   assert.strictEqual(result.samples.length, 1);
   assert.strictEqual(typeof result.samples[0].duration_ns, 'bigint');
   assert.notStrictEqual(result.samples[0].detail.pid, process.pid);
   assert(result.samples[0].detail.execArgv.includes('--expose-gc'));
+  assert(result.samples[0].detail.execArgv.includes('--no-warnings'));
   assert(result.samples[0].detail.execArgv.includes('-r'));
   assert.strictEqual(summary.success, true);
   assert.strictEqual(summary.file, fixture);
@@ -258,13 +267,16 @@ async function testParentExitCode() {
 }
 
 async function testDefaultExecArgvSnapshot() {
-  const stream = runFile(fixture);
-  process.execArgv.push('--require=/does/not/exist.cjs');
+  const original = Array.from(process.execArgv);
   try {
+    process.execArgv.push('--bench', '--eval', 'throw new Error()');
+    const stream = runFile(fixture);
+    process.execArgv.push('--require=/does/not/exist.cjs');
     const records = await stream.toArray();
     assert.strictEqual(records.at(-1).data.success, true);
   } finally {
-    process.execArgv.pop();
+    process.execArgv.length = 0;
+    process.execArgv.push(...original);
   }
 }
 
@@ -280,6 +292,22 @@ async function testExecPathSnapshot() {
   }
 }
 
+async function testSpawnFailure() {
+  const execPath = process.execPath;
+  let stream;
+  try {
+    process.execPath = fixtures.path('does-not-exist-node');
+    stream = runFile(fixture);
+  } finally {
+    process.execPath = execPath;
+  }
+  const records = await stream.toArray();
+  const diagnostic = records.find(
+    ({ type }) => type === 'bench:diagnostic').data;
+  assert.strictEqual(diagnostic.error.code, 'ENOENT');
+  assert.strictEqual(records.at(-1).data.success, false);
+}
+
 function testEvalParent() {
   const script = `
     require('node:bench').runFile(${JSON.stringify(fixture)})
@@ -394,6 +422,7 @@ async function testDestroy() {
   await testParentExitCode();
   await testDefaultExecArgvSnapshot();
   await testExecPathSnapshot();
+  await testSpawnFailure();
   await testPreAborted();
   await testDestroy();
   testEvalParent();

test/parallel/test-bench-stream.js

@@ -81,6 +81,34 @@ async function testPlanBackpressure() {
   );
 }
 
+async function testDestroyWhileBlocked() {
+  const runner = createRunner({ yieldBetweenSamples: false });
+  const completions = [];
+  for (let i = 0; i < 32; i++) {
+    completions.push(runner.bench(`destroyed ${i}`, {
+      samples: 1,
+    }, recordSample));
+  }
+  const stream = runner.run();
+  const unblocked = stream.waitForDrain();
+  const iterator = stream[Symbol.asyncIterator]();
+  await iterator.next();
+  await unblocked;
+  for (let i = 0; i < 100 &&
+       stream.readableLength < stream.readableHighWaterMark; i++) {
+    await setImmediate();
+  }
+  assert.strictEqual(stream.readableLength, stream.readableHighWaterMark);
+
+  const draining = stream.waitForDrain();
+  const closed = new Promise((resolve) => stream.once('close', resolve));
+  stream.destroy();
+  await assert.rejects(draining, { code: 'ERR_INVALID_STATE' });
+  await assert.rejects(stream.waitForDrain(), { code: 'ERR_INVALID_STATE' });
+  await closed;
+  await Promise.all(completions);
+}
+
 async function testNamedEventsWithoutReading() {
   const runner = createRunner();
   const sampleCount = 64;
@@ -192,7 +220,13 @@ async function testRecordOwnership() {
   expectedError.context = {
     note: 'preserved',
     callback() {},
+    get accessor() { return 'value'; },
+  };
+  expectedError.customContext = {
+    __proto__: {},
+    note: 'preserved',
   };
+  expectedError.arrayPayload = [new WeakMap()];
   const innerError = new Error('inner failure');
   innerError.code = 'ERR_INNER';
   const aggregate = new AggregateError([innerError], 'aggregate failure');
@@ -217,6 +251,16 @@ async function testRecordOwnership() {
   const caused = runner.bench('owned cause', { samples: 1 }, () => {
     throw causedError;
   });
+  const throwingNameError = new Error('throwing name');
+  Object.defineProperty(throwingNameError, 'name', {
+    configurable: true,
+    get() { throw new Error('name getter'); },
+  });
+  const throwingName = runner.bench('throwing name', {
+    samples: 1,
+  }, () => {
+    throw throwingNameError;
+  });
   const thrownValue = new WeakMap();
   const uncloneable = runner.bench('uncloneable error', {
     samples: 1,
@@ -265,6 +309,7 @@ async function testRecordOwnership() {
   const measuredResult = await measured;
   const failedResult = await failed;
   await caused;
+  const throwingNameResult = await throwingName;
   const uncloneableResult = await uncloneable;
   const trappedResult = await trapped;
   const afterTrapResult = await afterTrap;
@@ -278,6 +323,8 @@ async function testRecordOwnership() {
     ({ name }) => name === 'owned error');
   const streamCaused = streamResults.find(
     ({ name }) => name === 'owned cause');
+  const streamThrowingName = streamResults.find(
+    ({ name }) => name === 'throwing name');
   const streamSummary = records.find(
     ({ type }) => type === 'bench:summary').data;
 
@@ -299,12 +346,18 @@ async function testRecordOwnership() {
   assert.strictEqual(streamFailed.error.cause, streamFailed.error);
   assert.strictEqual(streamFailed.error.context.note, 'preserved');
   assert.strictEqual(streamFailed.error.context.callback, undefined);
+  assert.strictEqual(streamFailed.error.context.accessor, undefined);
+  assert.strictEqual(streamFailed.error.customContext.note, 'preserved');
+  assert.deepStrictEqual(streamFailed.error.arrayPayload, [undefined]);
   assert.strictEqual(failedResult.error, expectedError);
   assert.strictEqual(failedResult.error.code, 'ERR_EXPECTED');
   assert.strictEqual(failedResult.error.cause, failedResult.error);
   assert.strictEqual(uncloneableResult.error, thrownValue);
   assert.strictEqual(trappedResult.error, proxyError);
   assert.strictEqual(afterTrapResult.error, undefined);
+  assert.strictEqual(throwingNameResult.error, throwingNameError);
+  assert.strictEqual(streamThrowingName.error.name, 'Error');
+  assert.strictEqual(streamThrowingName.error.message, 'throwing name');
   assert(streamCaused.error.cause instanceof AggregateError);
   assert.strictEqual(streamCaused.error.cause.name, 'AggregateError');
   assert.strictEqual(streamCaused.error.cause.code, 'ERR_AGGREGATE');
@@ -313,12 +366,13 @@ async function testRecordOwnership() {
     streamCaused.error.references.get('self'), streamCaused.error);
   assert.strictEqual(streamCaused.error.members.has(streamCaused.error), true);
   assert.notStrictEqual(eventSummary, streamSummary);
-  assert.strictEqual(streamSummary.counts.total, 6);
+  assert.strictEqual(streamSummary.counts.total, 7);
 }
 
 (async () => {
   await testReadableBackpressure();
   await testPlanBackpressure();
+  await testDestroyWhileBlocked();
   await testNamedEventsWithoutReading();
   await testCancellationCompletesBenchmarks();
   await testDeliveryDoesNotConsumeTimeout();