diff --git a/doc/api/test.md b/doc/api/test.md index a0c2462b1d700d..bfcc7c3f9d121d 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -233,7 +233,8 @@ If Node.js is started with the [`--test-only`][] command-line option, it is possible to skip all top level tests except for a selected subset by passing the `only` option to the tests that should be run. When a test with the `only` option set is run, all subtests are also run. The test context's `runOnly()` -method can be used to implement the same behavior at the subtest level. +method can be used to implement the same behavior at the subtest level. Tests +that are not executed are omitted from the test runner output. ```js // Assume Node.js is run with the --test-only command-line option. @@ -270,7 +271,7 @@ whose name matches the provided pattern. Test name patterns are interpreted as JavaScript regular expressions. The `--test-name-pattern` option can be specified multiple times in order to run nested tests. For each test that is executed, any corresponding test hooks, such as `beforeEach()`, are also -run. +run. Tests that are not executed are omitted from the test runner output. Given the following test file, starting Node.js with the `--test-name-pattern="test [1-3]"` option would cause the test runner to execute diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 9f9c47edae7f62..a5b7714ad59d82 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -86,6 +86,7 @@ const { } = parseCommandLine(); let kResistStopPropagation; let findSourceMap; +let noopTestStream; function lazyFindSourceMap(file) { if (findSourceMap === undefined) { @@ -252,13 +253,20 @@ class Test extends AsyncResource { parent = null; } + this.name = name; + this.parent = parent; + this.testNumber = 0; + this.outputSubtestCount = 0; + this.filteredSubtestCount = 0; + this.filtered = false; + if (parent === null) { this.concurrency = 1; this.nesting = 0; this.only = testOnlyFlag; this.reporter = new TestsStream(); this.runOnlySubtests = this.only; - this.testNumber = 0; + this.childNumber = 0; this.timeout = kDefaultTimeout; this.root = this; this.hooks = { @@ -277,7 +285,7 @@ class Test extends AsyncResource { this.only = only ?? !parent.runOnlySubtests; this.reporter = parent.reporter; this.runOnlySubtests = !this.only; - this.testNumber = parent.subtests.length + 1; + this.childNumber = parent.subtests.length + 1; this.timeout = parent.timeout; this.root = parent.root; this.hooks = { @@ -287,6 +295,13 @@ class Test extends AsyncResource { beforeEach: ArrayPrototypeSlice(parent.hooks.beforeEach), afterEach: ArrayPrototypeSlice(parent.hooks.afterEach), }; + + if ((testNamePatterns !== null && !this.matchesTestNamePatterns()) || + (testOnlyFlag && !this.only)) { + skip = true; + this.filtered = true; + this.parent.filteredSubtestCount++; + } } switch (typeof concurrency) { @@ -314,17 +329,6 @@ class Test extends AsyncResource { this.timeout = timeout; } - this.name = name; - this.parent = parent; - - if (testNamePatterns !== null && !this.matchesTestNamePatterns()) { - skip = 'test name does not match pattern'; - } - - if (testOnlyFlag && !this.only) { - skip = '\'only\' option not set'; - } - if (skip) { fn = noop; } @@ -435,14 +439,14 @@ class Test extends AsyncResource { while (this.pendingSubtests.length > 0 && this.hasConcurrency()) { const deferred = ArrayPrototypeShift(this.pendingSubtests); const test = deferred.test; - this.reporter.dequeue(test.nesting, test.loc, test.name); + test.reporter.dequeue(test.nesting, test.loc, test.name); await test.run(); deferred.resolve(); } } addReadySubtest(subtest) { - this.readySubtests.set(subtest.testNumber, subtest); + this.readySubtests.set(subtest.childNumber, subtest); } processReadySubtestRange(canSend) { @@ -503,7 +507,7 @@ class Test extends AsyncResource { const test = new Factory({ __proto__: null, fn, name, parent, ...options, ...overrides }); if (parent.waitingOn === 0) { - parent.waitingOn = test.testNumber; + parent.waitingOn = test.childNumber; } if (preventAddingSubtests) { @@ -591,6 +595,14 @@ class Test extends AsyncResource { } start() { + if (this.filtered) { + noopTestStream ??= new TestsStream(); + this.reporter = noopTestStream; + this.run = this.filteredRun; + } else { + this.testNumber = ++this.parent.outputSubtestCount; + } + // If there is enough available concurrency to run the test now, then do // it. Otherwise, return a Promise to the caller and mark the test as // pending for later execution. @@ -639,6 +651,13 @@ class Test extends AsyncResource { } } + async filteredRun() { + this.pass(); + this.subtests = []; + this.report = noop; + this.postRun(); + } + async run() { if (this.parent !== null) { this.parent.activeSubtests++; @@ -784,11 +803,14 @@ class Test extends AsyncResource { this.mock?.reset(); if (this.parent !== null) { - const report = this.getReportDetails(); - report.details.passed = this.passed; - this.reporter.complete(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); + if (!this.filtered) { + const report = this.getReportDetails(); + report.details.passed = this.passed; + this.testNumber ||= ++this.parent.outputSubtestCount; + this.reporter.complete(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); + this.parent.activeSubtests--; + } - this.parent.activeSubtests--; this.parent.addReadySubtest(this); this.parent.processReadySubtestRange(false); this.parent.processPendingSubtests(); @@ -846,7 +868,7 @@ class Test extends AsyncResource { isClearToSend() { return this.parent === null || ( - this.parent.waitingOn === this.testNumber && this.parent.isClearToSend() + this.parent.waitingOn === this.childNumber && this.parent.isClearToSend() ); } @@ -893,8 +915,8 @@ class Test extends AsyncResource { report() { countCompletedTest(this); - if (this.subtests.length > 0) { - this.reporter.plan(this.subtests[0].nesting, this.loc, this.subtests.length); + if (this.outputSubtestCount > 0) { + this.reporter.plan(this.subtests[0].nesting, this.loc, this.outputSubtestCount); } else { this.reportStarted(); } @@ -996,6 +1018,13 @@ class Suite extends Test { }), () => { this.buildPhaseFinished = true; + + // A suite can transition from filtered to unfiltered based on the + // tests that it contains. + if (this.filtered && this.filteredSubtestCount !== this.subtests.length) { + this.filtered = false; + this.parent.filteredSubtestCount--; + } }, ); } catch (err) { diff --git a/test/fixtures/test-runner/output/name_pattern.snapshot b/test/fixtures/test-runner/output/name_pattern.snapshot index b4dab2a4653dee..0b13848bf4450b 100644 --- a/test/fixtures/test-runner/output/name_pattern.snapshot +++ b/test/fixtures/test-runner/output/name_pattern.snapshot @@ -1,59 +1,27 @@ TAP version 13 -# Subtest: top level test disabled -ok 1 - top level test disabled # SKIP test name does not match pattern - --- - duration_ms: * - ... -# Subtest: top level skipped test disabled -ok 2 - top level skipped test disabled # SKIP test name does not match pattern - --- - duration_ms: * - ... # Subtest: top level skipped test enabled -ok 3 - top level skipped test enabled # SKIP +ok 1 - top level skipped test enabled # SKIP --- duration_ms: * ... # Subtest: top level it enabled -ok 4 - top level it enabled - --- - duration_ms: * - ... -# Subtest: top level it disabled -ok 5 - top level it disabled # SKIP test name does not match pattern - --- - duration_ms: * - ... -# Subtest: top level skipped it disabled -ok 6 - top level skipped it disabled # SKIP test name does not match pattern +ok 2 - top level it enabled --- duration_ms: * ... # Subtest: top level skipped it enabled -ok 7 - top level skipped it enabled # SKIP +ok 3 - top level skipped it enabled # SKIP --- duration_ms: * ... -# Subtest: top level describe never disabled -ok 8 - top level describe never disabled - --- - duration_ms: * - type: 'suite' - ... -# Subtest: top level skipped describe disabled -ok 9 - top level skipped describe disabled # SKIP test name does not match pattern - --- - duration_ms: * - type: 'suite' - ... # Subtest: top level skipped describe enabled -ok 10 - top level skipped describe enabled # SKIP +ok 4 - top level skipped describe enabled # SKIP --- duration_ms: * type: 'suite' ... # Subtest: top level runs because name includes PaTtErN -ok 11 - top level runs because name includes PaTtErN +ok 5 - top level runs because name includes PaTtErN --- duration_ms: * ... @@ -64,7 +32,7 @@ ok 11 - top level runs because name includes PaTtErN duration_ms: * ... 1..1 -ok 12 - top level test enabled +ok 6 - top level test enabled --- duration_ms: * ... @@ -98,7 +66,7 @@ ok 12 - top level test enabled type: 'suite' ... 1..4 -ok 13 - top level describe enabled +ok 7 - top level describe enabled --- duration_ms: * type: 'suite' @@ -132,85 +100,60 @@ ok 13 - top level describe enabled type: 'suite' ... 1..3 -ok 14 - yes +ok 8 - yes --- duration_ms: * type: 'suite' ... # Subtest: no - # Subtest: no - ok 1 - no # SKIP test name does not match pattern - --- - duration_ms: * - ... # Subtest: yes - ok 2 - yes + ok 1 - yes --- duration_ms: * ... # Subtest: maybe - # Subtest: no - ok 1 - no # SKIP test name does not match pattern - --- - duration_ms: * - ... # Subtest: yes - ok 2 - yes + ok 1 - yes --- duration_ms: * ... - 1..2 - ok 3 - maybe + 1..1 + ok 2 - maybe --- duration_ms: * type: 'suite' ... - 1..3 -ok 15 - no + 1..2 +ok 9 - no --- duration_ms: * type: 'suite' ... # Subtest: no with todo - # Subtest: no - ok 1 - no # SKIP test name does not match pattern - --- - duration_ms: * - ... # Subtest: yes - ok 2 - yes + ok 1 - yes --- duration_ms: * ... # Subtest: maybe - # Subtest: no - ok 1 - no # SKIP test name does not match pattern - --- - duration_ms: * - ... # Subtest: yes - ok 2 - yes + ok 1 - yes --- duration_ms: * ... - 1..2 - ok 3 - maybe + 1..1 + ok 2 - maybe --- duration_ms: * type: 'suite' ... - 1..3 -ok 16 - no with todo # TODO test name does not match pattern + 1..2 +ok 10 - no with todo # TODO --- duration_ms: * type: 'suite' ... # Subtest: DescribeForMatchWithAncestors - # Subtest: NestedTest - ok 1 - NestedTest # SKIP test name does not match pattern - --- - duration_ms: * - ... # Subtest: NestedDescribeForMatchWithAncestors # Subtest: NestedTest ok 1 - NestedTest @@ -218,35 +161,23 @@ ok 16 - no with todo # TODO test name does not match pattern duration_ms: * ... 1..1 - ok 2 - NestedDescribeForMatchWithAncestors + ok 1 - NestedDescribeForMatchWithAncestors --- duration_ms: * type: 'suite' ... - 1..2 -ok 17 - DescribeForMatchWithAncestors - --- - duration_ms: * - type: 'suite' - ... -# Subtest: DescribeForMatchWithAncestors - # Subtest: NestedTest - ok 1 - NestedTest # SKIP test name does not match pattern - --- - duration_ms: * - ... 1..1 -ok 18 - DescribeForMatchWithAncestors +ok 11 - DescribeForMatchWithAncestors --- duration_ms: * type: 'suite' ... -1..18 -# tests 28 -# suites 15 +1..11 +# tests 18 +# suites 12 # pass 16 # fail 0 # cancelled 0 -# skipped 12 +# skipped 2 # todo 0 # duration_ms * diff --git a/test/fixtures/test-runner/output/name_pattern_with_only.snapshot b/test/fixtures/test-runner/output/name_pattern_with_only.snapshot index ddfb29afaa2198..64a390dec4c257 100644 --- a/test/fixtures/test-runner/output/name_pattern_with_only.snapshot +++ b/test/fixtures/test-runner/output/name_pattern_with_only.snapshot @@ -15,27 +15,12 @@ ok 1 - enabled and only --- duration_ms: * ... -# Subtest: enabled but not only -ok 2 - enabled but not only # SKIP 'only' option not set - --- - duration_ms: * - ... -# Subtest: only does not match pattern -ok 3 - only does not match pattern # SKIP test name does not match pattern - --- - duration_ms: * - ... -# Subtest: not only and does not match pattern -ok 4 - not only and does not match pattern # SKIP 'only' option not set - --- - duration_ms: * - ... -1..4 -# tests 6 +1..1 +# tests 3 # suites 0 # pass 3 # fail 0 # cancelled 0 -# skipped 3 +# skipped 0 # todo 0 # duration_ms * diff --git a/test/fixtures/test-runner/output/only_tests.snapshot b/test/fixtures/test-runner/output/only_tests.snapshot index ded19f3bec4c6a..33bc475b12c56e 100644 --- a/test/fixtures/test-runner/output/only_tests.snapshot +++ b/test/fixtures/test-runner/output/only_tests.snapshot @@ -1,51 +1,11 @@ TAP version 13 -# Subtest: only = undefined -ok 1 - only = undefined # SKIP 'only' option not set - --- - duration_ms: * - ... -# Subtest: only = undefined, skip = string -ok 2 - only = undefined, skip = string # SKIP 'only' option not set - --- - duration_ms: * - ... -# Subtest: only = undefined, skip = true -ok 3 - only = undefined, skip = true # SKIP 'only' option not set - --- - duration_ms: * - ... -# Subtest: only = undefined, skip = false -ok 4 - only = undefined, skip = false # SKIP 'only' option not set - --- - duration_ms: * - ... -# Subtest: only = false -ok 5 - only = false # SKIP 'only' option not set - --- - duration_ms: * - ... -# Subtest: only = false, skip = string -ok 6 - only = false, skip = string # SKIP 'only' option not set - --- - duration_ms: * - ... -# Subtest: only = false, skip = true -ok 7 - only = false, skip = true # SKIP 'only' option not set - --- - duration_ms: * - ... -# Subtest: only = false, skip = false -ok 8 - only = false, skip = false # SKIP 'only' option not set - --- - duration_ms: * - ... # Subtest: only = true, skip = string -ok 9 - only = true, skip = string # SKIP skip message +ok 1 - only = true, skip = string # SKIP skip message --- duration_ms: * ... # Subtest: only = true, skip = true -ok 10 - only = true, skip = true # SKIP +ok 2 - only = true, skip = true # SKIP --- duration_ms: * ... @@ -60,18 +20,8 @@ ok 10 - only = true, skip = true # SKIP --- duration_ms: * ... - # Subtest: skipped subtest 1 - ok 3 - skipped subtest 1 # SKIP 'only' option not set - --- - duration_ms: * - ... - # Subtest: skipped subtest 2 - ok 4 - skipped subtest 2 # SKIP 'only' option not set - --- - duration_ms: * - ... # Subtest: running subtest 3 - ok 5 - running subtest 3 + ok 3 - running subtest 3 --- duration_ms: * ... @@ -86,33 +36,18 @@ ok 10 - only = true, skip = true # SKIP --- duration_ms: * ... - # Subtest: skipped sub-subtest 1 - ok 3 - skipped sub-subtest 1 # SKIP 'only' option not set - --- - duration_ms: * - ... - # Subtest: skipped sub-subtest 2 - ok 4 - skipped sub-subtest 2 # SKIP 'only' option not set - --- - duration_ms: * - ... - 1..4 - ok 6 - running subtest 4 - --- - duration_ms: * - ... - # Subtest: skipped subtest 3 - ok 7 - skipped subtest 3 # SKIP 'only' option not set + 1..2 + ok 4 - running subtest 4 --- duration_ms: * ... # Subtest: skipped subtest 4 - ok 8 - skipped subtest 4 # SKIP + ok 5 - skipped subtest 4 # SKIP --- duration_ms: * ... - 1..8 -ok 11 - only = true, with subtests + 1..5 +ok 3 - only = true, with subtests --- duration_ms: * ... @@ -122,13 +57,8 @@ ok 11 - only = true, with subtests --- duration_ms: * ... - # Subtest: `it` subtest 2 should not run - ok 2 - `it` subtest 2 should not run # SKIP 'only' option not set - --- - duration_ms: * - ... - 1..2 -ok 12 - describe only = true, with subtests + 1..1 +ok 4 - describe only = true, with subtests --- duration_ms: * type: 'suite' @@ -149,53 +79,23 @@ ok 12 - describe only = true, with subtests --- duration_ms: * ... - # Subtest: `it` subtest 2 only=false - ok 4 - `it` subtest 2 only=false # SKIP 'only' option not set - --- - duration_ms: * - ... - # Subtest: `it` subtest 3 skip - ok 5 - `it` subtest 3 skip # SKIP 'only' option not set - --- - duration_ms: * - ... - # Subtest: `it` subtest 4 todo - ok 6 - `it` subtest 4 todo # SKIP 'only' option not set - --- - duration_ms: * - ... # Subtest: `test` subtest 1 - ok 7 - `test` subtest 1 + ok 4 - `test` subtest 1 --- duration_ms: * ... # Subtest: `test` async subtest 1 - ok 8 - `test` async subtest 1 + ok 5 - `test` async subtest 1 --- duration_ms: * ... # Subtest: `test` subtest 2 only=true - ok 9 - `test` subtest 2 only=true - --- - duration_ms: * - ... - # Subtest: `test` subtest 2 only=false - ok 10 - `test` subtest 2 only=false # SKIP 'only' option not set - --- - duration_ms: * - ... - # Subtest: `test` subtest 3 skip - ok 11 - `test` subtest 3 skip # SKIP 'only' option not set - --- - duration_ms: * - ... - # Subtest: `test` subtest 4 todo - ok 12 - `test` subtest 4 todo # SKIP 'only' option not set + ok 6 - `test` subtest 2 only=true --- duration_ms: * ... - 1..12 -ok 13 - describe only = true, with a mixture of subtests + 1..6 +ok 5 - describe only = true, with a mixture of subtests --- duration_ms: * type: 'suite' @@ -206,28 +106,18 @@ ok 13 - describe only = true, with a mixture of subtests --- duration_ms: * ... - # Subtest: async subtest should not run - ok 2 - async subtest should not run # SKIP 'only' option not set - --- - duration_ms: * - ... - # Subtest: subtest should be skipped - ok 3 - subtest should be skipped # SKIP 'only' option not set - --- - duration_ms: * - ... - 1..3 -ok 14 - describe only = true, with subtests + 1..1 +ok 6 - describe only = true, with subtests --- duration_ms: * type: 'suite' ... -1..14 -# tests 40 +1..6 +# tests 18 # suites 3 # pass 15 # fail 0 # cancelled 0 -# skipped 25 +# skipped 3 # todo 0 # duration_ms * diff --git a/test/parallel/test-runner-run.mjs b/test/parallel/test-runner-run.mjs index 8b050274a3f4b7..1dc5cf99da8d62 100644 --- a/test/parallel/test-runner-run.mjs +++ b/test/parallel/test-runner-run.mjs @@ -135,8 +135,9 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { }) .compose(tap) .toArray(); - assert.strictEqual(result[2], 'ok 1 - this should be skipped # SKIP test name does not match pattern\n'); - assert.strictEqual(result[5], 'ok 2 - this should be executed\n'); + assert.strictEqual(result[2], 'ok 1 - this should be executed\n'); + assert.strictEqual(result[4], '1..1\n'); + assert.strictEqual(result[5], '# tests 1\n'); }); it('should skip tests not matching testNamePatterns - string', async () => { @@ -146,8 +147,9 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { }) .compose(tap) .toArray(); - assert.strictEqual(result[2], 'ok 1 - this should be skipped # SKIP test name does not match pattern\n'); - assert.strictEqual(result[5], 'ok 2 - this should be executed\n'); + assert.strictEqual(result[2], 'ok 1 - this should be executed\n'); + assert.strictEqual(result[4], '1..1\n'); + assert.strictEqual(result[5], '# tests 1\n'); }); it('should pass only to children', async () => { @@ -158,8 +160,9 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { .compose(tap) .toArray(); - assert.strictEqual(result[2], 'ok 1 - this should be skipped # SKIP \'only\' option not set\n'); - assert.strictEqual(result[5], 'ok 2 - this should be executed\n'); + assert.strictEqual(result[2], 'ok 1 - this should be executed\n'); + assert.strictEqual(result[4], '1..1\n'); + assert.strictEqual(result[5], '# tests 1\n'); }); it('should emit "test:watch:drained" event on watch mode', async () => {