diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index a5402b4a6084ac..c7d5dd9bdb037f 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -113,6 +113,7 @@ function reportDetails(nesting, data = kEmptyObject) { let details = `${_indent} ---\n`; details += jsToYaml(_indent, 'duration_ms', duration_ms); + details += jsToYaml(_indent, 'type', data.type); details += jsToYaml(_indent, null, error); details += `${_indent} ...\n`; return details; diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index 8b54d53a2d5a8f..dc96bd57b02bbe 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -13,7 +13,6 @@ const { FunctionPrototypeCall, Number, ObjectAssign, - ObjectKeys, PromisePrototypeThen, SafePromiseAll, SafePromiseAllReturnVoid, @@ -151,10 +150,14 @@ function getRunArgs({ path, inspectPort }) { class FileTest extends Test { #buffer = []; - #counters = { __proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 }; + #reportedChildren = 0; failedSubtests = false; + omitFromTopLevelCounters = true; #skipReporting() { - return this.#counters.all > 0 && (!this.error || this.error.failureType === kSubtestsFailed); + return this.#reportedChildren > 0 && (!this.error || this.error.failureType === kSubtestsFailed); + } + get omitFromCounters() { + return this.#skipReporting(); } #checkNestedComment({ comment }) { const firstSpaceIndex = StringPrototypeIndexOf(comment, ' '); @@ -204,11 +207,19 @@ class FileTest extends Test { const method = pass ? 'ok' : 'fail'; this.reporter[method](nesting, this.name, testNumber, node.description, diagnostics, directive); if (nesting === 0) { - FunctionPrototypeCall(super.countSubtest, - { finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled }, - this.#counters); this.failedSubtests ||= !pass; } + this.#reportedChildren++; + FunctionPrototypeCall(super.countCompleted, { + name: node.description, + finished: true, + skipped: skip, + isTodo: todo, + passed: pass, + cancelled, + nesting, + omitFromCounters: diagnostics.type === 'suite', + }); break; } @@ -233,14 +244,6 @@ class FileTest extends Test { this.reportStarted(); this.#handleReportItem(ast); } - countSubtest(counters) { - if (this.#counters.all === 0) { - return super.countSubtest(counters); - } - ArrayPrototypeForEach(ObjectKeys(counters), (key) => { - counters[key] += this.#counters[key]; - }); - } reportStarted() {} report() { const skipReporting = this.#skipReporting(); diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index ffbf2d257aed62..7d640450fde9fc 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -71,6 +71,8 @@ const kUnwrapErrors = new SafeSet() .add('uncaughtException').add('unhandledRejection'); const { testNamePatterns, testOnlyFlag } = parseCommandLine(); +const completedTests = new SafeSet(); + function stopTest(timeout, signal) { if (timeout === kDefaultTimeout) { return once(signal, 'abort'); @@ -575,31 +577,11 @@ class Test extends AsyncResource { this.postRun(); } - countSubtest(counters) { - // Check SKIP and TODO tests first, as those should not be counted as - // failures. - if (this.skipped) { - counters.skipped++; - } else if (this.isTodo) { - counters.todo++; - } else if (this.cancelled) { - counters.cancelled++; - } else if (!this.passed) { - counters.failed++; - } else { - counters.passed++; - } - - if (!this.passed) { - counters.totalFailed++; - } - counters.all++; + countCompleted() { + completedTests.add(this); } postRun(pendingSubtestsError) { - const counters = { - __proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0, - }; // If the test was failed before it even started, then the end time will // be earlier than the start time. Correct that here. if (this.endTime < this.startTime) { @@ -610,6 +592,7 @@ class Test extends AsyncResource { // The test has run, so recursively cancel any outstanding subtests and // mark this test as failed if any subtests failed. this.pendingSubtests = []; + let failed = 0; for (let i = 0; i < this.subtests.length; i++) { const subtest = this.subtests[i]; @@ -617,12 +600,14 @@ class Test extends AsyncResource { subtest.#cancel(pendingSubtestsError); subtest.postRun(pendingSubtestsError); } - subtest.countSubtest(counters); + if (!subtest.passed) { + failed++; + } } - if ((this.passed || this.parent === null) && counters.totalFailed > 0) { - const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`; - const msg = `${counters.totalFailed} ${subtestString} failed`; + if ((this.passed || this.parent === null) && failed > 0) { + const subtestString = `subtest${failed > 1 ? 's' : ''}`; + const msg = `${failed} ${subtestString} failed`; this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed)); } @@ -635,9 +620,43 @@ class Test extends AsyncResource { this.parent.addReadySubtest(this); this.parent.processReadySubtestRange(false); this.parent.processPendingSubtests(); + this.countCompleted(); } else if (!this.reported) { this.reported = true; - this.reporter.plan(this.nesting, kFilename, counters.all); + + const counters = { + __proto__: null, + all: 0, + failed: 0, + passed: 0, + cancelled: 0, + skipped: 0, + todo: 0, + topLevel: 0, + }; + completedTests.forEach((test) => { + if (test.nesting === this.nesting && !test.omitFromTopLevelCounters) { + counters.topLevel++; + } + if (test.omitFromCounters) { + return; + } + // Check SKIP and TODO tests first, as those should not be counted as + // failures. + if (test.skipped) { + counters.skipped++; + } else if (test.isTodo) { + counters.todo++; + } else if (test.cancelled) { + counters.cancelled++; + } else if (!test.passed) { + counters.failed++; + } else { + counters.passed++; + } + counters.all++; + }); + this.reporter.plan(this.nesting, kFilename, counters.topLevel); for (let i = 0; i < this.diagnostics.length; i++) { this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i]); @@ -703,6 +722,10 @@ class Test extends AsyncResource { directive = this.reporter.getTodo(this.message); } + if (this.reportedType) { + details.type = this.reportedType; + } + if (this.passed) { this.reporter.ok(this.nesting, kFilename, this.testNumber, this.name, details, directive); } else { @@ -746,6 +769,8 @@ class TestHook extends Test { } class Suite extends Test { + omitFromCounters = true; + reportedType = 'suite'; constructor(options) { super(options); diff --git a/test/message/test_runner_abort.out b/test/message/test_runner_abort.out index 95a78243e729bf..1fd1750dc76f12 100644 --- a/test/message/test_runner_abort.out +++ b/test/message/test_runner_abort.out @@ -260,10 +260,10 @@ not ok 4 - callback abort signal * ... 1..4 -# tests 4 -# pass 0 +# tests 22 +# pass 8 # fail 0 -# cancelled 4 +# cancelled 14 # skipped 0 # todo 0 # duration_ms * diff --git a/test/message/test_runner_abort_suite.out b/test/message/test_runner_abort_suite.out index 06a70bdf012196..6ebf8a37503601 100644 --- a/test/message/test_runner_abort_suite.out +++ b/test/message/test_runner_abort_suite.out @@ -64,6 +64,7 @@ TAP version 13 not ok 1 - describe timeout signal --- duration_ms: * + type: 'suite' failureType: 'testAborted' error: 'The operation was aborted due to timeout' code: 23 @@ -78,6 +79,7 @@ not ok 1 - describe timeout signal not ok 2 - describe abort signal --- duration_ms: * + type: 'suite' failureType: 'testAborted' error: 'This operation was aborted' code: 20 @@ -94,10 +96,10 @@ not ok 2 - describe abort signal * ... 1..2 -# tests 2 -# pass 0 +# tests 9 +# pass 4 # fail 0 -# cancelled 2 +# cancelled 5 # skipped 0 # todo 0 # duration_ms * diff --git a/test/message/test_runner_describe_it.out b/test/message/test_runner_describe_it.out index 41c4e275b5b614..50e592c76b0955 100644 --- a/test/message/test_runner_describe_it.out +++ b/test/message/test_runner_describe_it.out @@ -210,6 +210,7 @@ ok 21 - immediate resolve pass not ok 22 - subtest sync throw fail --- duration_ms: * + type: 'suite' failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' @@ -247,11 +248,13 @@ not ok 23 - sync throw non-error fail ok 24 - level 0a --- duration_ms: * + type: 'suite' ... # Subtest: invalid subtest - pass but subtest fails ok 25 - invalid subtest - pass but subtest fails --- duration_ms: * + type: 'suite' ... # Subtest: sync skip option ok 26 - sync skip option # SKIP @@ -494,6 +497,7 @@ not ok 53 - custom inspect symbol that throws fail not ok 54 - subtest sync throw fails --- duration_ms: * + type: 'suite' failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' @@ -511,6 +515,7 @@ not ok 54 - subtest sync throw fails not ok 55 - describe sync throw fails --- duration_ms: * + type: 'suite' failureType: 'testCodeFailure' error: 'thrown from describe' code: 'ERR_TEST_FAILURE' @@ -539,6 +544,7 @@ not ok 55 - describe sync throw fails not ok 56 - describe async throw fails --- duration_ms: * + type: 'suite' failureType: 'testCodeFailure' error: 'thrown from describe' code: 'ERR_TEST_FAILURE' @@ -587,6 +593,7 @@ not ok 56 - describe async throw fails not ok 57 - timeouts --- duration_ms: * + type: 'suite' failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' @@ -612,6 +619,7 @@ not ok 57 - timeouts not ok 58 - successful thenable --- duration_ms: * + type: 'suite' failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' @@ -620,6 +628,7 @@ not ok 58 - successful thenable not ok 59 - rejected thenable --- duration_ms: * + type: 'suite' failureType: 'testCodeFailure' error: 'custom error' code: 'ERR_TEST_FAILURE' @@ -643,10 +652,10 @@ not ok 60 - invalid subtest fail # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. -# tests 60 -# pass 23 -# fail 22 -# cancelled 0 +# tests 67 +# pass 29 +# fail 19 +# cancelled 4 # skipped 10 # todo 5 # duration_ms * diff --git a/test/message/test_runner_describe_nested.out b/test/message/test_runner_describe_nested.out index 1d3fe31b75c37a..bd24045a5a4797 100644 --- a/test/message/test_runner_describe_nested.out +++ b/test/message/test_runner_describe_nested.out @@ -10,11 +10,13 @@ TAP version 13 ok 1 - nested --- duration_ms: * + type: 'suite' ... 1..1 ok 1 - nested - no tests --- duration_ms: * + type: 'suite' ... 1..1 # tests 1 diff --git a/test/message/test_runner_hooks.out b/test/message/test_runner_hooks.out index 7c82e9ff292ad5..5083a68c44409a 100644 --- a/test/message/test_runner_hooks.out +++ b/test/message/test_runner_hooks.out @@ -25,11 +25,13 @@ TAP version 13 ok 3 - nested --- duration_ms: * + type: 'suite' ... 1..3 ok 1 - describe hooks --- duration_ms: * + type: 'suite' ... # Subtest: before throws # Subtest: 1 @@ -52,6 +54,7 @@ ok 1 - describe hooks not ok 2 - before throws --- duration_ms: * + type: 'suite' failureType: 'hookFailed' error: 'failed running before hook' code: 'ERR_TEST_FAILURE' @@ -80,6 +83,7 @@ not ok 2 - before throws not ok 3 - after throws --- duration_ms: * + type: 'suite' failureType: 'hookFailed' error: 'failed running after hook' code: 'ERR_TEST_FAILURE' @@ -136,6 +140,7 @@ not ok 3 - after throws not ok 4 - beforeEach throws --- duration_ms: * + type: 'suite' failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' @@ -183,6 +188,7 @@ not ok 4 - beforeEach throws not ok 5 - afterEach throws --- duration_ms: * + type: 'suite' failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' @@ -216,6 +222,7 @@ not ok 5 - afterEach throws not ok 6 - afterEach when test fails --- duration_ms: * + type: 'suite' failureType: 'subtestsFailed' error: '1 subtest failed' code: 'ERR_TEST_FAILURE' @@ -263,6 +270,7 @@ not ok 6 - afterEach when test fails not ok 7 - afterEach throws and test fails --- duration_ms: * + type: 'suite' failureType: 'subtestsFailed' error: '2 subtests failed' code: 'ERR_TEST_FAILURE' @@ -491,10 +499,10 @@ not ok 13 - t.after() is called if test body throws ... # - after() called 1..13 -# tests 13 -# pass 2 -# fail 11 -# cancelled 0 +# tests 35 +# pass 14 +# fail 19 +# cancelled 2 # skipped 0 # todo 0 # duration_ms * diff --git a/test/message/test_runner_no_refs.out b/test/message/test_runner_no_refs.out index e8560c5720b762..b33ff75cee7e35 100644 --- a/test/message/test_runner_no_refs.out +++ b/test/message/test_runner_no_refs.out @@ -21,10 +21,10 @@ not ok 1 - does not keep event loop alive * ... 1..1 -# tests 1 +# tests 2 # pass 0 # fail 0 -# cancelled 1 +# cancelled 2 # skipped 0 # todo 0 # duration_ms * diff --git a/test/message/test_runner_only_tests.out b/test/message/test_runner_only_tests.out index 7d8240fef0c489..8f7000fdbab42e 100644 --- a/test/message/test_runner_only_tests.out +++ b/test/message/test_runner_only_tests.out @@ -131,6 +131,7 @@ ok 11 - only = true, with subtests ok 12 - describe only = true, with subtests --- duration_ms: * + type: 'suite' ... # Subtest: describe only = true, with a mixture of subtests # Subtest: `it` subtest 1 @@ -167,6 +168,7 @@ ok 12 - describe only = true, with subtests ok 13 - describe only = true, with a mixture of subtests --- duration_ms: * + type: 'suite' ... # Subtest: describe only = true, with subtests # Subtest: subtest should run @@ -188,12 +190,13 @@ ok 13 - describe only = true, with a mixture of subtests ok 14 - describe only = true, with subtests --- duration_ms: * + type: 'suite' ... 1..14 -# tests 14 -# pass 4 +# tests 34 +# pass 14 # fail 0 # cancelled 0 -# skipped 10 +# skipped 20 # todo 0 # duration_ms * diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out index b6f254708010e9..fe06515445abe4 100644 --- a/test/message/test_runner_output.out +++ b/test/message/test_runner_output.out @@ -642,10 +642,10 @@ not ok 65 - invalid subtest fail # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. -# tests 65 -# pass 27 -# fail 21 -# cancelled 2 +# tests 79 +# pass 37 +# fail 24 +# cancelled 3 # skipped 10 # todo 5 # duration_ms * diff --git a/test/message/test_runner_output_cli.out b/test/message/test_runner_output_cli.out index 3baeb534704b11..a62f1f98093f05 100644 --- a/test/message/test_runner_output_cli.out +++ b/test/message/test_runner_output_cli.out @@ -642,10 +642,10 @@ not ok 65 - invalid subtest fail # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. 1..65 -# tests 65 -# pass 27 -# fail 21 -# cancelled 2 +# tests 79 +# pass 37 +# fail 24 +# cancelled 3 # skipped 10 # todo 5 # duration_ms * diff --git a/test/message/test_runner_output_spec_reporter.out b/test/message/test_runner_output_spec_reporter.out index 3ff9aefe7a42ce..0ff1e74ed0495b 100644 --- a/test/message/test_runner_output_spec_reporter.out +++ b/test/message/test_runner_output_spec_reporter.out @@ -275,10 +275,10 @@ Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. - tests 65 - pass 27 - fail 21 - cancelled 2 + tests 79 + pass 37 + fail 24 + cancelled 3 skipped 10 todo 5 duration_ms * diff --git a/test/message/test_runner_test_name_pattern.out b/test/message/test_runner_test_name_pattern.out index be548ad0c6dfee..534c7444ea0783 100644 --- a/test/message/test_runner_test_name_pattern.out +++ b/test/message/test_runner_test_name_pattern.out @@ -38,16 +38,19 @@ ok 7 - top level skipped it enabled # SKIP ok 8 - top level describe disabled # SKIP test name does not match pattern --- duration_ms: * + type: 'suite' ... # Subtest: top level skipped describe disabled ok 9 - top level skipped describe disabled # SKIP test name does not match pattern --- duration_ms: * + type: 'suite' ... # Subtest: top level skipped describe enabled ok 10 - top level skipped describe enabled # SKIP --- duration_ms: * + type: 'suite' ... # Subtest: top level runs because name includes PaTtErN ok 11 - top level runs because name includes PaTtErN @@ -80,6 +83,7 @@ ok 12 - top level test enabled ok 3 - nested describe disabled # SKIP test name does not match pattern --- duration_ms: * + type: 'suite' ... # Subtest: nested describe enabled # Subtest: is enabled @@ -91,17 +95,19 @@ ok 12 - top level test enabled ok 4 - nested describe enabled --- duration_ms: * + type: 'suite' ... 1..4 ok 13 - top level describe enabled --- duration_ms: * + type: 'suite' ... 1..13 # tests 13 -# pass 4 +# pass 6 # fail 0 # cancelled 0 -# skipped 9 +# skipped 7 # todo 0 # duration_ms * diff --git a/test/message/test_runner_test_name_pattern_with_only.out b/test/message/test_runner_test_name_pattern_with_only.out index 2e1006409cf9d3..3d34bf0d42dd1b 100644 --- a/test/message/test_runner_test_name_pattern_with_only.out +++ b/test/message/test_runner_test_name_pattern_with_only.out @@ -31,10 +31,10 @@ ok 4 - not only and does not match pattern # SKIP 'only' option not set duration_ms: * ... 1..4 -# tests 4 -# pass 1 +# tests 6 +# pass 2 # fail 0 # cancelled 0 -# skipped 3 +# skipped 4 # todo 0 # duration_ms * diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js index 8cfceedfe6a53a..5e913eb6de9e5d 100644 --- a/test/parallel/test-runner-cli.js +++ b/test/parallel/test-runner-cli.js @@ -155,9 +155,10 @@ const testFixtures = fixtures.path('test-runner'); assert.match(stdout, /# Subtest: level 0b/); assert.match(stdout, /not ok 4 - level 0b/); assert.match(stdout, / {2}error: 'level 0b error'/); - assert.match(stdout, /# tests 4/); - assert.match(stdout, /# pass 2/); - assert.match(stdout, /# fail 2/); + assert.match(stdout, /# tests 8/); + assert.match(stdout, /# pass 4/); + assert.match(stdout, /# fail 3/); + assert.match(stdout, /# skipped 1/); } { diff --git a/test/parallel/test-runner-extraneous-async-activity.js b/test/parallel/test-runner-extraneous-async-activity.js index bc4be25d5f974b..a95925dbb75414 100644 --- a/test/parallel/test-runner-extraneous-async-activity.js +++ b/test/parallel/test-runner-extraneous-async-activity.js @@ -12,7 +12,7 @@ const { spawnSync } = require('child_process'); const stdout = child.stdout.toString(); assert.match(stdout, /^# Warning: Test "extraneous async activity test" generated asynchronous activity after the test ended/m); assert.match(stdout, /^# pass 1/m); - assert.match(stdout, /^# fail 0$/m); + assert.match(stdout, /^# fail 1$/m); assert.match(stdout, /^# cancelled 0$/m); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); @@ -26,7 +26,7 @@ const { spawnSync } = require('child_process'); const stdout = child.stdout.toString(); assert.match(stdout, /^# Warning: Test "extraneous async activity test" generated asynchronous activity after the test ended/m); assert.match(stdout, /^# pass 1$/m); - assert.match(stdout, /^# fail 0$/m); + assert.match(stdout, /^# fail 1$/m); assert.match(stdout, /^# cancelled 0$/m); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null);