Skip to content

Commit

Permalink
test_runner: count nested tests
Browse files Browse the repository at this point in the history
  • Loading branch information
MoLow committed Mar 14, 2023
1 parent 334bb17 commit 7955d04
Show file tree
Hide file tree
Showing 17 changed files with 129 additions and 82 deletions.
1 change: 1 addition & 0 deletions lib/internal/test_runner/reporter/tap.js
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ function reportDetails(nesting, data = kEmptyObject) {
let details = `${_indent} ---\n`;

details += jsToYaml(_indent, 'duration_ms', duration_ms);
details += jsToYaml(_indent, 'class', data.class);
details += jsToYaml(_indent, null, error);
details += `${_indent} ...\n`;
return details;
Expand Down
25 changes: 11 additions & 14 deletions lib/internal/test_runner/runner.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ const {
FunctionPrototypeCall,
Number,
ObjectAssign,
ObjectKeys,
PromisePrototypeThen,
SafePromiseAll,
SafePromiseAllReturnVoid,
Expand Down Expand Up @@ -151,10 +150,14 @@ function getRunArgs({ path, inspectPort }) {

class FileTest extends Test {
#buffer = [];
#counters = { __proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 };
#reportedChildren = 0;
failedSubtests = false;
omitFromTopLevelCounters = true;
#skipReporting() {
return this.#counters.all > 0 && (!this.error || this.error.failureType === kSubtestsFailed);
return this.#reportedChildren > 0 && (!this.error || this.error.failureType === kSubtestsFailed);
}
get omitFromCounters() {
return this.#skipReporting();
}
#checkNestedComment({ comment }) {
const firstSpaceIndex = StringPrototypeIndexOf(comment, ' ');
Expand Down Expand Up @@ -204,11 +207,13 @@ class FileTest extends Test {
const method = pass ? 'ok' : 'fail';
this.reporter[method](nesting, this.name, testNumber, node.description, diagnostics, directive);
if (nesting === 0) {
FunctionPrototypeCall(super.countSubtest,
{ finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled },
this.#counters);
this.failedSubtests ||= !pass;
}
this.#reportedChildren++;
FunctionPrototypeCall(super.countCompleted, {
name: node.description, finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled, nesting,
omitFromCounters: diagnostics.class === 'suite',
});
break;

}
Expand All @@ -233,14 +238,6 @@ class FileTest extends Test {
this.reportStarted();
this.#handleReportItem(ast);
}
countSubtest(counters) {
if (this.#counters.all === 0) {
return super.countSubtest(counters);
}
ArrayPrototypeForEach(ObjectKeys(counters), (key) => {
counters[key] += this.#counters[key];
});
}
reportStarted() {}
report() {
const skipReporting = this.#skipReporting();
Expand Down
72 changes: 45 additions & 27 deletions lib/internal/test_runner/test.js
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ const kUnwrapErrors = new SafeSet()
.add('uncaughtException').add('unhandledRejection');
const { testNamePatterns, testOnlyFlag } = parseCommandLine();

const completedTests = new SafeSet();

function stopTest(timeout, signal) {
if (timeout === kDefaultTimeout) {
return once(signal, 'abort');
Expand Down Expand Up @@ -575,31 +577,11 @@ class Test extends AsyncResource {
this.postRun();
}

countSubtest(counters) {
// Check SKIP and TODO tests first, as those should not be counted as
// failures.
if (this.skipped) {
counters.skipped++;
} else if (this.isTodo) {
counters.todo++;
} else if (this.cancelled) {
counters.cancelled++;
} else if (!this.passed) {
counters.failed++;
} else {
counters.passed++;
}

if (!this.passed) {
counters.totalFailed++;
}
counters.all++;
countCompleted() {
completedTests.add(this);
}

postRun(pendingSubtestsError) {
const counters = {
__proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0,
};
// If the test was failed before it even started, then the end time will
// be earlier than the start time. Correct that here.
if (this.endTime < this.startTime) {
Expand All @@ -610,19 +592,22 @@ class Test extends AsyncResource {
// The test has run, so recursively cancel any outstanding subtests and
// mark this test as failed if any subtests failed.
this.pendingSubtests = [];
let failed = 0;
for (let i = 0; i < this.subtests.length; i++) {
const subtest = this.subtests[i];

if (!subtest.finished) {
subtest.#cancel(pendingSubtestsError);
subtest.postRun(pendingSubtestsError);
}
subtest.countSubtest(counters);
if (!subtest.passed) {
failed++;
}
}

if ((this.passed || this.parent === null) && counters.totalFailed > 0) {
const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`;
const msg = `${counters.totalFailed} ${subtestString} failed`;
if ((this.passed || this.parent === null) && failed > 0) {
const subtestString = `subtest${failed > 1 ? 's' : ''}`;
const msg = `${failed} ${subtestString} failed`;

this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed));
}
Expand All @@ -635,9 +620,36 @@ class Test extends AsyncResource {
this.parent.addReadySubtest(this);
this.parent.processReadySubtestRange(false);
this.parent.processPendingSubtests();
this.countCompleted();
} else if (!this.reported) {
this.reported = true;
this.reporter.plan(this.nesting, kFilename, counters.all);

const counters = {
__proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, topLevel: 0,
};
completedTests.forEach((test) => {
if (test.nesting === this.nesting && !test.omitFromTopLevelCounters) {
counters.topLevel++;
}
if (test.omitFromCounters) {
return;
}
// Check SKIP and TODO tests first, as those should not be counted as
// failures.
if (test.skipped) {
counters.skipped++;
} else if (test.isTodo) {
counters.todo++;
} else if (test.cancelled) {
counters.cancelled++;
} else if (!test.passed) {
counters.failed++;
} else {
counters.passed++;
}
counters.all++;
});
this.reporter.plan(this.nesting, kFilename, counters.topLevel);

for (let i = 0; i < this.diagnostics.length; i++) {
this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i]);
Expand Down Expand Up @@ -703,6 +715,10 @@ class Test extends AsyncResource {
directive = this.reporter.getTodo(this.message);
}

if (this.reportedClass) {
details.class = this.reportedClass;
}

if (this.passed) {
this.reporter.ok(this.nesting, kFilename, this.testNumber, this.name, details, directive);
} else {
Expand Down Expand Up @@ -746,6 +762,8 @@ class TestHook extends Test {
}

class Suite extends Test {
omitFromCounters = true;
reportedClass = 'suite';
constructor(options) {
super(options);

Expand Down
6 changes: 3 additions & 3 deletions test/message/test_runner_abort.out
Original file line number Diff line number Diff line change
Expand Up @@ -260,10 +260,10 @@ not ok 4 - callback abort signal
*
...
1..4
# tests 4
# pass 0
# tests 22
# pass 8
# fail 0
# cancelled 4
# cancelled 14
# skipped 0
# todo 0
# duration_ms *
8 changes: 5 additions & 3 deletions test/message/test_runner_abort_suite.out
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ TAP version 13
not ok 1 - describe timeout signal
---
duration_ms: *
class: 'suite'
failureType: 'testAborted'
error: 'The operation was aborted due to timeout'
code: 23
Expand All @@ -78,6 +79,7 @@ not ok 1 - describe timeout signal
not ok 2 - describe abort signal
---
duration_ms: *
class: 'suite'
failureType: 'testAborted'
error: 'This operation was aborted'
code: 20
Expand All @@ -94,10 +96,10 @@ not ok 2 - describe abort signal
*
...
1..2
# tests 2
# pass 0
# tests 9
# pass 4
# fail 0
# cancelled 2
# cancelled 5
# skipped 0
# todo 0
# duration_ms *
17 changes: 13 additions & 4 deletions test/message/test_runner_describe_it.out
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ ok 21 - immediate resolve pass
not ok 22 - subtest sync throw fail
---
duration_ms: *
class: 'suite'
failureType: 'subtestsFailed'
error: '1 subtest failed'
code: 'ERR_TEST_FAILURE'
Expand Down Expand Up @@ -247,11 +248,13 @@ not ok 23 - sync throw non-error fail
ok 24 - level 0a
---
duration_ms: *
class: 'suite'
...
# Subtest: invalid subtest - pass but subtest fails
ok 25 - invalid subtest - pass but subtest fails
---
duration_ms: *
class: 'suite'
...
# Subtest: sync skip option
ok 26 - sync skip option # SKIP
Expand Down Expand Up @@ -494,6 +497,7 @@ not ok 53 - custom inspect symbol that throws fail
not ok 54 - subtest sync throw fails
---
duration_ms: *
class: 'suite'
failureType: 'subtestsFailed'
error: '2 subtests failed'
code: 'ERR_TEST_FAILURE'
Expand All @@ -511,6 +515,7 @@ not ok 54 - subtest sync throw fails
not ok 55 - describe sync throw fails
---
duration_ms: *
class: 'suite'
failureType: 'testCodeFailure'
error: 'thrown from describe'
code: 'ERR_TEST_FAILURE'
Expand Down Expand Up @@ -539,6 +544,7 @@ not ok 55 - describe sync throw fails
not ok 56 - describe async throw fails
---
duration_ms: *
class: 'suite'
failureType: 'testCodeFailure'
error: 'thrown from describe'
code: 'ERR_TEST_FAILURE'
Expand Down Expand Up @@ -587,6 +593,7 @@ not ok 56 - describe async throw fails
not ok 57 - timeouts
---
duration_ms: *
class: 'suite'
failureType: 'subtestsFailed'
error: '2 subtests failed'
code: 'ERR_TEST_FAILURE'
Expand All @@ -612,6 +619,7 @@ not ok 57 - timeouts
not ok 58 - successful thenable
---
duration_ms: *
class: 'suite'
failureType: 'subtestsFailed'
error: '1 subtest failed'
code: 'ERR_TEST_FAILURE'
Expand All @@ -620,6 +628,7 @@ not ok 58 - successful thenable
not ok 59 - rejected thenable
---
duration_ms: *
class: 'suite'
failureType: 'testCodeFailure'
error: 'custom error'
code: 'ERR_TEST_FAILURE'
Expand All @@ -643,10 +652,10 @@ not ok 60 - invalid subtest fail
# Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
# Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
# tests 60
# pass 23
# fail 22
# cancelled 0
# tests 67
# pass 29
# fail 19
# cancelled 4
# skipped 10
# todo 5
# duration_ms *
2 changes: 2 additions & 0 deletions test/message/test_runner_describe_nested.out
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ TAP version 13
ok 1 - nested
---
duration_ms: *
class: 'suite'
...
1..1
ok 1 - nested - no tests
---
duration_ms: *
class: 'suite'
...
1..1
# tests 1
Expand Down
Loading

0 comments on commit 7955d04

Please sign in to comment.