From 152cecca7469a85ae2e025d095ce3b75279c668e Mon Sep 17 00:00:00 2001 From: juergba Date: Thu, 9 Apr 2020 15:00:20 +0200 Subject: [PATCH 1/4] merge PR4221 - beforeAll --- docs/index.md | 44 +++- lib/reporters/base.js | 13 +- lib/reporters/json.js | 15 +- lib/reporters/spec.js | 6 + lib/runnable.js | 20 +- lib/runner.js | 114 ++++----- lib/stats-collector.js | 12 +- lib/suite.js | 12 + test/assertions.js | 42 +++- test/integration/fixtures/glob/glob.spec.js | 2 +- .../fixtures/glob/nested/glob.spec.js | 2 +- .../before-hook-async-error-tip.fixture.js | 4 +- .../hooks/before-hook-async-error.fixture.js | 13 +- .../before-hook-deepnested-error.fixture.js | 8 +- .../hooks/before-hook-error-tip.fixture.js | 4 +- .../hooks/before-hook-error.fixture.js | 13 +- .../hooks/before-hook-nested-error.fixture.js | 4 +- test/integration/glob.spec.js | 70 +++--- test/integration/helpers.js | 17 +- test/integration/hook-err.spec.js | 238 +++++++++++------- test/reporters/base.spec.js | 26 +- test/reporters/helpers.js | 1 + test/reporters/json.spec.js | 20 ++ test/reporters/spec.spec.js | 22 ++ 24 files changed, 473 insertions(+), 249 deletions(-) diff --git a/docs/index.md b/docs/index.md index aeaa1c318c..ef4e4f9764 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,7 +22,6 @@ Mocha is a feature-rich JavaScript test framework running on [Node.js][] and in - [test coverage reporting](#wallabyjs) - [string diff support](#diffs) - [javascript API for running tests](#more-information) -- proper exit status for CI support etc - [auto-detects and disables coloring for non-ttys](#reporters) - [async test timeout support](#delayed-root-suite) - [test retry support](#retry-tests) @@ -36,7 +35,6 @@ Mocha is a feature-rich JavaScript test framework running on [Node.js][] and in - [auto-exit to prevent "hanging" with an active loop](#-exit) - [easily meta-generate suites](#markdown) & [test-cases](#list) - [config file support](#-config-path) -- clickable suite titles to filter test execution - [node debugger support](#-inspect-inspect-brk-inspect) - [node native ES modules support](#nodejs-native-esm-support) - [detects multiple calls to `done()`](#detects-multiple-calls-to-done) @@ -449,9 +447,43 @@ setTimeout(function() { }, 5000); ``` +### Failing Hooks + +Upon a failing `before` hook all tests in the current suite and also its nested suites will be skipped. Skipped tests are included in the test results and marked as **skipped**. A skipped test is not considered a failed test. + +```js +describe('outer', function() { + before(function() { + throw new Error('Exception in before hook'); + }); + + it('should skip this outer test', function() { + // will be skipped and listed as 'skipped' + }); + + after(function() { + // will be executed + }); + + describe('inner', function() { + before(function() { + // will be skipped + }); + + it('should skip this inner test', function() { + // will be skipped and listed as 'skipped' + }); + + after(function() { + // will be skipped + }); + }); +}); +``` + ## Pending Tests -"Pending"--as in "someone should write these test cases eventually"--test-cases are simply those _without_ a callback: +"Pending" - as in "someone should write these test cases eventually" - test-cases are simply those _without_ a callback: ```js describe('Array', function() { @@ -462,7 +494,9 @@ describe('Array', function() { }); ``` -Pending tests will be included in the test results, and marked as pending. A pending test is not considered a failed test. +By appending `.skip()`, you may also tell Mocha to ignore a test: [inclusive tests](#inclusive-tests). + +Pending tests will be included in the test results, and marked as **pending**. A pending test is not considered a failed test. ## Exclusive Tests @@ -1641,7 +1675,7 @@ mocha.setup({ ### Browser-specific Option(s) Browser Mocha supports many, but not all [cli options](#command-line-usage). -To use a [cli option](#command-line-usage) that contains a "-", please convert the option to camel-case, (eg. `check-leaks` to `checkLeaks`). +To use a [cli option](#command-line-usage) that contains a "-", please convert the option to camel-case, (e.g. `check-leaks` to `checkLeaks`). #### Options that differ slightly from [cli options](#command-line-usage): diff --git a/lib/reporters/base.js b/lib/reporters/base.js index ea259445e3..7e2c20a8ca 100644 --- a/lib/reporters/base.js +++ b/lib/reporters/base.js @@ -343,18 +343,25 @@ Base.prototype.epilogue = function() { // passes fmt = color('bright pass', ' ') + - color('green', ' %d passing') + + color('green', ' %d / %d passing') + color('light', ' (%s)'); - Base.consoleLog(fmt, stats.passes || 0, milliseconds(stats.duration)); + Base.consoleLog(fmt, stats.passes, stats.tests, milliseconds(stats.duration)); // pending if (stats.pending) { - fmt = color('pending', ' ') + color('pending', ' %d pending'); + fmt = color('pending', ' %d pending'); Base.consoleLog(fmt, stats.pending); } + // skipped + if (stats.skipped) { + fmt = color('fail', ' %d skipped'); + + Base.consoleLog(fmt, stats.skipped); + } + // failures if (stats.failures) { fmt = color('fail', ' %d failing'); diff --git a/lib/reporters/json.js b/lib/reporters/json.js index 12b6289cd7..2540fd4d2e 100644 --- a/lib/reporters/json.js +++ b/lib/reporters/json.js @@ -9,10 +9,11 @@ var Base = require('./base'); var constants = require('../runner').constants; var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; var EVENT_TEST_END = constants.EVENT_TEST_END; var EVENT_RUN_END = constants.EVENT_RUN_END; -var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; /** * Expose `JSON`. @@ -35,9 +36,10 @@ function JSONReporter(runner, options) { var self = this; var tests = []; + var passes = []; var pending = []; + var skipped = []; var failures = []; - var passes = []; runner.on(EVENT_TEST_END, function(test) { tests.push(test); @@ -55,13 +57,18 @@ function JSONReporter(runner, options) { pending.push(test); }); + runner.on(EVENT_TEST_SKIPPED, function(test) { + skipped.push(test); + }); + runner.once(EVENT_RUN_END, function() { var obj = { stats: self.stats, tests: tests.map(clean), + passes: passes.map(clean), pending: pending.map(clean), - failures: failures.map(clean), - passes: passes.map(clean) + skipped: skipped.map(clean), + failures: failures.map(clean) }; runner.testResults = obj; diff --git a/lib/reporters/spec.js b/lib/reporters/spec.js index e51ed80ac4..1983387df8 100644 --- a/lib/reporters/spec.js +++ b/lib/reporters/spec.js @@ -15,6 +15,7 @@ var EVENT_SUITE_END = constants.EVENT_SUITE_END; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var inherits = require('../utils').inherits; var color = Base.color; @@ -88,6 +89,11 @@ function Spec(runner, options) { Base.consoleLog(indent() + color('fail', ' %d) %s'), ++n, test.title); }); + runner.on(EVENT_TEST_SKIPPED, function(test) { + var fmt = indent() + color('fail', ' - %s'); + Base.consoleLog(fmt, test.title); + }); + runner.once(EVENT_RUN_END, self.epilogue.bind(self)); } diff --git a/lib/runnable.js b/lib/runnable.js index bdd6fffe5c..0ce8b9882e 100644 --- a/lib/runnable.js +++ b/lib/runnable.js @@ -40,6 +40,7 @@ function Runnable(title, fn) { this._retries = -1; this._currentRetry = 0; this.pending = false; + this.skipped = false; } /** @@ -166,6 +167,17 @@ Runnable.prototype.isPassed = function() { return !this.isPending() && this.state === constants.STATE_PASSED; }; +/** + * Return `true` if this Runnable has been skipped. + * @return {boolean} + * @private + */ +Runnable.prototype.isSkipped = function() { + return ( + !this.pending && (this.skipped || (this.parent && this.parent.isSkipped())) + ); +}; + /** * Set or get number of retries. * @@ -270,7 +282,7 @@ Runnable.prototype.run = function(fn) { var finished; var emitted; - if (this.isPending()) return fn(); + if (this.isSkipped() || this.isPending()) return fn(); // Sometimes the ctx exists, but it is not runnable if (ctx && ctx.runnable) { @@ -458,7 +470,11 @@ var constants = utils.defineConstants( /** * Value of `state` prop when a `Runnable` has been skipped by user */ - STATE_PENDING: 'pending' + STATE_PENDING: 'pending', + /** + * Value of `state` prop when a `Runnable` has been skipped by failing hook + */ + STATE_SKIPPED: 'skipped' } ); diff --git a/lib/runner.js b/lib/runner.js index c60e562a81..0ef8b6fdba 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -19,6 +19,7 @@ var EVENT_ROOT_SUITE_RUN = Suite.constants.EVENT_ROOT_SUITE_RUN; var STATE_FAILED = Runnable.constants.STATE_FAILED; var STATE_PASSED = Runnable.constants.STATE_PASSED; var STATE_PENDING = Runnable.constants.STATE_PENDING; +var STATE_SKIPPED = Runnable.constants.STATE_SKIPPED; var dQuote = utils.dQuote; var sQuote = utils.sQuote; var stackFilter = utils.stackTraceFilter(); @@ -106,6 +107,10 @@ var constants = utils.defineConstants( * Emitted when {@link Test} becomes pending */ EVENT_TEST_PENDING: 'pending', + /** + * Emitted when {@link Test} becomes skipped + */ + EVENT_TEST_SKIPPED: 'skipped', /** * Emitted when {@link Test} execution has failed, but will retry */ @@ -358,7 +363,7 @@ Runner.prototype.hook = function(name, fn) { var hooks = suite.getHooks(name); var self = this; - function next(i) { + function nextHook(i) { var hook = hooks[i]; if (!hook) { return fn(); @@ -385,9 +390,8 @@ Runner.prototype.hook = function(name, fn) { hook.run(function cbHookRun(err) { var testError = hook.error(); - if (testError) { - self.fail(self.test, testError); - } + if (testError) self.fail(self.test, testError); + // conditional skip if (hook.pending) { if (name === HOOK_TYPE_AFTER_EACH) { @@ -418,17 +422,28 @@ Runner.prototype.hook = function(name, fn) { } } else if (err) { self.failHook(hook, err); - // stop executing hooks, notify callee of hook err - return fn(err); + + if (name === HOOK_TYPE_BEFORE_ALL) { + suite.tests.forEach(function(test) { + test.skipped = true; + }); + suite.suites.forEach(function(suite) { + suite.skipped = true; + }); + hooks = []; + } else { + // stop executing hooks, notify callee of hook err + return fn(err); + } } self.emit(constants.EVENT_HOOK_END, hook); delete hook.ctx.currentTest; - next(++i); + nextHook(++i); }); } Runner.immediately(function() { - next(0); + nextHook(0); }); }; @@ -557,15 +572,14 @@ Runner.prototype.runTests = function(suite, fn) { var test; function hookErr(_, errSuite, after) { - // before/after Each hook for errSuite failed: + // before-/afterEach hook for errSuite failed var orig = self.suite; - // for failed 'after each' hook start from errSuite parent, + // for failed afterEach hook start from errSuite parent, // otherwise start from errSuite itself self.suite = after ? errSuite.parent : errSuite; if (self.suite) { - // call hookUp afterEach self.hookUp(HOOK_TYPE_AFTER_EACH, function(err2, errSuite2) { self.suite = orig; // some hooks may fail even now @@ -576,33 +590,25 @@ Runner.prototype.runTests = function(suite, fn) { fn(errSuite); }); } else { - // there is no need calling other 'after each' hooks + // there is no need calling other afterEach hooks self.suite = orig; fn(errSuite); } } - function next(err, errSuite) { + function nextTest(err, errSuite) { // if we bail after first err - if (self.failures && suite._bail) { - tests = []; - } + if (self.failures && suite._bail) tests = []; - if (self._abort) { - return fn(); - } + if (self._abort) return fn(); - if (err) { - return hookErr(err, errSuite, true); - } + if (err) return hookErr(err, errSuite, true); // next test test = tests.shift(); // all done - if (!test) { - return fn(); - } + if (!test) return fn(); // grep var match = self._grep.test(test.fullTitle()); @@ -619,14 +625,14 @@ Runner.prototype.runTests = function(suite, fn) { // test suite don't do any immediate recursive loops. Thus, // allowing a JS runtime to breathe. if (self._grep !== self._defaultGrep) { - Runner.immediately(next); + Runner.immediately(nextTest); } else { - next(); + nextTest(); } return; } - // static skip, no hooks are executed + // static skip or conditional skip within beforeAll if (test.isPending()) { if (self.forbidPending) { self.fail(test, new Error('Pending test forbidden'), true); @@ -635,7 +641,15 @@ Runner.prototype.runTests = function(suite, fn) { self.emit(constants.EVENT_TEST_PENDING, test); } self.emit(constants.EVENT_TEST_END, test); - return next(); + return nextTest(); + } + + // skipped by failing beforeAll + if (test.isSkipped()) { + test.state = STATE_SKIPPED; + self.emit(constants.EVENT_TEST_SKIPPED, test); + self.emit(constants.EVENT_TEST_END, test); + return nextTest(); } // execute test and hook(s) @@ -655,7 +669,7 @@ Runner.prototype.runTests = function(suite, fn) { self.suite = errSuite || self.suite; return self.hookUp(HOOK_TYPE_AFTER_EACH, function(e, eSuite) { self.suite = origSuite; - next(e, eSuite); + nextTest(e, eSuite); }); } if (err) { @@ -673,7 +687,7 @@ Runner.prototype.runTests = function(suite, fn) { self.emit(constants.EVENT_TEST_PENDING, test); } self.emit(constants.EVENT_TEST_END, test); - return self.hookUp(HOOK_TYPE_AFTER_EACH, next); + return self.hookUp(HOOK_TYPE_AFTER_EACH, nextTest); } else if (err) { var retry = test.currentRetry(); if (retry < test.retries()) { @@ -685,25 +699,23 @@ Runner.prototype.runTests = function(suite, fn) { // Early return + hook trigger so that it doesn't // increment the count wrong - return self.hookUp(HOOK_TYPE_AFTER_EACH, next); + return self.hookUp(HOOK_TYPE_AFTER_EACH, nextTest); } else { self.fail(test, err); } self.emit(constants.EVENT_TEST_END, test); - return self.hookUp(HOOK_TYPE_AFTER_EACH, next); + return self.hookUp(HOOK_TYPE_AFTER_EACH, nextTest); } test.state = STATE_PASSED; self.emit(constants.EVENT_TEST_PASS, test); self.emit(constants.EVENT_TEST_END, test); - self.hookUp(HOOK_TYPE_AFTER_EACH, next); + self.hookUp(HOOK_TYPE_AFTER_EACH, nextTest); }); }); } - this.next = next; - this.hookErr = hookErr; - next(); + nextTest(); }; /** @@ -726,7 +738,7 @@ Runner.prototype.runSuite = function(suite, fn) { this.emit(constants.EVENT_SUITE_BEGIN, (this.suite = suite)); - function next(errSuite) { + function nextSuite(errSuite) { if (errSuite) { // current suite failed on a hook from errSuite if (errSuite === suite) { @@ -739,47 +751,37 @@ Runner.prototype.runSuite = function(suite, fn) { return done(errSuite); } - if (self._abort) { - return done(); - } + if (self._abort) return done(); var curr = suite.suites[i++]; - if (!curr) { - return done(); - } + if (!curr) return done(); // Avoid grep neglecting large number of tests causing a // huge recursive loop and thus a maximum call stack error. // See comment in `this.runTests()` for more information. if (self._grep !== self._defaultGrep) { Runner.immediately(function() { - self.runSuite(curr, next); + self.runSuite(curr, nextSuite); }); } else { - self.runSuite(curr, next); + self.runSuite(curr, nextSuite); } } function done(errSuite) { self.suite = suite; - self.nextSuite = next; // remove reference to test delete self.test; - self.hook(HOOK_TYPE_AFTER_ALL, function() { + self.hook(HOOK_TYPE_AFTER_ALL, function cbAfterAll() { self.emit(constants.EVENT_SUITE_END, suite); fn(errSuite); }); } - this.nextSuite = next; - - this.hook(HOOK_TYPE_BEFORE_ALL, function(err) { - if (err) { - return done(); - } - self.runTests(suite, next); + this.hook(HOOK_TYPE_BEFORE_ALL, function cbBeforeAll() { + self.runTests(suite, nextSuite); }); }; @@ -833,7 +835,7 @@ Runner.prototype.uncaught = function(err) { runnable.clearTimeout(); - if (runnable.isFailed()) { + if (runnable.isFailed() || runnable.isSkipped()) { // Ignore error if already failed return; } else if (runnable.isPending()) { diff --git a/lib/stats-collector.js b/lib/stats-collector.js index 938778fb0e..6b7209a959 100644 --- a/lib/stats-collector.js +++ b/lib/stats-collector.js @@ -6,13 +6,14 @@ */ var constants = require('./runner').constants; -var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_END = constants.EVENT_TEST_END; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; +var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var EVENT_SUITE_BEGIN = constants.EVENT_SUITE_BEGIN; var EVENT_RUN_BEGIN = constants.EVENT_RUN_BEGIN; -var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; var EVENT_RUN_END = constants.EVENT_RUN_END; -var EVENT_TEST_END = constants.EVENT_TEST_END; /** * Test statistics collector. @@ -23,6 +24,7 @@ var EVENT_TEST_END = constants.EVENT_TEST_END; * @property {number} tests - integer count of tests run. * @property {number} passes - integer count of passing tests. * @property {number} pending - integer count of pending tests. + * @property {number} skipped - integer count of skipped tests. * @property {number} failures - integer count of failed tests. * @property {Date} start - time when testing began. * @property {Date} end - time when testing concluded. @@ -47,6 +49,7 @@ function createStatsCollector(runner) { tests: 0, passes: 0, pending: 0, + skipped: 0, failures: 0 }; @@ -71,6 +74,9 @@ function createStatsCollector(runner) { runner.on(EVENT_TEST_PENDING, function() { stats.pending++; }); + runner.on(EVENT_TEST_SKIPPED, function() { + stats.skipped++; + }); runner.on(EVENT_TEST_END, function() { stats.tests++; }); diff --git a/lib/suite.js b/lib/suite.js index 191d946b50..00c47f51da 100644 --- a/lib/suite.js +++ b/lib/suite.js @@ -62,6 +62,7 @@ function Suite(title, parentContext, isRoot) { this.suites = []; this.tests = []; this.pending = false; + this.skipped = false; this._beforeEach = []; this._beforeAll = []; this._afterEach = []; @@ -210,6 +211,17 @@ Suite.prototype.isPending = function() { return this.pending || (this.parent && this.parent.isPending()); }; +/** + * Check if this suite or its parent suite is marked as skipped. + * + * @private + */ +Suite.prototype.isSkipped = function() { + return ( + !this.pending && (this.skipped || (this.parent && this.parent.isSkipped())) + ); +}; + /** * Generic hook-creator. * @private diff --git a/test/assertions.js b/test/assertions.js index 7453392059..f3d1018bfd 100644 --- a/test/assertions.js +++ b/test/assertions.js @@ -35,6 +35,7 @@ exports.mixinMochaAssertions = function(expect) { typeof v.passing === 'number' && typeof v.failing === 'number' && typeof v.pending === 'number' && + typeof v.skipped === 'number' && typeof v.output === 'string' && typeof v.code === 'number' ); @@ -143,6 +144,12 @@ exports.mixinMochaAssertions = function(expect) { expect(result.stats.pending, '[not] to be', count); } ) + .addAssertion( + ' [not] to have skipped [test] count ', + function(expect, result, count) { + expect(result.stats.skipped, '[not] to be', count); + } + ) .addAssertion( ' [not] to have run (test|tests) ', function(expect, result, titles) { @@ -266,12 +273,23 @@ exports.mixinMochaAssertions = function(expect) { ); } ) - .addAssertion(' [not] to have pending tests', function( - expect, - result - ) { - expect(result.stats.pending, '[not] to be greater than', 0); - }) + .addAssertion( + ' [not] to have skipped test order ', + function(expect, result, titles) { + expect(result, '[not] to have test order', 'skipped', titles); + } + ) + .addAssertion( + ' [not] to have skipped test order ', + function(expect, result, titles) { + expect( + result, + '[not] to have test order', + 'skipped', + Array.prototype.slice.call(arguments, 2) + ); + } + ) .addAssertion(' [not] to have passed tests', function( expect, result @@ -284,6 +302,18 @@ exports.mixinMochaAssertions = function(expect) { ) { expect(result.stats.failed, '[not] to be greater than', 0); }) + .addAssertion(' [not] to have pending tests', function( + expect, + result + ) { + expect(result.stats.pending, '[not] to be greater than', 0); + }) + .addAssertion(' [not] to have skipped tests', function( + expect, + result + ) { + expect(result.stats.skipped, '[not] to be greater than', 0); + }) .addAssertion(' [not] to have tests', function( expect, result diff --git a/test/integration/fixtures/glob/glob.spec.js b/test/integration/fixtures/glob/glob.spec.js index c6c3d26a69..e5e5a80f0d 100644 --- a/test/integration/fixtures/glob/glob.spec.js +++ b/test/integration/fixtures/glob/glob.spec.js @@ -1,7 +1,7 @@ 'use strict'; describe('globbing test', function() { - it('should find this test', function() { + it('should find this glob/test', function() { // see test/integration/glob.spec.js for details }); }); diff --git a/test/integration/fixtures/glob/nested/glob.spec.js b/test/integration/fixtures/glob/nested/glob.spec.js index c6c3d26a69..0bdaee1484 100644 --- a/test/integration/fixtures/glob/nested/glob.spec.js +++ b/test/integration/fixtures/glob/nested/glob.spec.js @@ -1,7 +1,7 @@ 'use strict'; describe('globbing test', function() { - it('should find this test', function() { + it('should find this glob/nested/test', function() { // see test/integration/glob.spec.js for details }); }); diff --git a/test/integration/fixtures/hooks/before-hook-async-error-tip.fixture.js b/test/integration/fixtures/hooks/before-hook-async-error-tip.fixture.js index 04801c1946..8e69274102 100644 --- a/test/integration/fixtures/hooks/before-hook-async-error-tip.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-async-error-tip.fixture.js @@ -1,7 +1,7 @@ 'use strict'; describe('spec 1', function () { - it('should not blame me', function () { }); + it('should run test-1', function () { }); }); describe('spec 2', function () { before(function (done) { @@ -9,5 +9,5 @@ describe('spec 2', function () { throw new Error('before hook error'); }); }); - it('skipped'); + it('should not run test-2', function () { }); }); diff --git a/test/integration/fixtures/hooks/before-hook-async-error.fixture.js b/test/integration/fixtures/hooks/before-hook-async-error.fixture.js index 2530eec783..52cc7139cc 100644 --- a/test/integration/fixtures/hooks/before-hook-async-error.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-async-error.fixture.js @@ -2,20 +2,13 @@ describe('spec 1', function () { before(function (done) { - console.log('before'); process.nextTick(function () { throw new Error('before hook error'); }); }); - it('should not be called because of error in before hook', function () { - console.log('test 1'); - }); - it('should not be called because of error in before hook', function () { - console.log('test 2'); - }); + it('should not run test-1', function () { }); + it('should not run test-2', function () { }); }); describe('spec 2', function () { - it('should be called, because hook error was in a sibling suite', function () { - console.log('test 3'); - }); + it('should run test-3', function () { }); }); diff --git a/test/integration/fixtures/hooks/before-hook-deepnested-error.fixture.js b/test/integration/fixtures/hooks/before-hook-deepnested-error.fixture.js index 804e5e415b..d456464bd7 100644 --- a/test/integration/fixtures/hooks/before-hook-deepnested-error.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-deepnested-error.fixture.js @@ -1,13 +1,13 @@ 'use strict'; describe('spec 1', function () { - it('should pass', function () { }); - describe('spec 2 nested - this title should be used', function () { + it('should run test-1', function () { }); + describe('nested spec 2', function () { before(function() { throw new Error('before hook nested error'); }); - describe('spec 3 nested', function () { - it('it nested - this title should not be used', function () { }); + describe('deepnested spec 3', function () { + it('should not run deepnested test-2', function () { }); }); }); }); diff --git a/test/integration/fixtures/hooks/before-hook-error-tip.fixture.js b/test/integration/fixtures/hooks/before-hook-error-tip.fixture.js index 64df731573..d439701e5c 100644 --- a/test/integration/fixtures/hooks/before-hook-error-tip.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-error-tip.fixture.js @@ -1,11 +1,11 @@ 'use strict'; describe('spec 1', function () { - it('should not blame me', function () { }); + it('should run test-1', function () { }); }); describe('spec 2', function () { before(function () { throw new Error('before hook error'); }); - it('skipped'); + it('should not run test-2', function () { }); }); diff --git a/test/integration/fixtures/hooks/before-hook-error.fixture.js b/test/integration/fixtures/hooks/before-hook-error.fixture.js index 547e54a243..8263d5c855 100644 --- a/test/integration/fixtures/hooks/before-hook-error.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-error.fixture.js @@ -2,18 +2,11 @@ describe('spec 1', function () { before(function () { - console.log('before'); throw new Error('before hook error'); }); - it('should not be called because of error in before hook', function () { - console.log('test 1'); - }); - it('should not be called because of error in before hook', function () { - console.log('test 2'); - }); + it('should not run test-1', function () { }); + it('should not run test-2', function () { }); }); describe('spec 2', function () { - it('should be called, because hook error was in a sibling suite', function () { - console.log('test 3'); - }); + it('should run test-3', function () { }); }); diff --git a/test/integration/fixtures/hooks/before-hook-nested-error.fixture.js b/test/integration/fixtures/hooks/before-hook-nested-error.fixture.js index c0ade3a9f4..ea06c31c81 100644 --- a/test/integration/fixtures/hooks/before-hook-nested-error.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-nested-error.fixture.js @@ -1,11 +1,11 @@ 'use strict'; describe('spec 1', function () { - it('should pass', function () { }); + it('should run test-1', function () { }); describe('spec nested', function () { before(function() { throw new Error('before hook nested error'); }); - it('it nested - this title should be used', function () { }); + it('should not run nested test-2', function () { }); }); }); diff --git a/test/integration/glob.spec.js b/test/integration/glob.spec.js index 4284320aa7..c6ce0077f8 100644 --- a/test/integration/glob.spec.js +++ b/test/integration/glob.spec.js @@ -11,10 +11,9 @@ describe('globbing', function() { testGlob.shouldSucceed( './*.js', function(results) { - expect( - results.stdout, + expect(results.stdout, 'to contain', '["start",{"total":1}]').and( 'to contain', - '["end",{"suites":1,"tests":1,"passes":1,"pending":0,"failures":0,' + '["pass",{"title":"should find this glob/test"' ); }, done @@ -39,8 +38,11 @@ describe('globbing', function() { testGlob.shouldFail( './*-none.js ./*-none-twice.js', function(results) { - expect(results.stderr, 'to contain', 'Error: No test files found'); - expect(results.stderr, 'not to contain', '*-none'); + expect( + results.stderr, + 'to contain', + 'Error: No test files found' + ).and('not to contain', '*-none'); }, done ); @@ -50,10 +52,9 @@ describe('globbing', function() { testGlob.shouldSucceed( './*.js ./*-none.js', function(results) { - expect( - results.stdout, + expect(results.stdout, 'to contain', '["start",{"total":1}]').and( 'to contain', - '["end",{"suites":1,"tests":1,"passes":1,"pending":0,"failures":0,' + '["pass",{"title":"should find this glob/test"' ); expect( results.stderr, @@ -71,10 +72,9 @@ describe('globbing', function() { testGlob.shouldSucceed( '"./*.js"', function(results) { - expect( - results.stdout, + expect(results.stdout, 'to contain', '["start",{"total":1}]').and( 'to contain', - '["end",{"suites":1,"tests":1,"passes":1,"pending":0,"failures":0,' + '["pass",{"title":"should find this glob/test"' ); }, done @@ -109,10 +109,9 @@ describe('globbing', function() { testGlob.shouldSucceed( '"./*.js" "./*-none.js"', function(results) { - expect( - results.stdout, + expect(results.stdout, 'to contain', '["start",{"total":1}]').and( 'to contain', - '["end",{"suites":1,"tests":1,"passes":1,"pending":0,"failures":0,' + '["pass",{"title":"should find this glob/test"' ); expect( results.stderr, @@ -129,11 +128,15 @@ describe('globbing', function() { testGlob.shouldSucceed( '"./**/*.js"', function(results) { - expect( - results.stdout, - 'to contain', - '["end",{"suites":2,"tests":2,"passes":2,"pending":0,"failures":0,' - ); + expect(results.stdout, 'to contain', '["start",{"total":2}]') + .and( + 'to contain', + '["pass",{"title":"should find this glob/test"' + ) + .and( + 'to contain', + '["pass",{"title":"should find this glob/nested/test"' + ); }, done ); @@ -157,11 +160,15 @@ describe('globbing', function() { testGlob.shouldSucceed( '"./**/*.js" "./**/*-none.js"', function(results) { - expect( - results.stdout, - 'to contain', - '["end",{"suites":2,"tests":2,"passes":2,"pending":0,"failures":0,' - ); + expect(results.stdout, 'to contain', '["start",{"total":2}]') + .and( + 'to contain', + '["pass",{"title":"should find this glob/test"' + ) + .and( + 'to contain', + '["pass",{"title":"should find this glob/nested/test"' + ); expect( results.stderr, 'to contain', @@ -187,13 +194,6 @@ var testGlob = { }) }; -var isFlakeyNode = (function() { - var version = process.versions.node.split('.'); - return ( - version[0] === '0' && version[1] === '10' && process.platform === 'win32' - ); -})(); - function execMochaWith(validate) { return function execMocha(glob, assertOn, done) { exec( @@ -206,12 +206,8 @@ function execMochaWith(validate) { function(error, stdout, stderr) { try { validate(error, stderr); - if (isFlakeyNode && error && stderr === '') { - execMocha(glob, assertOn, done); - } else { - assertOn({stdout: stdout, stderr: stderr}); - done(); - } + assertOn({stdout: stdout, stderr: stderr}); + done(); } catch (assertion) { done(assertion); } diff --git a/test/integration/helpers.js b/test/integration/helpers.js index 6cdf7e93cf..0465ba5671 100644 --- a/test/integration/helpers.js +++ b/test/integration/helpers.js @@ -15,15 +15,16 @@ module.exports = { * Invokes the mocha binary for the given fixture with color output disabled. * Accepts an array of additional command line args to pass. The callback is * invoked with a summary of the run, in addition to its output. The summary - * includes the number of passing, pending, and failing tests, as well as the - * exit code. Useful for testing different reporters. + * includes the number of passing, pending, skipped and failing tests, as well + * as the exit code. Useful for testing different reporters. * * By default, `STDERR` is ignored. Pass `{stdio: 'pipe'}` as `opts` if you * want it. * Example response: * { - * pending: 0, * passing: 0, + * pending: 0, + * skipped: 0, * failing: 1, * code: 1, * output: '...' @@ -306,13 +307,17 @@ function resolveFixturePath(fixture) { } function getSummary(res) { - return ['passing', 'pending', 'failing'].reduce(function(summary, type) { + return ['passing', 'pending', 'skipped', 'failing'].reduce(function( + summary, + type + ) { var pattern, match; - pattern = new RegExp(' (\\d+) ' + type + '\\s'); + pattern = new RegExp(' (\\d+) (?:/ \\d+ )?' + type + '\\s'); match = pattern.exec(res.output); summary[type] = match ? parseInt(match, 10) : 0; return summary; - }, res); + }, + res); } diff --git a/test/integration/hook-err.spec.js b/test/integration/hook-err.spec.js index d5fe6e858d..19c4cec727 100644 --- a/test/integration/hook-err.spec.js +++ b/test/integration/hook-err.spec.js @@ -9,72 +9,161 @@ var bang = require('../../lib/reporters/base').symbols.bang; describe('hook error handling', function() { var lines; - describe('before hook error', function() { - before(run('hooks/before-hook-error.fixture.js')); - it('should verify results', function() { - expect(lines, 'to equal', ['before', bang + 'test 3']); - }); - }); + describe('synchronous hook', function() { + describe('in before', function() { + it('before hook error', function(done) { + var fixture = 'hooks/before-hook-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook error') + .and('to have test count', 3) + .and('to have passed test count', 1) + .and('to have skipped test count', 2) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-3') + .and( + 'to have skipped test order', + 'should not run test-1', + 'should not run test-2' + ) + .and( + 'to have failed test', + '"before all" hook for "should not run test-1"' + ); + done(); + }); + }); - describe('before hook error tip', function() { - before(run('hooks/before-hook-error-tip.fixture.js', onlyErrorTitle())); - it('should verify results', function() { - expect(lines, 'to equal', [ - '1) spec 2', - '"before all" hook for "skipped":' - ]); - }); - }); + it('before hook error tip', function(done) { + var fixture = 'hooks/before-hook-error-tip.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook error') + .and('to have test count', 2) + .and('to have passed test count', 1) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and('to have skipped test order', 'should not run test-2') + .and( + 'to have failed test', + '"before all" hook for "should not run test-2"' + ); + done(); + }); + }); - describe('before hook root error', function() { - it('should verify results', function(done) { - var fixture = 'hooks/before-hook-root-error.fixture.js'; - runMochaJSON(fixture, [], function(err, res) { - if (err) { - return done(err); - } - expect(res, 'to have failed with error', 'before hook root error') - .and('to have failed test', '"before all" hook in "{root}"') - .and('to have passed test count', 0); - done(); + it('before hook root error', function(done) { + var fixture = 'hooks/before-hook-root-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook root error') + .and('to have test count', 1) + .and('to have passed test count', 0) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have skipped test order', 'should not be called') + .and('to have failed test', '"before all" hook in "{root}"'); + done(); + }); }); - }); - }); - describe('before hook nested error', function() { - it('should verify results', function(done) { - var fixture = 'hooks/before-hook-nested-error.fixture.js'; - runMochaJSON(fixture, [], function(err, res) { - if (err) { - return done(err); - } - expect(res, 'to have failed with error', 'before hook nested error') - .and( - 'to have failed test', - '"before all" hook for "it nested - this title should be used"' - ) - .and('to have passed test count', 1) - .and('to have passed test', 'should pass'); - done(); + it('before hook nested error', function(done) { + var fixture = 'hooks/before-hook-nested-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook nested error') + .and('to have test count', 2) + .and('to have passed test count', 1) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and('to have skipped test order', 'should not run nested test-2') + .and( + 'to have failed test', + '"before all" hook for "should not run nested test-2"' + ); + done(); + }); + }); + + it('before hook deepnested error', function(done) { + var fixture = 'hooks/before-hook-deepnested-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook nested error') + .and('to have test count', 2) + .and('to have passed test count', 1) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and( + 'to have skipped test order', + 'should not run deepnested test-2' + ) + .and('to have failed test', '"before all" hook in "nested spec 2"'); + done(); + }); }); }); }); - describe('before hook deepnested error', function() { - it('should verify results', function(done) { - var fixture = 'hooks/before-hook-deepnested-error.fixture.js'; - runMochaJSON(fixture, [], function(err, res) { - if (err) { - return done(err); - } - expect(res, 'to have failed with error', 'before hook nested error') - .and( - 'to have failed test', - '"before all" hook in "spec 2 nested - this title should be used"' - ) - .and('to have passed test count', 1) - .and('to have passed test', 'should pass'); - done(); + describe('asynchronous hook', function() { + describe('in before', function() { + it('before hook error', function(done) { + var fixture = 'hooks/before-hook-async-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook error') + .and('to have test count', 3) + .and('to have passed test count', 1) + .and('to have skipped test count', 2) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-3') + .and( + 'to have skipped test order', + 'should not run test-1', + 'should not run test-2' + ) + .and( + 'to have failed test', + '"before all" hook for "should not run test-1"' + ); + done(); + }); + }); + + it('before hook error tip', function(done) { + var fixture = 'hooks/before-hook-async-error-tip.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook error') + .and('to have test count', 2) + .and('to have passed test count', 1) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and('to have skipped test order', 'should not run test-2') + .and( + 'to have failed test', + '"before all" hook for "should not run test-2"' + ); + done(); + }); }); }); }); @@ -182,25 +271,6 @@ describe('hook error handling', function() { }); }); - describe('async - before hook error', function() { - before(run('hooks/before-hook-async-error.fixture.js')); - it('should verify results', function() { - expect(lines, 'to equal', ['before', bang + 'test 3']); - }); - }); - - describe('async - before hook error tip', function() { - before( - run('hooks/before-hook-async-error-tip.fixture.js', onlyErrorTitle()) - ); - it('should verify results', function() { - expect(lines, 'to equal', [ - '1) spec 2', - '"before all" hook for "skipped":' - ]); - }); - }); - describe('async - before each hook error', function() { before(run('hooks/beforeEach-hook-async-error.fixture.js')); it('should verify results', function() { @@ -284,17 +354,3 @@ function onlyConsoleOutput() { return !foundSummary && line.length > 0; }; } - -function onlyErrorTitle(line) { - var foundErrorTitle = false; - var foundError = false; - return function(line) { - if (!foundErrorTitle) { - foundErrorTitle = !!/^1\)/.exec(line); - } - if (!foundError) { - foundError = /Error:/.exec(line); - } - return foundErrorTitle && !foundError; - }; -} diff --git a/test/reporters/base.spec.js b/test/reporters/base.spec.js index 744b92e69b..8e81223eab 100644 --- a/test/reporters/base.spec.js +++ b/test/reporters/base.spec.js @@ -426,25 +426,43 @@ describe('Base reporter', function() { }); describe('when reporter output immune to user test changes', function() { - var sandbox; var baseConsoleLog; beforeEach(function() { - sandbox = sinon.createSandbox(); sandbox.stub(console, 'log'); baseConsoleLog = sandbox.stub(Base, 'consoleLog'); }); it('should let you stub out console.log without effecting reporters output', function() { Base.list([]); - baseConsoleLog.restore(); expect(baseConsoleLog, 'was called'); expect(console.log, 'was not called'); + sandbox.restore(); }); + }); + + describe('epilogue', function() { + it('should include "pending" count', function() { + var ctx = {stats: {passes: 0, pending: 2, skipped: 0, duration: 12}}; + var epilogue = Base.prototype.epilogue.bind(ctx); - afterEach(function() { + epilogue(); sandbox.restore(); + + var out = stdout.join('\n').trim(); + expect(out, 'to contain', '2 pending').and('not to contain', 'skipped'); + }); + + it('should include "skipped" count', function() { + var ctx = {stats: {passes: 0, pending: 0, skipped: 3, duration: 12}}; + var epilogue = Base.prototype.epilogue.bind(ctx); + + epilogue(); + sandbox.restore(); + + var out = stdout.join('\n').trim(); + expect(out, 'to contain', '3 skipped').and('not to contain', 'pending'); }); }); }); diff --git a/test/reporters/helpers.js b/test/reporters/helpers.js index 45c4d916de..90567df37e 100644 --- a/test/reporters/helpers.js +++ b/test/reporters/helpers.js @@ -60,6 +60,7 @@ function createRunnerFunction(runStr, ifStr1, ifStr2, ifStr3, arg1, arg2) { } }; case 'pending test': + case 'skipped test': case 'pass': case 'fail': case 'suite': diff --git a/test/reporters/json.spec.js b/test/reporters/json.spec.js index f6299dd134..142634fd57 100644 --- a/test/reporters/json.spec.js +++ b/test/reporters/json.spec.js @@ -80,6 +80,26 @@ describe('JSON reporter', function() { }); }); + it('should have 1 test skipped', function(done) { + suite.skipped = true; + suite.addTest(new Test(testTitle, noop)); + + runner.run(function(failureCount) { + sandbox.restore(); + expect(runner, 'to satisfy', { + testResults: { + skipped: [ + { + title: testTitle + } + ] + } + }); + expect(failureCount, 'to be', 0); + done(); + }); + }); + it('should handle circular objects in errors', function(done) { var testTitle = 'json test 1'; function CircleError() { diff --git a/test/reporters/spec.spec.js b/test/reporters/spec.spec.js index 608bc7f512..082d5a203e 100644 --- a/test/reporters/spec.spec.js +++ b/test/reporters/spec.spec.js @@ -14,6 +14,7 @@ var EVENT_SUITE_BEGIN = events.EVENT_SUITE_BEGIN; var EVENT_TEST_FAIL = events.EVENT_TEST_FAIL; var EVENT_TEST_PASS = events.EVENT_TEST_PASS; var EVENT_TEST_PENDING = events.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = events.EVENT_TEST_SKIPPED; describe('Spec reporter', function() { var runReporter = makeRunReporter(Spec); @@ -73,6 +74,27 @@ describe('Spec reporter', function() { }); }); + describe("on 'skipped' event", function() { + it('should return title', function() { + var suite = { + title: expectedTitle + }; + var runner = createMockRunner( + 'skipped test', + EVENT_TEST_SKIPPED, + null, + null, + suite + ); + var options = {}; + var stdout = runReporter({epilogue: noop}, runner, options); + sandbox.restore(); + + var expectedArray = [' - ' + expectedTitle + '\n']; + expect(stdout, 'to equal', expectedArray); + }); + }); + describe("on 'pass' event", function() { describe('when test speed is slow', function() { it('should return expected tick, title, and duration', function() { From 434e823d34ac389564c725a90563c4c3e0111d9b Mon Sep 17 00:00:00 2001 From: juergba Date: Tue, 21 Apr 2020 08:26:44 +0200 Subject: [PATCH 2/4] report skipped tests upon failing beforeEach --- lib/runner.js | 74 ++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/lib/runner.js b/lib/runner.js index 0ef8b6fdba..9cd9c4ea7d 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -365,9 +365,8 @@ Runner.prototype.hook = function(name, fn) { function nextHook(i) { var hook = hooks[i]; - if (!hook) { - return fn(); - } + if (!hook) return fn(); + self.currentRunnable = hook; if (name === HOOK_TYPE_BEFORE_ALL) { @@ -399,13 +398,6 @@ Runner.prototype.hook = function(name, fn) { if (self.test) { self.test.pending = true; } - } else if (name === HOOK_TYPE_BEFORE_EACH) { - if (self.test) { - self.test.pending = true; - } - self.emit(constants.EVENT_HOOK_END, hook); - hook.pending = false; // activates hook for next test - return fn(new Error('abort hookDown')); } else if (name === HOOK_TYPE_BEFORE_ALL) { suite.tests.forEach(function(test) { test.pending = true; @@ -414,6 +406,13 @@ Runner.prototype.hook = function(name, fn) { suite.pending = true; }); hooks = []; + } else if (name === HOOK_TYPE_BEFORE_EACH) { + if (self.test) { + self.test.pending = true; + } + self.emit(constants.EVENT_HOOK_END, hook); + hook.pending = false; + return fn(new Error('abort hookDown')); } else { hook.pending = false; var errForbid = createUnsupportedError('`this.skip` forbidden'); @@ -431,6 +430,17 @@ Runner.prototype.hook = function(name, fn) { suite.skipped = true; }); hooks = []; + } else if (name === HOOK_TYPE_BEFORE_EACH) { + (function skipTests(sui) { + sui.tests.forEach(function(t) { + if (!t.state) t.skipped = true; + }); + sui.suites.forEach(function(s) { + skipTests(s); + }); + })(suite); + self.emit(constants.EVENT_HOOK_END, hook); + return fn(err); } else { // stop executing hooks, notify callee of hook err return fn(err); @@ -468,7 +478,7 @@ Runner.prototype.hooks = function(name, suites, fn) { return fn(); } - self.hook(name, function(err) { + self.hook(name, function cbHook(err) { if (err) { var errSuite = self.suite; self.suite = orig; @@ -571,20 +581,18 @@ Runner.prototype.runTests = function(suite, fn) { var tests = suite.tests.slice(); var test; - function hookErr(_, errSuite, after) { - // before-/afterEach hook for errSuite failed + function hookErr(_, errSuite) { + // afterEach hook for errSuite failed var orig = self.suite; - // for failed afterEach hook start from errSuite parent, - // otherwise start from errSuite itself - self.suite = after ? errSuite.parent : errSuite; + self.suite = errSuite.parent; if (self.suite) { self.hookUp(HOOK_TYPE_AFTER_EACH, function(err2, errSuite2) { self.suite = orig; // some hooks may fail even now if (err2) { - return hookErr(err2, errSuite2, true); + return hookErr(err2, errSuite2); } // report error suite fn(errSuite); @@ -602,12 +610,10 @@ Runner.prototype.runTests = function(suite, fn) { if (self._abort) return fn(); - if (err) return hookErr(err, errSuite, true); + if (err) return hookErr(err, errSuite); // next test test = tests.shift(); - - // all done if (!test) return fn(); // grep @@ -632,7 +638,7 @@ Runner.prototype.runTests = function(suite, fn) { return; } - // static skip or conditional skip within beforeAll + // static skip or conditional skip within hooks if (test.isPending()) { if (self.forbidPending) { self.fail(test, new Error('Pending test forbidden'), true); @@ -644,7 +650,7 @@ Runner.prototype.runTests = function(suite, fn) { return nextTest(); } - // skipped by failing beforeAll + // skipped by failing hooks if (test.isSkipped()) { test.state = STATE_SKIPPED; self.emit(constants.EVENT_TEST_SKIPPED, test); @@ -654,27 +660,29 @@ Runner.prototype.runTests = function(suite, fn) { // execute test and hook(s) self.emit(constants.EVENT_TEST_BEGIN, (self.test = test)); - self.hookDown(HOOK_TYPE_BEFORE_EACH, function(err, errSuite) { - // conditional skip within beforeEach - if (test.isPending()) { - if (self.forbidPending) { - self.fail(test, new Error('Pending test forbidden'), true); + self.hookDown(HOOK_TYPE_BEFORE_EACH, function cbHookDown(err, errSuite) { + if (err || test.isPending()) { + if (test.isPending()) { + if (self.forbidPending) { + self.fail(test, new Error('Pending test forbidden'), true); + } else { + test.state = STATE_PENDING; + self.emit(constants.EVENT_TEST_PENDING, test); + } } else { - test.state = STATE_PENDING; - self.emit(constants.EVENT_TEST_PENDING, test); + test.state = STATE_SKIPPED; + self.emit(constants.EVENT_TEST_SKIPPED, test); } self.emit(constants.EVENT_TEST_END, test); // skip inner afterEach hooks below errSuite level var origSuite = self.suite; self.suite = errSuite || self.suite; - return self.hookUp(HOOK_TYPE_AFTER_EACH, function(e, eSuite) { + return self.hookUp(HOOK_TYPE_AFTER_EACH, function cbHookUp(e, eSuite) { self.suite = origSuite; nextTest(e, eSuite); }); } - if (err) { - return hookErr(err, errSuite, false); - } + self.currentRunnable = self.test; self.runTest(function(err) { test = self.test; From 9bd10b3f1d0e8b324105c774077b74646663dacf Mon Sep 17 00:00:00 2001 From: juergba Date: Wed, 22 Apr 2020 18:09:39 +0200 Subject: [PATCH 3/4] extend existing tests, additional tests --- ...foreEach-async-deepnested-error.fixture.js | 18 +++ .../hooks/beforeEach-async-error.fixture.js | 15 +++ .../beforeEach-deepnested-error.fixture.js | 16 +++ .../hooks/beforeEach-error.fixture.js | 13 ++ .../beforeEach-hook-async-error.fixture.js | 21 ---- .../hooks/beforeEach-hook-error.fixture.js | 19 --- test/integration/hook-err.spec.js | 114 ++++++++++++++++-- 7 files changed, 164 insertions(+), 52 deletions(-) create mode 100644 test/integration/fixtures/hooks/beforeEach-async-deepnested-error.fixture.js create mode 100644 test/integration/fixtures/hooks/beforeEach-async-error.fixture.js create mode 100644 test/integration/fixtures/hooks/beforeEach-deepnested-error.fixture.js create mode 100644 test/integration/fixtures/hooks/beforeEach-error.fixture.js delete mode 100644 test/integration/fixtures/hooks/beforeEach-hook-async-error.fixture.js delete mode 100644 test/integration/fixtures/hooks/beforeEach-hook-error.fixture.js diff --git a/test/integration/fixtures/hooks/beforeEach-async-deepnested-error.fixture.js b/test/integration/fixtures/hooks/beforeEach-async-deepnested-error.fixture.js new file mode 100644 index 0000000000..9b73bb7bd5 --- /dev/null +++ b/test/integration/fixtures/hooks/beforeEach-async-deepnested-error.fixture.js @@ -0,0 +1,18 @@ +'use strict'; + +describe('spec 1', function () { + it('should run test-1', function () { }); + + describe('nested spec 2', function () { + beforeEach(function (done) { + process.nextTick(function () { + throw new Error('before each hook error'); + }); + }); + it('should not run nested test-2', function () { }); + + describe('deepnested spec 3', function () { + it('should not run deepnested test-3', function () { }); + }); + }); +}); diff --git a/test/integration/fixtures/hooks/beforeEach-async-error.fixture.js b/test/integration/fixtures/hooks/beforeEach-async-error.fixture.js new file mode 100644 index 0000000000..64dc8998b6 --- /dev/null +++ b/test/integration/fixtures/hooks/beforeEach-async-error.fixture.js @@ -0,0 +1,15 @@ +'use strict'; + +describe('spec 1', function () { + beforeEach(function (done) { + process.nextTick(function () { + throw new Error('before each hook error'); + }); + }); + it('should not run test-1', function () { }); + it('should not run test-2', function () { }); +}); + +describe('spec 2', function () { + it('should run test-3', function () { }); +}); diff --git a/test/integration/fixtures/hooks/beforeEach-deepnested-error.fixture.js b/test/integration/fixtures/hooks/beforeEach-deepnested-error.fixture.js new file mode 100644 index 0000000000..6a0de1a23e --- /dev/null +++ b/test/integration/fixtures/hooks/beforeEach-deepnested-error.fixture.js @@ -0,0 +1,16 @@ +'use strict'; + +describe('spec 1', function () { + it('should run test-1', function () { }); + + describe('nested spec 2', function () { + beforeEach(function() { + throw new Error('before each hook error'); + }); + it('should not run nested test-2', function () { }); + + describe('deepnested spec 3', function () { + it('should not run deepnested test-3', function () { }); + }); + }); +}); diff --git a/test/integration/fixtures/hooks/beforeEach-error.fixture.js b/test/integration/fixtures/hooks/beforeEach-error.fixture.js new file mode 100644 index 0000000000..701ff52b80 --- /dev/null +++ b/test/integration/fixtures/hooks/beforeEach-error.fixture.js @@ -0,0 +1,13 @@ +'use strict'; + +describe('spec 1', function () { + beforeEach(function () { + throw new Error('before each hook error'); + }); + it('should not run test-1', function () { }); + it('should not run test-2', function () { }); +}); + +describe('spec 2', function () { + it('should run test-3', function () { }); +}); diff --git a/test/integration/fixtures/hooks/beforeEach-hook-async-error.fixture.js b/test/integration/fixtures/hooks/beforeEach-hook-async-error.fixture.js deleted file mode 100644 index 6ce27784a6..0000000000 --- a/test/integration/fixtures/hooks/beforeEach-hook-async-error.fixture.js +++ /dev/null @@ -1,21 +0,0 @@ -'use strict'; - -describe('spec 1', function () { - beforeEach(function (done) { - console.log('before'); - process.nextTick(function () { - throw new Error('before each hook error'); - }); - }); - it('should not be called because of error in before each hook', function () { - console.log('test 1'); - }); - it('should not be called because of error in before each hook', function () { - console.log('test 2'); - }); -}); -describe('spec 2', function () { - it('should be called, because hook error was in a sibling suite', function () { - console.log('test 3'); - }); -}); diff --git a/test/integration/fixtures/hooks/beforeEach-hook-error.fixture.js b/test/integration/fixtures/hooks/beforeEach-hook-error.fixture.js deleted file mode 100644 index 4c0ab2f237..0000000000 --- a/test/integration/fixtures/hooks/beforeEach-hook-error.fixture.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict'; - -describe('spec 1', function () { - beforeEach(function () { - console.log('before'); - throw new Error('before each hook error'); - }); - it('should not be called because of error in before each hook', function () { - console.log('test 1'); - }); - it('should not be called because of error in before each hook', function () { - console.log('test 2'); - }); -}); -describe('spec 2', function () { - it('should be called, because hook error was in a sibling suite', function () { - console.log('test 3'); - }); -}); diff --git a/test/integration/hook-err.spec.js b/test/integration/hook-err.spec.js index 19c4cec727..80bf176a2a 100644 --- a/test/integration/hook-err.spec.js +++ b/test/integration/hook-err.spec.js @@ -116,6 +116,58 @@ describe('hook error handling', function() { }); }); }); + + describe('in before each', function() { + it('before each hook error', function(done) { + var fixture = 'hooks/beforeEach-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before each hook error') + .and('to have test count', 3) + .and('to have passed test count', 1) + .and('to have skipped test count', 2) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-3') + .and( + 'to have skipped test order', + 'should not run test-1', + 'should not run test-2' + ) + .and( + 'to have failed test', + '"before each" hook for "should not run test-1"' + ); + done(); + }); + }); + + it('before each hook deepnested error', function(done) { + var fixture = 'hooks/beforeEach-deepnested-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before each hook error') + .and('to have test count', 3) + .and('to have passed test count', 1) + .and('to have skipped test count', 2) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and( + 'to have skipped test order', + 'should not run nested test-2', + 'should not run deepnested test-3' + ) + .and( + 'to have failed test', + '"before each" hook for "should not run nested test-2"' + ); + done(); + }); + }); + }); }); describe('asynchronous hook', function() { @@ -166,12 +218,57 @@ describe('hook error handling', function() { }); }); }); - }); - describe('before each hook error', function() { - before(run('hooks/beforeEach-hook-error.fixture.js')); - it('should verify results', function() { - expect(lines, 'to equal', ['before', bang + 'test 3']); + describe('in before each', function() { + it('before each hook error', function(done) { + var fixture = 'hooks/beforeEach-async-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before each hook error') + .and('to have test count', 3) + .and('to have passed test count', 1) + .and('to have skipped test count', 2) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-3') + .and( + 'to have skipped test order', + 'should not run test-1', + 'should not run test-2' + ) + .and( + 'to have failed test', + '"before each" hook for "should not run test-1"' + ); + done(); + }); + }); + + it('before each hook deepnested error', function(done) { + var fixture = 'hooks/beforeEach-async-deepnested-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before each hook error') + .and('to have test count', 3) + .and('to have passed test count', 1) + .and('to have skipped test count', 2) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and( + 'to have skipped test order', + 'should not run nested test-2', + 'should not run deepnested test-3' + ) + .and( + 'to have failed test', + '"before each" hook for "should not run nested test-2"' + ); + done(); + }); + }); }); }); @@ -271,13 +368,6 @@ describe('hook error handling', function() { }); }); - describe('async - before each hook error', function() { - before(run('hooks/beforeEach-hook-async-error.fixture.js')); - it('should verify results', function() { - expect(lines, 'to equal', ['before', bang + 'test 3']); - }); - }); - describe('async - after hook error', function() { before(run('hooks/after-hook-async-error.fixture.js')); it('should verify results', function() { From d6b17005fbdcda16c9e741c92072176c343b4be0 Mon Sep 17 00:00:00 2001 From: juergba Date: Thu, 23 Apr 2020 19:13:36 +0200 Subject: [PATCH 4/4] adapt reporter xunit, json-stream, list, tap --- lib/reporters/json-stream.js | 10 ++++++++++ lib/reporters/list.js | 8 +++++++- lib/reporters/tap.js | 16 +++++++++++----- lib/reporters/xunit.js | 16 ++++++++++++---- test/reporters/tap.spec.js | 2 ++ test/reporters/xunit.spec.js | 5 ++--- 6 files changed, 44 insertions(+), 13 deletions(-) diff --git a/lib/reporters/json-stream.js b/lib/reporters/json-stream.js index 27282987ea..7275c24df9 100644 --- a/lib/reporters/json-stream.js +++ b/lib/reporters/json-stream.js @@ -9,6 +9,8 @@ var Base = require('./base'); var constants = require('../runner').constants; var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; var EVENT_RUN_BEGIN = constants.EVENT_RUN_BEGIN; var EVENT_RUN_END = constants.EVENT_RUN_END; @@ -43,6 +45,14 @@ function JSONStream(runner, options) { writeEvent(['pass', clean(test)]); }); + runner.on(EVENT_TEST_PENDING, function(test) { + writeEvent(['pend', clean(test)]); + }); + + runner.on(EVENT_TEST_SKIPPED, function(test) { + writeEvent(['skip', clean(test)]); + }); + runner.on(EVENT_TEST_FAIL, function(test, err) { test = clean(test); test.err = err.message; diff --git a/lib/reporters/list.js b/lib/reporters/list.js index c7ff8c4ea8..4baaec1bbc 100644 --- a/lib/reporters/list.js +++ b/lib/reporters/list.js @@ -15,6 +15,7 @@ var EVENT_TEST_BEGIN = constants.EVENT_TEST_BEGIN; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var color = Base.color; var cursor = Base.cursor; @@ -49,7 +50,12 @@ function List(runner, options) { }); runner.on(EVENT_TEST_PENDING, function(test) { - var fmt = color('checkmark', ' -') + color('pending', ' %s'); + var fmt = color('pending', ' - %s'); + Base.consoleLog(fmt, test.fullTitle()); + }); + + runner.on(EVENT_TEST_SKIPPED, function(test) { + var fmt = color('fail', ' - %s'); Base.consoleLog(fmt, test.fullTitle()); }); diff --git a/lib/reporters/tap.js b/lib/reporters/tap.js index 12257a745f..02fceaa930 100644 --- a/lib/reporters/tap.js +++ b/lib/reporters/tap.js @@ -10,11 +10,12 @@ var util = require('util'); var Base = require('./base'); var constants = require('../runner').constants; var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; +var EVENT_TEST_END = constants.EVENT_TEST_END; var EVENT_RUN_BEGIN = constants.EVENT_RUN_BEGIN; var EVENT_RUN_END = constants.EVENT_RUN_END; -var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; -var EVENT_TEST_END = constants.EVENT_TEST_END; var inherits = require('../utils').inherits; var sprintf = util.format; @@ -63,6 +64,10 @@ function TAP(runner, options) { self._producer.writePending(n, test); }); + runner.on(EVENT_TEST_SKIPPED, function(test) { + self._producer.writePending(n, test); + }); + runner.on(EVENT_TEST_PASS, function(test) { self._producer.writePass(n, test); }); @@ -199,10 +204,11 @@ TAPProducer.prototype.writeFail = function(n, test, err) { * @param {Object} stats - Object containing run statistics. */ TAPProducer.prototype.writeEpilogue = function(stats) { - // :TBD: Why is this not counting pending tests? - println('# tests ' + (stats.passes + stats.failures)); + println('# tests ' + stats.tests); println('# pass ' + stats.passes); - // :TBD: Why are we not showing pending results? + if (stats.pending || stats.skipped) { + println('# skip ' + (stats.pending + stats.skipped)); + } println('# fail ' + stats.failures); }; diff --git a/lib/reporters/xunit.js b/lib/reporters/xunit.js index a690ac5343..ad97bc7d1d 100644 --- a/lib/reporters/xunit.js +++ b/lib/reporters/xunit.js @@ -14,10 +14,14 @@ var errors = require('../errors'); var createUnsupportedError = errors.createUnsupportedError; var constants = require('../runner').constants; var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; var EVENT_RUN_END = constants.EVENT_RUN_END; -var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; -var STATE_FAILED = require('../runnable').constants.STATE_FAILED; +var Runnable = require('../runnable'); +var STATE_FAILED = Runnable.constants.STATE_FAILED; +var STATE_PENDING = Runnable.constants.STATE_PENDING; +var STATE_SKIPPED = Runnable.constants.STATE_SKIPPED; var inherits = utils.inherits; var escape = utils.escape; @@ -78,6 +82,10 @@ function XUnit(runner, options) { tests.push(test); }); + runner.on(EVENT_TEST_SKIPPED, function(test) { + tests.push(test); + }); + runner.on(EVENT_TEST_PASS, function(test) { tests.push(test); }); @@ -95,7 +103,7 @@ function XUnit(runner, options) { tests: stats.tests, failures: 0, errors: stats.failures, - skipped: stats.tests - stats.failures - stats.passes, + skipped: stats.pending + stats.skipped, timestamp: new Date().toUTCString(), time: stats.duration / 1000 || 0 }, @@ -180,7 +188,7 @@ XUnit.prototype.test = function(test) { ) ) ); - } else if (test.isPending()) { + } else if (test.state === STATE_PENDING || test.state === STATE_SKIPPED) { this.write(tag('testcase', attrs, false, tag('skipped', {}, true))); } else { this.write(tag('testcase', attrs, true)); diff --git a/test/reporters/tap.spec.js b/test/reporters/tap.spec.js index f3bfe8d473..333047e029 100644 --- a/test/reporters/tap.spec.js +++ b/test/reporters/tap.spec.js @@ -270,6 +270,7 @@ describe('TAP reporter', function() { EVENT_TEST_PASS, test ); + runner.stats.tests = 2; runner.suite = ''; runner.grepTotal = noop; stdout = runReporter({}, runner, options); @@ -544,6 +545,7 @@ describe('TAP reporter', function() { EVENT_TEST_PASS, test ); + runner.stats.tests = 2; runner.suite = ''; runner.grepTotal = noop; stdout = runReporter({}, runner, options); diff --git a/test/reporters/xunit.spec.js b/test/reporters/xunit.spec.js index 323db703a9..68f338f0ad 100644 --- a/test/reporters/xunit.spec.js +++ b/test/reporters/xunit.spec.js @@ -22,6 +22,7 @@ var EVENT_TEST_PENDING = events.EVENT_TEST_PENDING; var STATE_FAILED = states.STATE_FAILED; var STATE_PASSED = states.STATE_PASSED; +var STATE_PENDING = states.STATE_PENDING; describe('XUnit reporter', function() { var sandbox; @@ -397,15 +398,13 @@ describe('XUnit reporter', function() { it('should write expected tag', function() { var xunit = new XUnit(runner); var expectedTest = { - isPending: function() { - return true; - }, title: expectedTitle, parent: { fullTitle: function() { return expectedClassName; } }, + state: STATE_PENDING, duration: 1000 };