diff --git a/README.md b/README.md index 35fff9cf..79a1dd7d 100755 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ global manipulation. Our goal with **lab** is to keep the execution engine as si **lab** supports the following command line options: - `-a`, `--assert` - name of assert library to use. +- `--bail` - terminate the process with a non-zero exit code on the first test failure. Defaults to `false`. - `-c`, `--coverage` - enables code coverage analysis. - `--coverage-path` - sets code coverage path. - `--coverage-exclude` - sets code coverage excludes. diff --git a/lib/cli.js b/lib/cli.js index 3732cdbb..0cf1c035 100755 --- a/lib/cli.js +++ b/lib/cli.js @@ -146,6 +146,11 @@ internals.options = function () { description: 'specify an assertion library module path to require and make available under Lab.assertions', default: null }, + bail: { + type: 'boolean', + description: 'exit the process with a non zero exit code on the first test failure', + default: null + }, colors: { alias: 'C', type: 'boolean', @@ -354,7 +359,7 @@ internals.options = function () { }; const defaults = { - paths: ['test'], + bail: false, coverage: false, debug: false, dry: false, @@ -367,6 +372,7 @@ internals.options = function () { 'lint-errors-threshold': 0, 'lint-warnings-threshold': 0, parallel: false, + paths: ['test'], rejections: false, reporter: 'console', shuffle: false, @@ -398,7 +404,7 @@ internals.options = function () { const options = Utils.mergeOptions(defaults, internals.rc); options.paths = argv._ ? [].concat(argv._) : options.paths; - const keys = ['assert', 'colors', 'context-timeout', 'coverage', 'coverage-exclude', + const keys = ['assert', 'bail', 'colors', 'context-timeout', 'coverage', 'coverage-exclude', 'coverage-path', 'debug', 'dry', 'environment', 'flat', 'globals', 'grep', 'lint', 'lint-errors-threshold', 'lint-fix', 'lint-options', 'lint-warnings-threshold', 'linter', 'output', 'parallel', 'pattern', 'rejections', 'reporter', 'seed', 'shuffle', 'silence', diff --git a/lib/runner.js b/lib/runner.js index bce1dc1f..8814e3a5 100755 --- a/lib/runner.js +++ b/lib/runner.js @@ -31,6 +31,7 @@ Error.stackTraceLimit = Infinity; // Set Error stack size internals.defaults = { // assert: { incomplete(), count() }, + bail: false, coverage: false, // coveragePath: process.cwd(), @@ -326,6 +327,10 @@ internals.executeExperiments = function (experiments, state, skip, callback) { state.report.errors.push(err); } + if (state.report.errors.length && state.options.bail === true) { + return callback(); + } + nextExperiment(); }); }, @@ -356,6 +361,10 @@ internals.executeTests = function (experiment, state, skip, callback) { return callback(); } + // Set to an error when options.bail is set and a test fails + + let bail = null; + // Collect beforeEach and afterEach from parents const befores = skip ? [] : internals.collectDeps(experiment, 'beforeEaches'); @@ -430,12 +439,14 @@ internals.executeTests = function (experiment, state, skip, callback) { state.report.failures++; test.err = err; test.timeout = err.timeout; + bail = state.options.bail === true ? err : null; } test.duration = Date.now() - start; state.report.tests.push(test); state.reporter.test(test); + return next(); }); }, @@ -461,6 +472,10 @@ internals.executeTests = function (experiment, state, skip, callback) { state.report.errors.push(err); } + if (bail) { + return callback(bail); + } + return nextTest(); }); }; diff --git a/test/cli.js b/test/cli.js index 66174d6d..c90b3247 100755 --- a/test/cli.js +++ b/test/cli.js @@ -118,6 +118,21 @@ describe('CLI', () => { }); }); + it('(--bail) exits with code 1 running a directory of tests after one fails', (done) => { + + RunCli(['test/cli_bail', '-m', '2000', '--bail'], (error, result) => { + + if (error) { + done(error); + } + + expect(result.code).to.equal(1); + expect(result.output).to.contain('Expected 1 to equal specified value'); + expect(result.output).to.contain('1 of 2 tests failed'); + done(); + }); + }); + it('exits with code 1 when function returns error with multiple reporters', (done) => { RunCli(['test/cli_failure/failure.js', '-r', 'console', '-r', 'lcov'], (error, result) => { diff --git a/test/cli_bail/test1.js b/test/cli_bail/test1.js new file mode 100644 index 00000000..313de24a --- /dev/null +++ b/test/cli_bail/test1.js @@ -0,0 +1,44 @@ +'use strict'; + +// Load modules + +const Code = require('code'); +const _Lab = require('../../test_runner'); + + +// Declare internals + +const internals = {}; + + +// Test shortcuts + +const lab = exports.lab = _Lab.script(); +const before = lab.before; +const after = lab.after; +const describe = lab.describe; +const it = lab.it; +const expect = Code.expect; + + +describe('test1', () => { + + before((done) => { + + process.nextTick(done); + }); + + it('should add numbers', (done) => { + + process.nextTick(() => { + + expect(1 + 1).to.equal(2); + done(); + }); + }); + + after((done) => { + + process.nextTick(done); + }); +}); diff --git a/test/cli_bail/test2.js b/test/cli_bail/test2.js new file mode 100644 index 00000000..7bae60d8 --- /dev/null +++ b/test/cli_bail/test2.js @@ -0,0 +1,44 @@ +'use strict'; + +// Load modules + +const Code = require('code'); +const _Lab = require('../../test_runner'); + + +// Declare internals + +const internals = {}; + + +// Test shortcuts + +const lab = exports.lab = _Lab.script(); +const before = lab.before; +const after = lab.after; +const describe = lab.describe; +const it = lab.it; +const expect = Code.expect; + + +describe('test2', () => { + + before((done) => { + + process.nextTick(done); + }); + + it('should multiply numbers', (done) => { + + process.nextTick(() => { + + expect(1 * 1).to.equal(2); + done(); + }); + }); + + after((done) => { + + process.nextTick(done); + }); +}); diff --git a/test/cli_bail/test3.js b/test/cli_bail/test3.js new file mode 100644 index 00000000..fa4c8d98 --- /dev/null +++ b/test/cli_bail/test3.js @@ -0,0 +1,44 @@ +'use strict'; + +// Load modules + +const Code = require('code'); +const _Lab = require('../../test_runner'); + + +// Declare internals + +const internals = {}; + + +// Test shortcuts + +const lab = exports.lab = _Lab.script(); +const before = lab.before; +const after = lab.after; +const describe = lab.describe; +const it = lab.it; +const expect = Code.expect; + + +describe('test2', () => { + + before((done) => { + + process.nextTick(done); + }); + + it('should multiply numbers', (done) => { + + process.nextTick(() => { + + expect(1 * 1).to.equal(1); + done(); + }); + }); + + after((done) => { + + process.nextTick(done); + }); +}); diff --git a/test/runner.js b/test/runner.js index e0068dbd..78382064 100755 --- a/test/runner.js +++ b/test/runner.js @@ -786,7 +786,38 @@ describe('Runner', () => { }); }); - it('dry run', (done) => { + it('bail will terminate on the first test failure', (done) => { + + const script = Lab.script(); + script.experiment('test', () => { + + script.test('1', (testDone) => { + + testDone(); + }); + + script.test('2', (testDone) => { + + throw new Error('bailing'); + }); + + script.test('3', (testDone) => { + + testDone(); + }); + }); + + Lab.execute(script, { bail: true }, null, (err, notebook) => { + + expect(err).not.to.exist(); + expect(notebook.tests).to.have.length(2); + expect(notebook.failures).to.equal(1); + expect(notebook.errors[0].message).to.contain('bailing'); + done(); + }); + }); + + it('dry run won\'t execute tests', (done) => { const script = Lab.script(); script.experiment('test', () => {