diff --git a/api.js b/api.js index 80f6e6a1f..2b20af905 100644 --- a/api.js +++ b/api.js @@ -233,6 +233,7 @@ function getBlankResults() { stats: { testCount: 0, passCount: 0, + knownFailureCount: 0, skipCount: 0, todoCount: 0, failCount: 0 diff --git a/lib/reporters/mini.js b/lib/reporters/mini.js index 8c04c2b4b..7f7c8cbbb 100644 --- a/lib/reporters/mini.js +++ b/lib/reporters/mini.js @@ -50,6 +50,7 @@ MiniReporter.prototype.start = function () { MiniReporter.prototype.reset = function () { this.clearInterval(); this.passCount = 0; + this.knownFailureCount = 0; this.failCount = 0; this.skipCount = 0; this.todoCount = 0; @@ -80,6 +81,9 @@ MiniReporter.prototype.test = function (test) { this.failCount++; } else { this.passCount++; + if (test.failing) { + this.knownFailureCount++; + } } if (test.todo || test.skip) { @@ -102,7 +106,7 @@ MiniReporter.prototype._test = function (test) { var PADDING = 1; var title = cliTruncate(test.title, process.stdout.columns - SPINNER_WIDTH - PADDING); - if (test.error) { + if (test.error || test.failing) { title = colors.error(test.title); } @@ -120,6 +124,7 @@ MiniReporter.prototype.unhandledError = function (err) { MiniReporter.prototype.reportCounts = function (time) { var lines = [ this.passCount > 0 ? '\n ' + colors.pass(this.passCount, 'passed') : '', + this.knownFailureCount > 0 ? '\n ' + colors.error(this.knownFailureCount, plur('known failure', this.knownFailureCount)) : '', this.failCount > 0 ? '\n ' + colors.error(this.failCount, 'failed') : '', this.skipCount > 0 ? '\n ' + colors.skip(this.skipCount, 'skipped') : '', this.todoCount > 0 ? '\n ' + colors.todo(this.todoCount, 'todo') : '' @@ -156,6 +161,18 @@ MiniReporter.prototype.finish = function (runStatus) { var i = 0; + if (this.knownFailureCount > 0) { + runStatus.knownFailures.forEach(function (test) { + i++; + + var title = test.title; + + status += '\n\n\n ' + colors.error(i + '.', title); + // TODO output description with link + // status += colors.stack(description); + }); + } + if (this.failCount > 0) { runStatus.errors.forEach(function (test) { if (!test.error || !test.error.message) { diff --git a/lib/reporters/verbose.js b/lib/reporters/verbose.js index 05cd41134..95afb014d 100644 --- a/lib/reporters/verbose.js +++ b/lib/reporters/verbose.js @@ -33,6 +33,10 @@ VerboseReporter.prototype.test = function (test, runStatus) { return ' ' + colors.skip('- ' + test.title); } + if (test.failing) { + return ' ' + colors.error(figures.tick) + ' ' + colors.error(test.title); + } + if (runStatus.fileCount === 1 && runStatus.testCount === 1 && test.title === '[anonymous]') { return undefined; } @@ -74,6 +78,7 @@ VerboseReporter.prototype.finish = function (runStatus) { runStatus.failCount > 0 ? ' ' + colors.error(runStatus.failCount, plur('test', runStatus.failCount), 'failed') : ' ' + colors.pass(runStatus.passCount, plur('test', runStatus.passCount), 'passed'), + runStatus.knownFailureCount > 0 ? ' ' + colors.error(runStatus.knownFailureCount, plur('known failure', runStatus.knownFailureCount)) : '', runStatus.skipCount > 0 ? ' ' + colors.skip(runStatus.skipCount, plur('test', runStatus.skipCount), 'skipped') : '', runStatus.todoCount > 0 ? ' ' + colors.todo(runStatus.todoCount, plur('test', runStatus.todoCount), 'todo') : '', runStatus.rejectionCount > 0 ? ' ' + colors.error(runStatus.rejectionCount, 'unhandled', plur('rejection', runStatus.rejectionCount)) : '', @@ -86,9 +91,16 @@ VerboseReporter.prototype.finish = function (runStatus) { output += lines.join('\n'); } - if (runStatus.failCount > 0) { - var i = 0; + var i = 0; + + if (runStatus.knownFailureCount > 0) { + runStatus.knownFailures.forEach(function (test) { + i++; + output += '\n\n\n ' + colors.error(i + '.', test.title); + }); + } + if (runStatus.failCount > 0) { runStatus.tests.forEach(function (test) { if (!(test.error && test.error.message)) { return; diff --git a/lib/run-status.js b/lib/run-status.js index 62d18a326..7cbebd4c8 100644 --- a/lib/run-status.js +++ b/lib/run-status.js @@ -22,12 +22,14 @@ function RunStatus(opts) { this.rejectionCount = 0; this.exceptionCount = 0; this.passCount = 0; + this.knownFailureCount = 0; this.skipCount = 0; this.todoCount = 0; this.failCount = 0; this.fileCount = 0; this.testCount = 0; this.previousFailCount = 0; + this.knownFailures = []; this.errors = []; this.stats = []; this.tests = []; @@ -123,6 +125,10 @@ RunStatus.prototype.handleTest = function (test) { this.errors.push(test); } + if (test.failing && !test.error) { + this.knownFailures.push(test); + } + this.emit('test', test, this); }; @@ -172,6 +178,7 @@ RunStatus.prototype.processResults = function (results) { this.tests = flatten(this.tests); this.passCount = sum(this.stats, 'passCount'); + this.knownFailureCount = sum(this.stats, 'knownFailureCount'); this.skipCount = sum(this.stats, 'skipCount'); this.todoCount = sum(this.stats, 'todoCount'); this.failCount = sum(this.stats, 'failCount'); diff --git a/lib/runner.js b/lib/runner.js index 909db8a6f..250159372 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -138,7 +138,8 @@ Runner.prototype._addTestResult = function (result) { error: result.reason, type: test.metadata.type, skip: test.metadata.skipped, - todo: test.metadata.todo + todo: test.metadata.todo, + failing: test.metadata.failing }; this.results.push(result); @@ -177,6 +178,12 @@ Runner.prototype._buildStats = function () { }) .length; + stats.knownFailureCount = this.results + .filter(function (result) { + return result.passed === true && result.result.metadata.failing; + }) + .length; + stats.passCount = stats.testCount - stats.failCount - stats.skipCount - stats.todoCount; return stats; diff --git a/lib/test.js b/lib/test.js index 84d698556..d682e4bd6 100644 --- a/lib/test.js +++ b/lib/test.js @@ -190,7 +190,9 @@ Test.prototype._result = function () { var passed = reason === undefined; if (this.metadata.failing) { passed = !passed; - if (!passed) { + if (passed) { + reason = undefined; + } else { reason = new Error('Test was expected to fail, but succeeded, you should stop marking the test as failing'); } } diff --git a/test/reporters/mini.js b/test/reporters/mini.js index 2889c5296..c55813d75 100644 --- a/test/reporters/mini.js +++ b/test/reporters/mini.js @@ -55,6 +55,26 @@ test('passing test', function (t) { t.end(); }); +test('known failure test', function (t) { + var reporter = miniReporter(); + + var actualOutput = reporter.test({ + title: 'known failure', + failing: true + }); + + var expectedOutput = [ + ' ', + ' ' + graySpinner + ' ' + chalk.red('known failure'), + '', + ' ' + chalk.green('1 passed'), + ' ' + chalk.red('1 known failure') + ].join('\n'); + + t.is(actualOutput, expectedOutput); + t.end(); +}); + test('failing test', function (t) { var reporter = miniReporter(); @@ -76,6 +96,28 @@ test('failing test', function (t) { t.end(); }); +test('failed known failure test', function (t) { + var reporter = miniReporter(); + + var actualOutput = reporter.test({ + title: 'known failure', + failing: true, + error: { + message: 'Test was expected to fail, but succeeded, you should stop marking the test as failing' + } + }); + + var expectedOutput = [ + ' ', + ' ' + graySpinner + ' ' + chalk.red('known failure'), + '', + ' ' + chalk.red('1 failed') + ].join('\n'); + + t.is(actualOutput, expectedOutput); + t.end(); +}); + test('passing test after failing', function (t) { var reporter = miniReporter(); @@ -164,6 +206,29 @@ test('results with passing tests', function (t) { t.end(); }); +test('results with passing known failure tests', function (t) { + var reporter = miniReporter(); + reporter.passCount = 1; + reporter.knownFailureCount = 1; + reporter.failCount = 0; + + var runStatus = { + knownFailures: [{title: 'known failure', failing: true}] + }; + var actualOutput = reporter.finish(runStatus); + var expectedOutput = [ + '\n ' + chalk.green('1 passed'), + ' ' + chalk.red('1 known failure'), + '', + '', + ' ' + chalk.red('1. known failure'), + '' + ].join('\n'); + + t.is(actualOutput, expectedOutput); + t.end(); +}); + test('results with skipped tests', function (t) { var reporter = miniReporter(); reporter.passCount = 0; diff --git a/test/reporters/verbose.js b/test/reporters/verbose.js index 8eca900e9..83d5c92a4 100644 --- a/test/reporters/verbose.js +++ b/test/reporters/verbose.js @@ -89,6 +89,20 @@ test('don\'t display test title if there is only one anonymous test', function ( t.end(); }); +test('known failure test', function (t) { + var reporter = createReporter(); + + var actualOutput = reporter.test({ + title: 'known failure', + failing: true + }, createRunStatus()); + + var expectedOutput = ' ' + chalk.red(figures.tick) + ' ' + chalk.red('known failure'); + + t.is(actualOutput, expectedOutput); + t.end(); +}); + test('failing test', function (t) { var reporter = createReporter(); @@ -214,6 +228,28 @@ test('results with passing tests', function (t) { t.end(); }); +test('results with passing known failure tests', function (t) { + var reporter = createReporter(); + var runStatus = createRunStatus(); + runStatus.passCount = 1; + runStatus.knownFailureCount = 1; + runStatus.knownFailures = [{title: 'known failure', failing: true}]; + + var actualOutput = reporter.finish(runStatus); + var expectedOutput = [ + '', + ' ' + chalk.green('1 test passed') + time, + ' ' + chalk.red('1 known failure'), + '', + '', + ' ' + chalk.red('1. known failure'), + '' + ].join('\n'); + + t.is(actualOutput, expectedOutput); + t.end(); +}); + test('results with skipped tests', function (t) { var reporter = createReporter(); var runStatus = createRunStatus(); diff --git a/test/test.js b/test/test.js index 3f45a329a..09efdeb64 100644 --- a/test/test.js +++ b/test/test.js @@ -622,13 +622,13 @@ test('failing tests should fail', function (t) { t.end(); }); -test('failing callback tests should end with an error', function (t) { +test('failing callback tests should end without error', function (t) { var err = new Error('failed'); ava.cb.failing(function (a) { a.end(err); }).run().then(function (result) { t.is(result.passed, true); - t.is(result.reason, err); + t.is(result.reason, undefined); t.end(); }); });