add output for failing tests (#837)

api.js

@@ -233,6 +233,7 @@ function getBlankResults() {
 		stats: {
 			testCount: 0,
 			passCount: 0,
+			knownFailureCount: 0,
 			skipCount: 0,
 			todoCount: 0,
 			failCount: 0

lib/reporters/mini.js

@@ -50,6 +50,7 @@ MiniReporter.prototype.start = function () {
 MiniReporter.prototype.reset = function () {
 	this.clearInterval();
 	this.passCount = 0;
+	this.knownFailureCount = 0;
 	this.failCount = 0;
 	this.skipCount = 0;
 	this.todoCount = 0;
@@ -80,6 +81,9 @@ MiniReporter.prototype.test = function (test) {
 		this.failCount++;
 	} else {
 		this.passCount++;
+		if (test.failing) {
+			this.knownFailureCount++;
+		}
 	}
 
 	if (test.todo || test.skip) {
@@ -102,7 +106,7 @@ MiniReporter.prototype._test = function (test) {
 	var PADDING = 1;
 	var title = cliTruncate(test.title, process.stdout.columns - SPINNER_WIDTH - PADDING);
 
-	if (test.error) {
+	if (test.error || test.failing) {
 		title = colors.error(test.title);
 	}
 
@@ -120,6 +124,7 @@ MiniReporter.prototype.unhandledError = function (err) {
 MiniReporter.prototype.reportCounts = function (time) {
 	var lines = [
 		this.passCount > 0 ? '\n   ' + colors.pass(this.passCount, 'passed') : '',
+		this.knownFailureCount > 0 ? '\n   ' + colors.error(this.knownFailureCount, plur('known failure', this.knownFailureCount)) : '',
 		this.failCount > 0 ? '\n   ' + colors.error(this.failCount, 'failed') : '',
 		this.skipCount > 0 ? '\n   ' + colors.skip(this.skipCount, 'skipped') : '',
 		this.todoCount > 0 ? '\n   ' + colors.todo(this.todoCount, 'todo') : ''
@@ -156,6 +161,18 @@ MiniReporter.prototype.finish = function (runStatus) {
 
 	var i = 0;
 
+	if (this.knownFailureCount > 0) {
+		runStatus.knownFailures.forEach(function (test) {
+			i++;
+
+			var title = test.title;
+
+			status += '\n\n\n   ' + colors.error(i + '.', title);
+			// TODO output description with link
+			// status += colors.stack(description);
+		});
+	}
+
 	if (this.failCount > 0) {
 		runStatus.errors.forEach(function (test) {
 			if (!test.error || !test.error.message) {

lib/reporters/verbose.js

@@ -33,6 +33,10 @@ VerboseReporter.prototype.test = function (test, runStatus) {
 		return '  ' + colors.skip('- ' + test.title);
 	}
 
+	if (test.failing) {
+		return '  ' + colors.error(figures.tick) + ' ' + colors.error(test.title);
+	}
+
 	if (runStatus.fileCount === 1 && runStatus.testCount === 1 && test.title === '[anonymous]') {
 		return undefined;
 	}
@@ -74,6 +78,7 @@ VerboseReporter.prototype.finish = function (runStatus) {
 		runStatus.failCount > 0 ?
 			'  ' + colors.error(runStatus.failCount, plur('test', runStatus.failCount), 'failed') :
 			'  ' + colors.pass(runStatus.passCount, plur('test', runStatus.passCount), 'passed'),
+		runStatus.knownFailureCount > 0 ? '  ' + colors.error(runStatus.knownFailureCount, plur('known failure', runStatus.knownFailureCount)) : '',
 		runStatus.skipCount > 0 ? '  ' + colors.skip(runStatus.skipCount, plur('test', runStatus.skipCount), 'skipped') : '',
 		runStatus.todoCount > 0 ? '  ' + colors.todo(runStatus.todoCount, plur('test', runStatus.todoCount), 'todo') : '',
 		runStatus.rejectionCount > 0 ? '  ' + colors.error(runStatus.rejectionCount, 'unhandled', plur('rejection', runStatus.rejectionCount)) : '',
@@ -86,9 +91,16 @@ VerboseReporter.prototype.finish = function (runStatus) {
 		output += lines.join('\n');
 	}
 
-	if (runStatus.failCount > 0) {
-		var i = 0;
+	var i = 0;
+
+	if (runStatus.knownFailureCount > 0) {
+		runStatus.knownFailures.forEach(function (test) {
+			i++;
+			output += '\n\n\n  ' + colors.error(i + '.', test.title);
+		});
+	}
 
+	if (runStatus.failCount > 0) {
 		runStatus.tests.forEach(function (test) {
 			if (!(test.error && test.error.message)) {
 				return;

lib/run-status.js

@@ -22,12 +22,14 @@ function RunStatus(opts) {
 	this.rejectionCount = 0;
 	this.exceptionCount = 0;
 	this.passCount = 0;
+	this.knownFailureCount = 0;
 	this.skipCount = 0;
 	this.todoCount = 0;
 	this.failCount = 0;
 	this.fileCount = 0;
 	this.testCount = 0;
 	this.previousFailCount = 0;
+	this.knownFailures = [];
 	this.errors = [];
 	this.stats = [];
 	this.tests = [];
@@ -123,6 +125,10 @@ RunStatus.prototype.handleTest = function (test) {
 		this.errors.push(test);
 	}
 
+	if (test.failing && !test.error) {
+		this.knownFailures.push(test);
+	}
+
 	this.emit('test', test, this);
 };
 
@@ -172,6 +178,7 @@ RunStatus.prototype.processResults = function (results) {
 	this.tests = flatten(this.tests);
 
 	this.passCount = sum(this.stats, 'passCount');
+	this.knownFailureCount = sum(this.stats, 'knownFailureCount');
 	this.skipCount = sum(this.stats, 'skipCount');
 	this.todoCount = sum(this.stats, 'todoCount');
 	this.failCount = sum(this.stats, 'failCount');

lib/runner.js

@@ -138,7 +138,8 @@ Runner.prototype._addTestResult = function (result) {
 		error: result.reason,
 		type: test.metadata.type,
 		skip: test.metadata.skipped,
-		todo: test.metadata.todo
+		todo: test.metadata.todo,
+		failing: test.metadata.failing
 	};
 
 	this.results.push(result);
@@ -177,6 +178,12 @@ Runner.prototype._buildStats = function () {
 		})
 		.length;
 
+	stats.knownFailureCount = this.results
+		.filter(function (result) {
+			return result.passed === true && result.result.metadata.failing;
+		})
+		.length;
+
 	stats.passCount = stats.testCount - stats.failCount - stats.skipCount - stats.todoCount;
 
 	return stats;

lib/test.js

@@ -190,7 +190,9 @@ Test.prototype._result = function () {
 	var passed = reason === undefined;
 	if (this.metadata.failing) {
 		passed = !passed;
-		if (!passed) {
+		if (passed) {
+			reason = undefined;
+		} else {
 			reason = new Error('Test was expected to fail, but succeeded, you should stop marking the test as failing');
 		}
 	}

test/reporters/mini.js

@@ -55,6 +55,26 @@ test('passing test', function (t) {
 	t.end();
 });
 
+test('known failure test', function (t) {
+	var reporter = miniReporter();
+
+	var actualOutput = reporter.test({
+		title: 'known failure',
+		failing: true
+	});
+
+	var expectedOutput = [
+		' ',
+		' ' + graySpinner + ' ' + chalk.red('known failure'),
+		'',
+		'   ' + chalk.green('1 passed'),
+		'   ' + chalk.red('1 known failure')
+	].join('\n');
+
+	t.is(actualOutput, expectedOutput);
+	t.end();
+});
+
 test('failing test', function (t) {
 	var reporter = miniReporter();
 
@@ -76,6 +96,28 @@ test('failing test', function (t) {
 	t.end();
 });
 
+test('failed known failure test', function (t) {
+	var reporter = miniReporter();
+
+	var actualOutput = reporter.test({
+		title: 'known failure',
+		failing: true,
+		error: {
+			message: 'Test was expected to fail, but succeeded, you should stop marking the test as failing'
+		}
+	});
+
+	var expectedOutput = [
+		' ',
+		' ' + graySpinner + ' ' + chalk.red('known failure'),
+		'',
+		'   ' + chalk.red('1 failed')
+	].join('\n');
+
+	t.is(actualOutput, expectedOutput);
+	t.end();
+});
+
 test('passing test after failing', function (t) {
 	var reporter = miniReporter();
 
@@ -164,6 +206,29 @@ test('results with passing tests', function (t) {
 	t.end();
 });
 
+test('results with passing known failure tests', function (t) {
+	var reporter = miniReporter();
+	reporter.passCount = 1;
+	reporter.knownFailureCount = 1;
+	reporter.failCount = 0;
+
+	var runStatus = {
+		knownFailures: [{title: 'known failure', failing: true}]
+	};
+	var actualOutput = reporter.finish(runStatus);
+	var expectedOutput = [
+		'\n   ' + chalk.green('1 passed'),
+		'   ' + chalk.red('1 known failure'),
+		'',
+		'',
+		'   ' + chalk.red('1. known failure'),
+		''
+	].join('\n');
+
+	t.is(actualOutput, expectedOutput);
+	t.end();
+});
+
 test('results with skipped tests', function (t) {
 	var reporter = miniReporter();
 	reporter.passCount = 0;

test/reporters/verbose.js

@@ -89,6 +89,20 @@ test('don\'t display test title if there is only one anonymous test', function (
 	t.end();
 });
 
+test('known failure test', function (t) {
+	var reporter = createReporter();
+
+	var actualOutput = reporter.test({
+		title: 'known failure',
+		failing: true
+	}, createRunStatus());
+
+	var expectedOutput = '  ' + chalk.red(figures.tick) + ' ' + chalk.red('known failure');
+
+	t.is(actualOutput, expectedOutput);
+	t.end();
+});
+
 test('failing test', function (t) {
 	var reporter = createReporter();
 
@@ -214,6 +228,28 @@ test('results with passing tests', function (t) {
 	t.end();
 });
 
+test('results with passing known failure tests', function (t) {
+	var reporter = createReporter();
+	var runStatus = createRunStatus();
+	runStatus.passCount = 1;
+	runStatus.knownFailureCount = 1;
+	runStatus.knownFailures = [{title: 'known failure', failing: true}];
+
+	var actualOutput = reporter.finish(runStatus);
+	var expectedOutput = [
+		'',
+		'  ' + chalk.green('1 test passed') + time,
+		'  ' + chalk.red('1 known failure'),
+		'',
+		'',
+		'  ' + chalk.red('1. known failure'),
+		''
+	].join('\n');
+
+	t.is(actualOutput, expectedOutput);
+	t.end();
+});
+
 test('results with skipped tests', function (t) {
 	var reporter = createReporter();
 	var runStatus = createRunStatus();

test/test.js

@@ -622,13 +622,13 @@ test('failing tests should fail', function (t) {
 	t.end();
 });
 
-test('failing callback tests should end with an error', function (t) {
+test('failing callback tests should end without error', function (t) {
 	var err = new Error('failed');
 	ava.cb.failing(function (a) {
 		a.end(err);
 	}).run().then(function (result) {
 		t.is(result.passed, true);
-		t.is(result.reason, err);
+		t.is(result.reason, undefined);
 		t.end();
 	});
 });