Skip to content

Commit

Permalink
Implement support for todo tests and revamp reporting logic (#137)
Browse files Browse the repository at this point in the history 
* Update qunit.log and qunit.testDone logic to handle todo tests

* Revamp reporting logic to be based on failed tests

In order to properly support todo tests, the status of the test run
should be determined by status of complete tests and not individual
assertions.
  • Loading branch information
@trentmwillis @vladikoff
trentmwillis authored and vladikoff committed Feb 7, 2017
1 parent 0a4834f commit 347c126
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 28 deletions.
2 changes: 1 addition & 1 deletion Gruntfile.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ module.exports = function(grunt) {
options: {
callback: function(err, stdout, stderr, cb) {
if (/test\/qunit[45]\.html/.test(stdout) &&
/[12] assertions passed/.test(stdout)) {
/passed: [12]/.test(stdout)) {
cb();
} else {
cb(false);
Expand Down
4 changes: 2 additions & 2 deletions phantomjs/bridge.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,15 @@
expected = dump.parse(obj.expected);
}
// Send it.
sendMessage('qunit.log', obj.result, actual, expected, obj.message, obj.source);
sendMessage('qunit.log', obj.result, actual, expected, obj.message, obj.source, obj.todo);
});

QUnit.testStart(function(obj) {
sendMessage('qunit.testStart', obj.name);
});

QUnit.testDone(function(obj) {
sendMessage('qunit.testDone', obj.name, obj.failed, obj.passed, obj.total, obj.duration);
sendMessage('qunit.testDone', obj.name, obj.failed, obj.passed, obj.total, obj.runtime, obj.skipped, obj.todo);
});

QUnit.moduleStart(function(obj) {
Expand Down
123 changes: 98 additions & 25 deletions tasks/qunit.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ module.exports = function(grunt) {
// External lib.
var phantomjs = require('grunt-lib-phantomjs').init(grunt);

// Keep track of the last-started module, test and status.
var options, currentModule, currentTest, status;
// Keep track of the last-started module and test. Additionally, keep track
// of status for individual test files and the entire test suite.
var options, currentModule, currentTest, currentStatus, status;

// Keep track of the last-started test(s).
var unfinished = {};

Expand Down Expand Up @@ -62,6 +64,54 @@ module.exports = function(grunt) {
grunt.log.writeln();
}
};

var createStatus = function() {
return {
passed: 0,
failed: 0,
skipped: 0,
todo: 0,
runtime: 0,
assertions: {
passed: 0,
failed: 0
}
};
};

var mergeStatus = function(statusA, statusB) {
statusA.passed += statusB.passed;
statusA.failed += statusB.failed;
statusA.skipped += statusB.skipped;
statusA.todo += statusB.todo;
statusA.runtime += statusB.runtime;
statusA.assertions.passed += statusB.assertions.passed;
statusA.assertions.failed += statusB.assertions.failed;
};

var generateMessage = function(status) {
var totalTests = status.passed + status.failed + status.skipped + status.todo;
var totalAssertions = status.assertions.passed + status.assertions.failed;

return [
totalTests,
" tests completed with ",
status.failed,
" failed, " +
status.skipped,
" skipped, and ",
status.todo,
" todo. \n" +
totalAssertions,
" assertions (in ",
status.runtime,
"ms), passed: " +
status.assertions.passed,
", failed: ",
status.assertions.failed
].join( "" );
};

// Copied from QUnit source code
var generateHash = function(module) {
var hex;
Expand All @@ -87,6 +137,10 @@ module.exports = function(grunt) {
};

// QUnit hooks.
phantomjs.on('qunit.begin', function() {
currentStatus = createStatus();
});

phantomjs.on('qunit.moduleStart', function(name) {
unfinished[name] = true;
currentModule = name;
Expand All @@ -96,10 +150,13 @@ module.exports = function(grunt) {
delete unfinished[name];
});

phantomjs.on('qunit.log', function(result, actual, expected, message, source) {
if (!result) {
phantomjs.on('qunit.log', function(result, actual, expected, message, source, todo) {
if (!result && !todo) {
failedAssertions.push({
actual: actual, expected: expected, message: message, source: source,
actual: actual,
expected: expected,
message: message,
source: source,
testName: currentTest
});
}
Expand All @@ -110,13 +167,30 @@ module.exports = function(grunt) {
grunt.verbose.write(currentTest + '...');
});

phantomjs.on('qunit.testDone', function(name, failed/*, passed, total*/) {
phantomjs.on('qunit.testDone', function(name, failed, passed, total, runtime, skipped, todo) {
var testPassed = failed > 0 ? todo : !todo;

if (skipped) {
currentStatus.skipped++;
} else if (!testPassed) {
currentStatus.failed++;
} else if (todo) {
currentStatus.todo++;
} else {
currentStatus.passed++;
}

// Log errors if necessary, otherwise success.
if (failed > 0) {
// list assertions
if (!testPassed) {
// list assertions or message about todo failure
if (grunt.option('verbose')) {
grunt.log.error();
logFailedAssertions();

if (todo) {
grunt.log.error('Expected at least one failing assertion in todo test:' + name);
} else {
logFailedAssertions();
}
} else {
grunt.log.write('F'.red);
}
Expand All @@ -125,23 +199,24 @@ module.exports = function(grunt) {
}
});

phantomjs.on('qunit.done', function(failed, passed, total, duration) {
phantomjs.on('qunit.done', function(failed, passed, total, runtime) {
phantomjs.halt();
status.failed += failed;
status.passed += passed;
status.total += total;
status.duration += duration;

currentStatus.runtime += runtime;
currentStatus.assertions.passed += passed;
currentStatus.assertions.failed += failed;

// Print assertion errors here, if verbose mode is disabled.
if (!grunt.option('verbose')) {
if (failed > 0) {
if (currentStatus.failed > 0) {
grunt.log.writeln();
logFailedAssertions();
} else if (total === 0) {
warnUnlessForced('0/0 assertions ran (' + duration + 'ms)');
} else {
grunt.log.ok();
}
}

mergeStatus(status, currentStatus);
});

// Re-broadcast qunit events on grunt.event.
Expand All @@ -156,8 +231,8 @@ module.exports = function(grunt) {
grunt.verbose.write('...');
grunt.event.emit('qunit.fail.load', url);
grunt.log.error('PhantomJS unable to load "' + url + '" URI.');

status.failed += 1;
status.total += 1;
});

phantomjs.on('fail.timeout', function() {
Expand All @@ -167,8 +242,8 @@ module.exports = function(grunt) {
grunt.log.error('PhantomJS timed out, possibly due to:\n' +
'- QUnit is not loaded correctly.\n- A missing QUnit start() call.\n' +
'- Or, a misconfiguration of this task.');

status.failed += 1;
status.total += 1;
});

phantomjs.on('error.onError', function (msg, stackTrace) {
Expand Down Expand Up @@ -239,7 +314,7 @@ module.exports = function(grunt) {
var done = this.async();

// Reset status.
status = {failed: 0, passed: 0, total: 0, duration: 0};
status = createStatus();

// Pass-through console.log statements.
if(options.console) {
Expand Down Expand Up @@ -272,17 +347,15 @@ module.exports = function(grunt) {
},
// All tests have been run.
function() {
var message = generateMessage(status);
var success;

// Log results.
if (status.failed > 0) {
warnUnlessForced(status.failed + '/' + status.total +
' assertions failed (' + status.duration + 'ms)');
} else if (status.total === 0) {
warnUnlessForced('0/0 assertions ran (' + status.duration + 'ms)');
warnUnlessForced(message);
} else {
grunt.verbose.writeln();
grunt.log.ok(status.total + ' assertions passed (' + status.duration + 'ms)');
grunt.log.ok(message);
}

if (options && options.force) {
Expand Down

0 comments on commit 347c126

Please sign in to comment.