Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Make unexpected errors kill the process

I realized I was going to A LOT of effort to catch and report on
run-time errors in code, and this was majorly complicating the way the
whole library was being written.

But I figure if there is an error, it is something that is a show
stopper anyway, and you need to be notified about it immediately.

Suite runners will be notified of the error, right before the process
exits, so they can say what happened.

If you still really want errors to be caught, use the wrap function to
add a setup to all your tests and then set the uncaughtExceptionHandler
in the test.  You'll have to run the suite in parallel then.
  • Loading branch information...
commit e5090414e469636702d2a7617c418726e0c58555 1 parent 1221b9e
Benjamin Thomas authored
View
119 lib/console-runner.js
@@ -92,10 +92,7 @@ exports.run = function(list, options, callback) {
}
// some state
- var currentSuite = null
- , printedCurrentSuite = null
- , numSuites = null
- ;
+ var currentSuite, printedCurrentSuite, numSuites, lastStart, start = new Date();
opts =
{ parallel: options.parallel
@@ -105,6 +102,7 @@ exports.run = function(list, options, callback) {
numSuites = num;
}
, onSuiteStart: function(name) {
+ lastStart = new Date();
currentSuite = name;
printedCurrentSuite = false;
}
@@ -128,17 +126,9 @@ exports.run = function(list, options, callback) {
var totalAssertions = 0;
- var multiErrors = [];
-
for(var i = 0; i < tests.length; i++) {
var r = tests[i];
- if (typeof r.numAssertions != 'undefined') {
- totalAssertions += r.numAssertions;
- }
- else if (Array.isArray(r.failure)) {
- multiErrors.push(r);
- }
- else if (r.failure instanceof assert.AssertionError) {
+ if (r.failure) {
sys.puts(' Failure: '+red(r.name));
var s = r.failure.stack.split("\n");
sys.puts(' '+ s[0].substr(16));
@@ -157,58 +147,10 @@ exports.run = function(list, options, callback) {
}
}
else {
- sys.puts(' Error: '+yellow(r.name));
-
- if (r.failure.message) {
- sys.puts(' '+r.failure.message);
- }
- var s = r.failure.stack.split("\n");
- if (options.verbosity == 1) {
- if (s.length > 1) {
- sys.puts(s[1].replace(process.cwd(), '.'));
- }
- if (s.length > 2) {
- sys.puts(s[2].replace(process.cwd(), '.'));
- }
- }
- else {
- for(var k = 1; k < s.length; k++) {
- sys.puts(s[k].replace(process.cwd(), '.'));
- }
- }
+ totalAssertions += r.numAssertions;
}
}
- if (multiErrors.length) {
- sys.puts(' Non-specific errors: ' +
- multiErrors.map(function(el) { return yellow(el.name) }).join(', '));
-
- var errs = [];
-
- multiErrors.forEach(function(t) {
- t.failure.forEach(function(e) {
- if (errs.indexOf(e) < 0) {
- errs.push(e);
- var s = e.stack.split("\n");
- sys.puts(' + '+s[0]);
- if (options.verbosity == 1) {
- if (s.length > 1) {
- sys.puts(s[1].replace(process.cwd(), '.'));
- }
- if (s.length > 2) {
- sys.puts(s[2].replace(process.cwd(), '.'));
- }
- }
- else {
- for(var k = 1; k < s.length; k++) {
- sys.puts(s[k]);
- }
- }
- }
- });
- });
- }
-
var total = suiteResults.numFailures+suiteResults.numSuccesses;
if (suiteResults.numFailures > 0) {
@@ -218,10 +160,8 @@ exports.run = function(list, options, callback) {
sys.print(' FAILURES: '+suiteResults.numFailures+'/'+total+' tests failed.');
}
}
- else {
- if (options.verbosity > 0 && numSuites > 1) {
- sys.print(' '+green('OK: ')+total+' test'+(total == 1 ? '' : 's')+'. '+totalAssertions+' assertion'+(totalAssertions == 1 ? '' : 's')+'.');
- }
+ else if (options.verbosity > 0 && numSuites > 1) {
+ sys.print(' '+green('OK: ')+total+' test'+(total == 1 ? '' : 's')+'. '+totalAssertions+' assertion'+(totalAssertions == 1 ? '' : 's')+'.');
}
}
@@ -231,7 +171,7 @@ exports.run = function(list, options, callback) {
}
}
else if(suiteResults.numFailures > 0 || numSuites > 1) {
- sys.puts(' '+(suiteResults.duration/1000)+' seconds.');
+ sys.puts(' '+((new Date() - lastStart)/1000)+' seconds.');
sys.puts('');
}
}
@@ -246,18 +186,13 @@ exports.run = function(list, options, callback) {
}
if (result.failure) {
- if (result.failure instanceof assert.AssertionError) {
- sys.puts(red('' + result.name));
- }
- else {
- sys.puts(yellow('' + result.name))
- }
+ sys.puts(red('' + result.name));
}
else if (options.printSuccesses) {
sys.puts('' + result.name);
}
}
- , onDone: function(allResults, duration) {
+ , onDone: function(allResults) {
var successes = 0;
var total = 0;
var tests = 0;
@@ -287,10 +222,42 @@ exports.run = function(list, options, callback) {
sys.print(bold(' '+total+'/'+total+' suites passed successfully.'));
}
}
- sys.puts(bold(' ' + tests+(tests == 1 ? ' test' : ' total tests')+'. '+(duration/1000)+' seconds.'));
+ sys.puts(bold(' ' + tests+(tests == 1 ? ' test' : ' total tests')+'. '+((new Date() - start)/1000)+' seconds.'));
if (callback) {
- callback(allResults, duration);
+ callback(allResults);
+ }
+ }
+ , onError: function(err, tests) {
+ if (!printedCurrentSuite && currentSuite) {
+ sys.puts(bold(currentSuite));
+ }
+
+
+ if (tests.length > 1) {
+ sys.puts(' One of the following tests threw an error: ');
+ sys.puts(' ' + tests.map(function(name) { return yellow(name); }).join(', '));
+ }
+ else {
+ sys.puts(' Error: ' + yellow(tests[0]));
+ }
+
+ var s = err.stack.split("\n");
+ if (err.message) {
+ sys.puts(' '+err.message);
+ }
+ if (options.verbosity == 1) {
+ if (s.length > 1) {
+ sys.puts(s[1].replace(process.cwd(), '.'));
+ }
+ if (s.length > 2) {
+ sys.puts(s[2].replace(process.cwd(), '.'));
+ }
+ }
+ else {
+ for(var k = 1; k < s.length; k++) {
+ sys.puts(s[k]);
+ }
}
}
, onPrematureExit: function(tests) {
View
215 lib/testing.js
@@ -1,3 +1,6 @@
+var testAsyncE;
+
+var sys = require('sys');
var assert = require('assert')
, path = require('path')
, fs = require('fs')
@@ -21,6 +24,7 @@ var assert = require('assert')
* + onSuiteDone
* + onTestStart
* + onTestDone
+ * + onError
* + onPrematureExit
*/
exports.runSuite = function(obj, options) {
@@ -29,16 +33,13 @@ exports.runSuite = function(obj, options) {
// keep track of internal state
var suite =
- { startTime: new Date()
- , todo: exports.getTestsFromObject(obj, options.testName)
- , queue: []
- , errored: []
+ { todo: exports.getTestsFromObject(obj, options.testName)
+ , started: []
, results: []
}
if (suite.todo.length < 1) { return suiteFinished(); }
- // TODO also pass the number of tests?
if (options.onSuiteStart) { options.onSuiteStart(options.name); }
process.on('uncaughtException', errorHandler);
@@ -50,43 +51,35 @@ exports.runSuite = function(obj, options) {
/******** functions ********/
function startNextTest() {
- var curTest = suite.todo.shift();
+ var test = suite.todo.shift();
- if (!curTest) {
- return;
- }
+ if (!test) { return; }
- suite.queue.push(curTest);
+ suite.started.push(test);
- curTest.startTime = new Date(); // for calculating how long it takes
- curTest.func = Array.isArray(curTest.func) ? curTest.func : [curTest.func];
- curTest.progress = 0;
- curTest.numAssertions = 0; // assertions made
- curTest.obj = // object that is passed to the tests
- { get uncaughtExceptionHandler() { return curTest.UEHandler; }
+ test.progress = 0;
+ test.numAssertions = 0; // assertions made
+ test.obj = // object that is passed to the tests
+ { get uncaughtExceptionHandler() { return test.UEHandler; }
, set uncaughtExceptionHandler(h) {
if (options.parallel) {
- throw new Error("Cannot set an 'uncaughtExceptionHandler' when running tests in parallel");
+ test.obj.equal('serial', 'parallel',
+ "Cannot set an 'uncaughtExceptionHandler' when running tests in parallel");
}
- curTest.UEHandler = h;
+ test.UEHandler = h;
}
- , finish: function() { testProgressed(curTest); }
+ , finish: function() { testFinished(test); }
};
- addAssertionFunctions(curTest);
+ addAssertionFunctions(test);
- if (options.onTestStart) {
- // notify listeners
- options.onTestStart(curTest.name);
- }
+ if (options.onTestStart) { options.onTestStart(test.name); }
try {
- // actually run the test
- // TODO pass finish function?
- curTest.func[curTest.progress](curTest.obj);
+ test.func(test.obj);
}
catch(err) {
- errorHandler(err, curTest);
+ errorHandler(err, test);
}
// if we are supposed to run the tests in parallel, start the next test
@@ -97,57 +90,13 @@ exports.runSuite = function(obj, options) {
}
}
- function testProgressed(test, problem) {
- if (!test.failure && problem) {
+ function testFinished(test, problem) {
+ if (problem) {
test.failure = problem;
+ delete test.numAssertions;
}
-
- if (test.errors) {
- test.errors.forEach(function(e) {
- e.candidates.splice(e.candidates.indexOf(test), 1);
- });
- delete test.errors;
- }
-
- test.progress++;
-
- if (test.func.length == 1) {
- testFinished(test);
- }
- else if (test.func.length == 3) {
- if (test.progress == 1) {
- if (test.failure) {
- // if there was a failure skip the test function and just run teardown
- test.progress++;
- }
-
- try {
- test.func[test.progress](test.obj);
- }
- catch(err) {
- errorHandler(err, test);
- }
- }
- else if (test.progress == 2) {
- try {
- test.func[test.progress](test.obj);
- }
- catch(err) {
- errorHandler(err, test);
- }
- }
- else {
- testFinished(test);
- }
- }
- else {
- console.log('Oops, tests need to be 3-tuples [setup, function teardown] or 1-tuples [function]');
- }
- }
-
- function testFinished(test) {
- // if they specified the number of assertions, make sure they match up
- if (test.obj.numAssertions && test.obj.numAssertions != test.numAssertions) {
+ else if (test.obj.numAssertions && test.obj.numAssertions != test.numAssertions) {
+ // if they specified the number of assertions, make sure they match up
test.failure = new assert.AssertionError(
{ message: 'Wrong number of assertions: ' + test.numAssertions +
' != ' + test.obj.numAssertions
@@ -156,148 +105,84 @@ exports.runSuite = function(obj, options) {
});
}
-
- // calculate the time it took
- test.duration = test.duration || (new Date() - test.startTime);
-
// remove it from the list of tests that have been started
- suite.queue.splice(suite.queue.indexOf(test), 1);
+ suite.started.splice(suite.started.indexOf(test), 1);
// clean up properties that are no longer needed
delete test.obj;
delete test.progress;
delete test.func;
- delete test.startTime;
- if (test.failure) { delete test.numAssertions; }
suite.results.push(test);
- if (options.onTestDone) {
- // notify listener
- options.onTestDone(test);
- }
+
+ if (options.onTestDone) { options.onTestDone(test); }
process.nextTick(function() {
// if we have no more tests to start and none still running, we're done
- if (suite.todo.length == 0 && suite.queue.length == 0) {
+ if (suite.todo.length == 0 && suite.started.length == 0) {
suiteFinished();
}
- // check to see if we can isolate any errors
- checkErrors();
-
startNextTest();
});
}
- // listens for uncaught errors and keeps track of which tests they could be from
- function errorHandler(err, test) {
+ function errorHandler(err) {
// assertions throw an error, but we can't just catch those errors, because
// then the rest of the test will run. So, we don't catch it and it ends up
// here. When that happens just finish the test.
if (err instanceof assert.AssertionError && err.TEST) {
var t = err.TEST;
delete err.TEST;
- return testProgressed(t, err);
+ return testFinished(t, err);
}
// We want to allow tests to supply a function for handling uncaught errors,
// and since all uncaught errors come here, this is where we have to handle
// them.
// (you can only handle uncaught errors when not in parallel mode)
- if (!options.parallel && suite.queue[0].UEHandler) {
+ if (!options.parallel && suite.started[0].UEHandler) {
// an error could possibly be thrown in the UncaughtExceptionHandler, in
// this case we do not want to call the handler again, so we move it
- suite.queue[0].UEHandlerUsed = suite.queue[0].UEHandler;
- delete suite.queue[0].UEHandler;
+ suite.started[0].UEHandlerUsed = suite.started[0].UEHandler;
+ delete suite.started[0].UEHandler;
try {
// run the UncaughtExceptionHandler
- suite.queue[0].UEHandlerUsed(err);
+ suite.started[0].UEHandlerUsed(err);
return;
}
catch(e) {
// we had an error, just run our error handler function on this error
// again. We don't have to worry about it triggering the uncaught
// exception handler again because we moved it just a second ago
- return errorHandler(e, test);
+ return errorHandler(e);
}
}
- if (test) {
- // shortcut if we know the test that caused this error
- return testProgressed(test, err);
- }
-
- var summary =
- { error: err
- , candidates: suite.queue.slice()
- , endTime: new Date()
- };
- summary.candidates.forEach(function(t) {
- // add this error to the list of errors a test is a candidate for
- if (t.errors) {
- t.errors.push(summary);
- }
- else {
- t.errors = [summary];
- }
- });
-
- suite.errored.push(summary);
-
- // check to see if we can isolate any errors
- checkErrors();
- }
-
- function checkErrors() {
- // any time a test finishes, we could learn more about errors that had
- // multiple candidates, so loop through and see if anything has changed
- for(var i = 0; i < suite.errored.length; i++) {
- var err = suite.errored[i];
-
- if (err.error.message == 'E teardown') {
- testAsyncE = err;
- }
- // if there is only one candidate then we can finish that test
- if (err.candidates.length == 1) {
- suite.errored.splice(i,1);
- i = -1;
-
- var test = err.candidates[0];
-
- test.duration = err.endTime - test.startTime;
- delete err.endTime;
+ process.removeListener('uncaughtException', errorHandler);
+ process.removeListener('exit', exitHandler);
- testProgressed(test, err.error);
- }
+ if (options.onError) {
+ options.onError(err, suite.started.map(function(t) { return t.name; }));
+ process.exit(1);
}
-
- // if the number of errors we've found equals the number of tests still
- // running then we know that the errors must match up with the tests, so
- // finish each of the tests.
-
- if (suite.errored.length && suite.errored.length == suite.queue.length) {
- var errs = suite.errored.map(function(d) { return d.error; });
- suite.errored = [];
- suite.queue.slice().forEach(function(t) {
- testProgressed(t, errs);
- });
+ else {
+ throw err;
}
}
function exitHandler() {
- if (suite.queue.length > 0) {
+ if (suite.started.length > 0) {
if (options.onPrematureExit) {
- options.onPrematureExit(suite.queue.map(function(t) { return t.name; }));
+ options.onPrematureExit(suite.started.map(function(t) { return t.name; }));
}
}
}
// clean up method which notifies all listeners of what happened
function suiteFinished() {
- if (suite.finished) {
- return;
- }
+ if (suite.finished) { return; }
suite.finished = true;
@@ -307,7 +192,6 @@ exports.runSuite = function(obj, options) {
if (options.onSuiteDone) {
var result =
{ name: options.name
- , duration: new Date() - suite.startTime
, tests: suite.results
, numFailures: 0
, numSuccesses: 0
@@ -349,7 +233,6 @@ exports.runFiles = function(list, options) {
options = options || {};
var suites
- , startTime
, allResults = []
, index = 0
;
@@ -363,7 +246,6 @@ exports.runFiles = function(list, options) {
options.onStart(suites.length);
}
- startTime = new Date();
runNextSuite();
}
@@ -373,7 +255,7 @@ exports.runFiles = function(list, options) {
if (!item) {
if (options.onDone) {
- options.onDone(allResults, new Date()-startTime);
+ options.onDone(allResults);
}
return;
}
@@ -399,6 +281,7 @@ exports.runFiles = function(list, options) {
}
, onTestStart: options.onTestStart
, onTestDone: options.onTestDone
+ , onError: options.onError
, onPrematureExit: options.onPrematureExit
}
View
16 test/test-multiple_errors.js → test/error-async.js
@@ -1,27 +1,17 @@
module.exports = {
'test async error 1': function(test) {
- process.nextTick(function() {
+ setTimeout(function() {
throw new Error('error 1');
- });
- },
-
- 'test sync error': function(test) {
- throw new Error('sync error');
+ }, 500);
},
'test async error 2': function(test) {
setTimeout(function() {
throw new Error('error 2');
}, 500);
- },
-
- 'test async error 3': function(test) {
- setTimeout(function() {
- throw new Error('error 3');
- }, 500);
}
-};
+}
if (module == require.main) {
require('../lib/async_testing').run(__filename, process.ARGV);
View
8 test/test-errors.js → test/error-sync.js
@@ -2,14 +2,8 @@
module.exports = {
'test sync error': function(test) {
throw new Error();
- },
-
- 'test async error': function(test) {
- setTimeout(function() {
- throw new Error();
- }, 500);
}
-};
+}
if (module == require.main) {
require('../lib/async_testing').run(__filename, process.ARGV);
View
16 test/error-uncaught_exception_handler.js
@@ -0,0 +1,16 @@
+
+module.exports = {
+ 'test sync error error again': function(test) {
+ var e = new Error('first error');
+
+ test.uncaughtExceptionHandler = function(err) {
+ throw new Error('second error');
+ }
+
+ throw e;
+ }
+}
+
+if (module == require.main) {
+ require('../lib/async_testing').run(__filename, process.ARGV);
+}
View
40 test/readme-suite.js
@@ -1,40 +0,0 @@
-
-// create suite:
-exports['asynchronousTest'] = function(test) {
- setTimeout(function() {
- // make an assertion (these are just regular assertions)
- test.ok(true);
- // finish the test
- test.finish();
- },500);
-};
-
-exports['synchronousTest'] = function(test) {
- test.ok(true);
- test.finish();
-};
-
-exports['test assertions expected'] = function(test) {
- test.numAssertions = 1;
-
- test.ok(true);
- test.finish();
-}
-
-exports['test catch async error'] = function(test) {
- var e = new Error();
-
- test.uncaughtExceptionHandler = function(err) {
- test.equal(e, err);
- test.finish();
- }
-
- setTimeout(function() {
- throw e;
- }, 500);
-};
-
-// if this module is the script being run, then run the tests:
-if (module == require.main) {
- require('../lib/async_testing').run(__filename, process.ARGV);
-}
View
62 test/test-uncaught_exception_handlers.js
@@ -38,7 +38,7 @@ module.exports = {
var e = new Error();
test.uncaughtExceptionHandler = function(err) {
- test.ok(false, 'this fails asynchronously');
+ test.ok(false, 'this fails synchronously');
test.finish();
}
@@ -51,8 +51,10 @@ module.exports = {
var e = new Error();
test.uncaughtExceptionHandler = function(err) {
- test.ok(false, 'this errors synchronously');
- test.finish();
+ setTimeout(function() {
+ test.ok(false, 'this fails asynchronously');
+ test.finish();
+ }, 500);
}
throw e;
@@ -62,63 +64,17 @@ module.exports = {
var e = new Error();
test.uncaughtExceptionHandler = function(err) {
- test.ok(false, 'this errors asynchronously');
- test.finish();
- }
-
- setTimeout(function() {
- throw e;
- }, 500);
- },
-
- 'test sync error error again': function(test) {
- var e = new Error('first error');
-
- test.uncaughtExceptionHandler = function(err) {
- throw new Error('second error');
- }
-
- throw e;
- },
-
- 'test async error error again': function(test) {
- var e = new Error('first error');
-
- test.uncaughtExceptionHandler = function(err) {
- throw new Error('second error');
- }
-
- setTimeout(function() {
- throw e;
+ setTimeout(function() {
+ test.ok(false, 'this fails asynchronously');
+ test.finish();
}, 500);
- },
-
- 'test sync error error again async': function(test) {
- var e = new Error('first error');
-
- test.uncaughtExceptionHandler = function(err) {
- process.nextTick(function() {
- throw new Error('second error');
- });
- }
-
- throw e;
- },
-
- 'test async error error again async': function(test) {
- var e = new Error('first error');
-
- test.uncaughtExceptionHandler = function(err) {
- process.nextTick(function() {
- throw new Error('second error');
- });
}
setTimeout(function() {
throw e;
}, 500);
}
-};
+}
if (module == require.main) {
require('../lib/async_testing').run(__filename, process.ARGV);
Please sign in to comment.
Something went wrong with that request. Please try again.