From 1221b9e8f6d5c59889d00d355645c40bb83b2e5e Mon Sep 17 00:00:00 2001 From: Benjamin Thomas Date: Sat, 9 Oct 2010 01:00:51 -0600 Subject: [PATCH] Beginning support for built-in setup/teardown functions Basically, you can't do them properly without building them in. The API will probably change. --- lib/console-runner.js | 4 +- lib/testing.js | 184 +++++++++++++--------- test/flow.js | 328 +++++++++++++++++++++++++++++++++++++++ test/test-all_passing.js | 8 +- test/test-readme.js | 41 +++++ 5 files changed, 486 insertions(+), 79 deletions(-) create mode 100644 test/flow.js create mode 100644 test/test-readme.js diff --git a/lib/console-runner.js b/lib/console-runner.js index 3a2e71b..6fa4704 100644 --- a/lib/console-runner.js +++ b/lib/console-runner.js @@ -132,10 +132,10 @@ exports.run = function(list, options, callback) { for(var i = 0; i < tests.length; i++) { var r = tests[i]; - if (r.numAssertions) { + if (typeof r.numAssertions != 'undefined') { totalAssertions += r.numAssertions; } - else if (r.failure.constructor == Array) { + else if (Array.isArray(r.failure)) { multiErrors.push(r); } else if (r.failure instanceof assert.AssertionError) { diff --git a/lib/testing.js b/lib/testing.js index d8e7a22..420f3af 100644 --- a/lib/testing.js +++ b/lib/testing.js @@ -29,49 +29,40 @@ exports.runSuite = function(obj, options) { // keep track of internal state var suite = - { started: [] - , results: [] + { startTime: new Date() + , todo: exports.getTestsFromObject(obj, options.testName) + , queue: [] , errored: [] + , results: [] } - suite.todo = exports.getTestsFromObject(obj, options.testName); + if (suite.todo.length < 1) { return suiteFinished(); } - if (suite.todo.length < 1) { // No tests to run - return suiteFinished(); - } - - if (options.onSuiteStart) { - // TODO also pass the number of tests? - options.onSuiteStart(options.name); - } + // TODO also pass the number of tests? + if (options.onSuiteStart) { options.onSuiteStart(options.name); } process.on('uncaughtException', errorHandler); process.on('exit', exitHandler); - suite.startTime = new Date(); - // start the test chain startNextTest(); /******** functions ********/ function startNextTest() { - // grab the next test var curTest = suite.todo.shift(); if (!curTest) { return; } - // move our test to the list of started tests - suite.started.push(curTest); - - curTest.startTime = new Date(); - // keep track of the number of assertions made - curTest.numAssertions = 0; + suite.queue.push(curTest); - // this is the object that the tests get for manipulating how the tests work - var testObj = + curTest.startTime = new Date(); // for calculating how long it takes + curTest.func = Array.isArray(curTest.func) ? curTest.func : [curTest.func]; + curTest.progress = 0; + curTest.numAssertions = 0; // assertions made + curTest.obj = // object that is passed to the tests { get uncaughtExceptionHandler() { return curTest.UEHandler; } , set uncaughtExceptionHandler(h) { if (options.parallel) { @@ -79,12 +70,9 @@ exports.runSuite = function(obj, options) { } curTest.UEHandler = h; } - , finish: function() { testFinished(curTest); } + , finish: function() { testProgressed(curTest); } }; - // store the testObj in the test - curTest.obj = testObj; - addAssertionFunctions(curTest); if (options.onTestStart) { @@ -95,10 +83,10 @@ exports.runSuite = function(obj, options) { try { // actually run the test // TODO pass finish function? - curTest.func(curTest.obj); + curTest.func[curTest.progress](curTest.obj); } catch(err) { - errorHandler(err); + errorHandler(err, curTest); } // if we are supposed to run the tests in parallel, start the next test @@ -109,25 +97,10 @@ exports.runSuite = function(obj, options) { } } - // Called when a test finishes, either successfully or from an assertion error - function testFinished(test, problem) { - // calculate the time it took - test.duration = test.duration || (new Date() - test.startTime); - - if (problem) { + function testProgressed(test, problem) { + if (!test.failure && problem) { test.failure = problem; } - else { - // if they specified the number of assertions, make sure they match up - if (test.obj.numAssertions && test.obj.numAssertions != test.numAssertions) { - test.failure = new assert.AssertionError( - { message: 'Wrong number of assertions: ' + test.numAssertions + - ' != ' + test.obj.numAssertions - , actual: test.numAssertions - , expected: test.obj.numAssertions - }); - } - } if (test.errors) { test.errors.forEach(function(e) { @@ -136,15 +109,66 @@ exports.runSuite = function(obj, options) { delete test.errors; } - // TODO check progress of test + test.progress++; + + if (test.func.length == 1) { + testFinished(test); + } + else if (test.func.length == 3) { + if (test.progress == 1) { + if (test.failure) { + // if there was a failure skip the test function and just run teardown + test.progress++; + } + + try { + test.func[test.progress](test.obj); + } + catch(err) { + errorHandler(err, test); + } + } + else if (test.progress == 2) { + try { + test.func[test.progress](test.obj); + } + catch(err) { + errorHandler(err, test); + } + } + else { + testFinished(test); + } + } + else { + console.log('Oops, tests need to be 3-tuples [setup, function teardown] or 1-tuples [function]'); + } + } + + function testFinished(test) { + // if they specified the number of assertions, make sure they match up + if (test.obj.numAssertions && test.obj.numAssertions != test.numAssertions) { + test.failure = new assert.AssertionError( + { message: 'Wrong number of assertions: ' + test.numAssertions + + ' != ' + test.obj.numAssertions + , actual: test.numAssertions + , expected: test.obj.numAssertions + }); + } + + + // calculate the time it took + test.duration = test.duration || (new Date() - test.startTime); // remove it from the list of tests that have been started - suite.started.splice(suite.started.indexOf(test), 1); + suite.queue.splice(suite.queue.indexOf(test), 1); // clean up properties that are no longer needed delete test.obj; + delete test.progress; delete test.func; delete test.startTime; + if (test.failure) { delete test.numAssertions; } suite.results.push(test); if (options.onTestDone) { @@ -152,12 +176,12 @@ exports.runSuite = function(obj, options) { options.onTestDone(test); } - // if we have no more tests to start and none still running, we're done - if (suite.todo.length == 0 && suite.started.length == 0) { - suiteFinished(); - } - process.nextTick(function() { + // if we have no more tests to start and none still running, we're done + if (suite.todo.length == 0 && suite.queue.length == 0) { + suiteFinished(); + } + // check to see if we can isolate any errors checkErrors(); @@ -166,43 +190,47 @@ exports.runSuite = function(obj, options) { } // listens for uncaught errors and keeps track of which tests they could be from - function errorHandler(err) { + function errorHandler(err, test) { // assertions throw an error, but we can't just catch those errors, because // then the rest of the test will run. So, we don't catch it and it ends up // here. When that happens just finish the test. if (err instanceof assert.AssertionError && err.TEST) { var t = err.TEST; delete err.TEST; - testFinished(t, err); - return; + return testProgressed(t, err); } // We want to allow tests to supply a function for handling uncaught errors, // and since all uncaught errors come here, this is where we have to handle // them. // (you can only handle uncaught errors when not in parallel mode) - if (!options.parallel && suite.started[0].UEHandler) { + if (!options.parallel && suite.queue[0].UEHandler) { // an error could possibly be thrown in the UncaughtExceptionHandler, in // this case we do not want to call the handler again, so we move it - suite.started[0].UEHandlerUsed = suite.started[0].UEHandler; - delete suite.started[0].UEHandler; + suite.queue[0].UEHandlerUsed = suite.queue[0].UEHandler; + delete suite.queue[0].UEHandler; try { // run the UncaughtExceptionHandler - suite.started[0].UEHandlerUsed(err); + suite.queue[0].UEHandlerUsed(err); return; } catch(e) { // we had an error, just run our error handler function on this error // again. We don't have to worry about it triggering the uncaught // exception handler again because we moved it just a second ago - return errorHandler(e); + return errorHandler(e, test); } } + if (test) { + // shortcut if we know the test that caused this error + return testProgressed(test, err); + } + var summary = { error: err - , candidates: suite.started.slice() + , candidates: suite.queue.slice() , endTime: new Date() }; summary.candidates.forEach(function(t) { @@ -227,42 +255,52 @@ exports.runSuite = function(obj, options) { for(var i = 0; i < suite.errored.length; i++) { var err = suite.errored[i]; + if (err.error.message == 'E teardown') { + testAsyncE = err; + } // if there is only one candidate then we can finish that test - if (suite.errored[i].candidates.length == 1) { + if (err.candidates.length == 1) { suite.errored.splice(i,1); - i = 0; + i = -1; var test = err.candidates[0]; test.duration = err.endTime - test.startTime; delete err.endTime; - testFinished(test, err.error); + testProgressed(test, err.error); } } // if the number of errors we've found equals the number of tests still // running then we know that the errors must match up with the tests, so // finish each of the tests. - if (suite.errored.length && suite.errored.length == suite.started.length) { - suite.started.slice().forEach(function(t) { - testFinished(t, t.errors.map(function(d) { return d.error; })); - }); + if (suite.errored.length && suite.errored.length == suite.queue.length) { + var errs = suite.errored.map(function(d) { return d.error; }); suite.errored = []; + suite.queue.slice().forEach(function(t) { + testProgressed(t, errs); + }); } } function exitHandler() { - if (suite.started.length > 0) { + if (suite.queue.length > 0) { if (options.onPrematureExit) { - options.onPrematureExit(suite.started.map(function(t) { return t.name; })); + options.onPrematureExit(suite.queue.map(function(t) { return t.name; })); } } } // clean up method which notifies all listeners of what happened function suiteFinished() { + if (suite.finished) { + return; + } + + suite.finished = true; + process.removeListener('uncaughtException', errorHandler); process.removeListener('exit', exitHandler); @@ -384,10 +422,10 @@ exports.expandFiles = function(list, suiteNames, cb) { suiteNames = null; } - if (list.constructor != Array) { + if (!Array.isArray(list)) { list = [list]; } - if (suiteNames && suiteNames.constructor != Array) { + if (suiteNames && !Array.isArray(suiteNames)) { suiteNames = [suiteNames]; } if (suiteNames && suiteNames.length === 0) { @@ -520,7 +558,7 @@ exports.getTestsFromObject = function(o, filter, namespace) { var tests = []; for(var key in o) { var displayName = (namespace ? namespace+' \u2192 ' : '') + key; - if (typeof o[key] == 'function') { + if (typeof o[key] == 'function' || Array.isArray(o[key])) { // if the testName option is set, then only add the test to the todo // list if the name matches if (!filter || filter.indexOf(key) >= 0) { diff --git a/test/flow.js b/test/flow.js new file mode 100644 index 0000000..091d8d7 --- /dev/null +++ b/test/flow.js @@ -0,0 +1,328 @@ +var order = ''; + +module.exports = { + 'test A': [ function setup(test) { + test.numAssertions = 3 + order += 'A1'; + test.equal('A1', order); + test.finish(); + } + , function test(test) { + order += 'A2'; + test.equal('A1A2', order); + test.finish(); + } + , function teardown(test) { + order += 'A3'; + test.equal('A1A2A3', order); + test.finish(); + } + ] + +// test that even if the test function throws an error, teardown is run +, 'test B': [ function setup(test) { + order += '\nB1'; + test.finish(); + } + , function test(test) { + order += 'B2e'; + throw new Error('B func'); + } + , function teardown(test) { + order += 'B3'; + test.finish(); + } + ] + +// test that even if setup throws an error, teardown is run +, 'test C': [ function setup(test) { + order += '\nC1e'; + throw new Error('C setup'); + } + , function test(test) { + order += 'SHOULD NOT GET HERE'; + } + , function teardown(test) { + order += 'C3'; + test.finish(); + } + ] + +// test that things don't break if teardown throws an error +, 'test D': [ function setup(test) { + order += '\nD1'; + test.finish(); + } + , function test(test) { + order += 'D2'; + test.finish(); + } + , function teardown(test) { + order += 'D3e'; + throw new Error('D teardown'); + } + ] + +// test that later errors don't trump earlier errors +, 'test E (error in setup)': [ function setup(test) { + order += '\nE1e'; + throw new Error('E setup'); + } + , function test(test) { + order += 'SHOULD NOT GET HERE'; + } + , function teardown(test) { + order += 'E3e'; + throw new Error('E teardown'); + } + ] + +// test that later errors don't trump earlier errors again +, 'test F (error in func)': [ function setup(test) { + order += '\nF1'; + test.finish(); + } + , function test(test) { + order += 'F2e'; + throw new Error('F func'); + } + , function teardown(test) { + order += 'F3e'; + throw new Error('F teardown'); + } + ] + +// test failure in setup +, 'test G (failure in setup)': [ function setup(test) { + order += '\nG1f'; + test.ok(false); + } + , function test(test) { + order += 'SHOULD NOT GET HERE'; + } + , function teardown(test) { + order += 'G3'; + test.finish(); + } + ] + +// test failure in func +, 'test H (failure in func)': [ function setup(test) { + order += '\nH1'; + test.finish(); + } + , function test(test) { + order += 'H2f'; + test.ok(false); + } + , function teardown(test) { + order += 'H3'; + test.finish(); + } + ] + +// test failure in teardown +, 'test I (failure in teardown)': [ function setup(test) { + order += '\nI1'; + test.finish(); + } + , function test(test) { + order += 'I2'; + test.finish(); + } + , function teardown(test) { + order += 'I3f'; + test.ok(false); + } + ] + +, 'test J (async A)': [ function setup(test) { + setTimeout(function() { + order += '\nA1'; + test.finish(); + }, 500); + } + , function test(test) { + setTimeout(function() { + order += 'A2'; + test.finish(); + }, 500); + } + , function teardown(test) { + setTimeout(function() { + order += 'A3'; + test.finish(); + }, 500); + } + ] + +// test that even if the test function throws an error, teardown is run +, 'test async B': [ function setup(test) { + setTimeout(function() { + order += '\nB1'; + test.finish(); + }, 500); + } + , function test(test) { + setTimeout(function() { + order += 'B2e'; + throw new Error('aB func'); + }, 500); + } + , function teardown(test) { + setTimeout(function() { + order += 'B3'; + test.finish(); + }, 500); + } + ] + +// test that even if setup throws an error, teardown is run +, 'test async C': [ function setup(test) { + setTimeout(function() { + order += '\nC1e'; + throw new Error('aC setup'); + //test.finish(); + }, 500); + } + , function test(test) { + setTimeout(function() { + order += 'SHOULD NOT GET HERE'; + }, 500); + } + , function teardown(test) { + setTimeout(function() { + order += 'C3'; + test.finish(); + }, 500); + } + ] + +// test that things don't break if teardown throws an error +, 'test async D': [ function setup(test) { + order += '\nD1'; + setTimeout(function() { + test.finish(); + }, 500); + } + , function test(test) { + order += 'D2'; + setTimeout(function() { + test.finish(); + }, 500); + } + , function teardown(test) { + order += 'D3e'; + setTimeout(function() { + throw new Error('aD teardown'); + }, 500); + } + ] + +// test that later errors don't trump earlier errors +, 'test async E (error in setup)': [ function setup(test) { + order += '\nE1e'; + setTimeout(function() { + throw new Error('aE setup'); + }, 500); + } + , function test(test) { + order += 'SHOULD NOT GET HERE'; + } + , function teardown(test) { + order += 'E3e'; + setTimeout(function() { + throw new Error('aE teardown'); + }, 500); + } + ] + +// test that later errors don't trump earlier errors again +, 'test async F (error in func)': [ function setup(test) { + order += '\nF1'; + setTimeout(function() { + test.finish(); + }, 500); + } + , function test(test) { + order += 'F2e'; + setTimeout(function() { + throw new Error('aF func'); + }, 500); + } + , function teardown(test) { + order += 'F3e'; + setTimeout(function() { + throw new Error('aF teardown'); + }, 500); + } + ] + +// test failure in setup +, 'test async G (failure in setup)': [ function setup(test) { + order += '\nG1f'; + setTimeout(function() { + test.ok(false); + }, 500); + } + , function test(test) { + order += 'SHOULD NOT GET HERE'; + } + , function teardown(test) { + order += 'G3'; + setTimeout(function() { + test.finish(); + }, 500); + } + ] + +// test failure in func +, 'test async H (failure in func)': [ function setup(test) { + order += '\nH1'; + setTimeout(function() { + test.finish(); + }, 500); + } + , function test(test) { + order += 'H2f'; + setTimeout(function() { + test.ok(false); + }, 500); + } + , function teardown(test) { + order += 'H3'; + setTimeout(function() { + test.finish(); + }, 500); + } + ] + +// test failure in teardown +, 'test async I (failure in teardown)': [ function setup(test) { + order += '\nI1'; + setTimeout(function() { + test.finish(); + }, 500); + } + , function test(test) { + order += 'I2'; + setTimeout(function() { + test.finish(); + }, 500); + } + , function teardown(test) { + order += 'I3f'; + setTimeout(function() { + test.ok(false); + }, 500); + } + ] +}; + +process.on('exit', function() { + console.log(order); + }); + +if (module == require.main) { + require('../lib/async_testing').run(__filename, process.ARGV); +} diff --git a/test/test-all_passing.js b/test/test-all_passing.js index d57bc96..42cca10 100644 --- a/test/test-all_passing.js +++ b/test/test-all_passing.js @@ -1,21 +1,21 @@ module.exports = { - 'test A': function(test) { + 'test A': function(test) { test.ok(true); test.finish(); }, - 'test B': function(test) { + 'test B': function(test) { test.ok(true); test.finish(); }, - 'test C': function(test) { + 'test C': function(test) { test.ok(true); test.finish(); }, - 'test D': function(test) { + 'test D': function(test) { test.ok(true); test.finish(); } diff --git a/test/test-readme.js b/test/test-readme.js new file mode 100644 index 0000000..2a1424c --- /dev/null +++ b/test/test-readme.js @@ -0,0 +1,41 @@ + +module.exports = { + 'asynchronousTest': function(test) { + setTimeout(function() { + // make an assertion (these are just regular assertions) + test.ok(true); + // finish the test + test.finish(); + },500); + }, + + 'synchronousTest': function(test) { + test.ok(true); + test.finish(); + }, + + 'test assertions expected': function(test) { + test.numAssertions = 1; + + test.ok(true); + test.finish(); + }, + + 'test catch async error': function(test) { + var e = new Error(); + + test.uncaughtExceptionHandler = function(err) { + test.equal(e, err); + test.finish(); + } + + setTimeout(function() { + throw e; + }, 500); + } +}; + +// if this module is the script being run, then run the tests: +if (module == require.main) { + require('../lib/async_testing').run(__filename, process.ARGV); +}