Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge pull request #42 from sam-falvo/parallel_suites

Parallel suites
  • Loading branch information...
commit d957a56882ba96f3c740e5cb95dda2f2a3e56071 2 parents b946662 + cd8ac22
@Kami Kami authored
View
18 README.md
@@ -7,7 +7,8 @@ Features
========
* Each test file runs isolated in a separate process
-* Support for running multiple tests in parallel (`--concurrency` option)
+* Support for running multiple tests in parallel in a single suite (`--concurrency` option)
+* Support for running multiple suites in parallel (`--independent-tests` option)
* Support for a test initialization function which is run before running the tests in a test file
* Support for a test file timeout
* setUp / tearDown function support
@@ -42,9 +43,15 @@ Usage
whiskey [options] --tests "<test files>"
+ whiskey [options] --independent-tests "<test files>"
+
+ whiskey [options] --tests "<test files>" --independent-tests "<test files>"
+
+
#### Available options
- * **-t, --tests** - Whitespace separated list of test files to run
+ * **-t, --tests** - Whitespace separated list of test suites to run sequentially
+ * **-T, --independent-tests** - Whitespace separated list of test suites to run concurrently
* **-ti, --test-init-file** - A path to the initialization file which must export
`init` function and it is called in a child process *before running the tests in
each test file
@@ -135,7 +142,7 @@ describe('the bdd module', function(it) {
});
```
-For more examples please check the `example/` folder.
+For more examples please check the `example/` folder, and the `test/run.sh` script.
# Build status
@@ -169,7 +176,10 @@ and put you into the Node debugger prompt.
Whiskey will also by default set a breakpoint at the beginning of your test
file.
-Note: This option can only be used with a single test file.
+Note: This option can only be used with a single test file. Further, you
+cannot use the `--debug` and `--independent-tests` options together. The
+semantics just don't make any sense. To debug a test, make sure you invoke it
+with `--tests` instead.
Troubleshooting
===============
View
6 example/test-long-running-1.js
@@ -0,0 +1,6 @@
+exports.test_long_running_1 = function(test, assert) {
+ setTimeout(function() {
+ assert.equal(1,1);
+ test.finish();
+ }, 5000);
+}
View
6 example/test-long-running-2.js
@@ -0,0 +1,6 @@
+exports.test_long_running_2 = function(test, assert) {
+ setTimeout(function() {
+ assert.equal(2,2);
+ test.finish();
+ }, 5000);
+}
View
4 lib/constants.js
@@ -75,7 +75,9 @@ var DEFAULT_OPTIONS = [
* Other options for the Whiskey option parser.
*/
var WHISKEY_OPTIONS = [
- ['-t', '--tests STRING', 'Whitespace separated list of tests to run'],
+ ['-t', '--tests STRING', 'Whitespace separated list of test suites to run sequentially.'],
+ ['-T', '--independent-tests STRING', 'Whitespace separated list of test suites capable of running independently and concurrently.'],
+ ['-m', '--max-suites NUMBER', 'The number of concurrently executing test suites (defaults to 5)'],
['-ti', '--test-init-file STRING', 'An initialization file which is run before each test file'],
['-c', '--chdir STRING', 'Directory to which each test process chdirs before running the tests'],
['-v', '--verbosity [NUMBER]', 'Test runner verbosity'],
View
138 lib/run.js
@@ -26,6 +26,7 @@ var execSync = require('child_process').execSync;
var sprintf = require('sprintf').sprintf;
var async = require('async');
var logmagic = require('logmagic');
+var underscore = require('underscore');
var constants = require('./constants');
var parser = require('./parser');
@@ -49,6 +50,8 @@ function TestRunner(options) {
defaultSocketPath = sprintf('%s-%s', constants.DEFAULT_SOCKET_PATH,
Math.random() * 10000);
this._tests = options['tests'];
+ this._independent_tests = options['independent-tests'];
+ this._max_suites = options['max-suites'];
this._dependencies = options['dependencies'] || null;
this._onlyEssential = options['only-essential'] || false;
this._socketPath = options['socket-path'] || defaultSocketPath;
@@ -135,72 +138,72 @@ TestRunner.prototype.runTests = function(testInitFile, chdir,
}
}
- function onBound() {
- self._testReporter.handleTestsStart();
-
- async.forEachSeries(self._tests, function(filePath, callback) {
- var result, pattern, cwd, child, timeoutId, testFileData;
-
- if (self._completed || self._forceStopped) {
- callback(new Error('Runner has been stopped'));
- return;
- }
+ function runSuite(filePath, callback) {
+ var result, pattern, cwd, child, timeoutId, testFileData;
- cwd = process.cwd();
- result = common.getTestFilePathAndPattern(filePath);
- filePath = result[0];
- pattern = result[1];
- filePath = (filePath.charAt(0) !== '/') ? path.join(cwd, filePath) : filePath;
- child = self._spawnTestProcess(filePath, testInitFile, chdir,
- customAssertModule, timeout,
- concurrency, pattern);
-
- if (!self._debug) {
- timeoutId = setTimeout(function() {
- handleChildTimeout(child, filePath, callback);
- }, timeout);
- }
+ if (self._completed || self._forceStopped) {
+ callback(new Error('Runner has been stopped'));
+ return;
+ }
- self._testFilesData[filePath] = {
- 'child': child,
- 'callback': callback,
- 'timeout_id': timeoutId,
- 'stdout': '',
- 'stderr': ''
- };
+ cwd = process.cwd();
+ result = common.getTestFilePathAndPattern(filePath);
+ filePath = result[0];
+ pattern = result[1];
+ filePath = (filePath.charAt(0) !== '/') ? path.join(cwd, filePath) : filePath;
+ child = self._spawnTestProcess(filePath, testInitFile, chdir,
+ customAssertModule, timeout,
+ concurrency, pattern);
+
+ if (!self._debug) {
+ timeoutId = setTimeout(function() {
+ handleChildTimeout(child, filePath, callback);
+ }, timeout);
+ }
- testFileData = self._testFilesData[filePath];
+ self._testFilesData[filePath] = {
+ 'child': child,
+ 'callback': callback,
+ 'timeout_id': timeoutId,
+ 'stdout': '',
+ 'stderr': ''
+ };
- child.stdout.on('data', function(chunk) {
- if (self._realTime) {
- process.stdout.write(chunk.toString());
- }
- else {
- testFileData['stdout'] += chunk;
- }
- });
+ testFileData = self._testFilesData[filePath];
- child.stderr.on('data', function(chunk) {
- if (self._realTime) {
- process.stderr.write(chunk.toString());
- }
- else {
- testFileData['stderr'] += chunk;
- }
- });
+ child.stdout.on('data', function(chunk) {
+ if (self._realTime) {
+ process.stdout.write(chunk.toString());
+ }
+ else {
+ testFileData['stdout'] += chunk;
+ }
+ });
- child.on('exit', function() {
- clearTimeout(timeoutId);
- self._handleTestFileEnd(filePath);
- delete self._testFilesData[filePath];
- });
- },
+ child.stderr.on('data', function(chunk) {
+ if (self._realTime) {
+ process.stderr.write(chunk.toString());
+ }
+ else {
+ testFileData['stderr'] += chunk;
+ }
+ });
- function(err) {
- self._handleTestsCompleted();
+ child.on('exit', function() {
+ clearTimeout(timeoutId);
+ self._handleTestFileEnd(filePath);
+ delete self._testFilesData[filePath];
});
}
+ function onBound() {
+ self._testReporter.handleTestsStart();
+ async.series([
+ async.forEachLimit.bind(null, self._independent_tests, self._max_suites, runSuite),
+ async.forEachSeries.bind(null, self._tests, runSuite)
+ ], self._handleTestsCompleted.bind(self));
+ }
+
function startServer(callback) {
self._startServer(self._handleConnection.bind(self),
onBound);
@@ -416,6 +419,7 @@ function run(cwd, argv) {
var customAssertModule, exportedFunctions;
var runner, testReporter, coverageArgs;
var socketPath, concurrency, scopeLeaks, runnerArgs;
+ var intersection;
if ((argv === undefined) && (cwd instanceof Array)) {
argv = cwd;
@@ -439,12 +443,30 @@ function run(cwd, argv) {
var coverageObj = coverage.aggregateCoverage(options['coverage-files'].split(','));
this._coverageReporter.handleTestsComplete(coverageObj);
}
- else if (options.tests && options.tests.length > 0) {
- options.tests = options.tests.split(' ');
+ else if ((options.tests && options.tests.length > 0) ||
+ (options['independent-tests'] && options['independent-tests'].length > 0)) {
+ options.tests = options.tests ? options.tests.split(' ') : [];
+ options['independent-tests'] = options['independent-tests'] ? options['independent-tests'].split(' ') : [];
+
+ var ms = parseInt(options['max-suites'], 10);
+ if (ms) {
+ options['max-suites'] = ms;
+ } else {
+ options['max-suites'] = 5;
+ }
+
+ intersection = underscore.intersection(options.tests, options['independent-tests']);
+ if(intersection.length > 0) {
+ util.puts(sprintf('The following tests cannot appear in both --tests and --independen-tests: %s', intersection));
+ process.exit(1);
+ }
if (options['debug'] && options.tests.length > 1) {
throw new Error('--debug option can currently only be used with a single test file');
}
+ else if (options['debug'] && options['independent-tests'].length > 1) {
+ throw new Error('--debug cannot be used with --independent-tests.');
+ }
else if (options['gen-makefile'] && options['makefile-path']) {
generateMakefile(options.tests, options['makefile-path'], function(err) {
if (err) {
View
3  package.json
@@ -25,7 +25,8 @@
"terminal": "= 0.1.3",
"gex": "= 0.0.1",
"simplesets": "= 1.1.6",
- "logmagic": "= 0.1.4"
+ "logmagic": "= 0.1.4",
+ "underscore": ">= 1.4.2"
},
"engines": {
"node": ">= 0.4.0"
View
22 test/run.sh
@@ -1,5 +1,25 @@
#!/usr/bin/env bash
-CWD=`pwd`
+CWD=`dirname $0`
+CWD=`cd "$APP_DIR";pwd`
+
+START=$(date +%s)
+"${CWD}/bin/whiskey" -m 2 --independent-tests "${CWD}/example/test-long-running-1.js ${CWD}/example/test-long-running-2.js"
+if [ $? -ne 0 ]; then
+ echo "long-running tests should pass"
+ exit 1
+fi
+END=$(date +%s)
+DIFF=$(( $END - $START ))
+if [ $DIFF -ge 7 ]; then
+ echo "Total test execution time should be less than 6 seconds (assuming code hasn't been changed.)"
+ exit 1
+fi
+
+"${CWD}/bin/whiskey" --independent-tests "${CWD}/example/test-long-running-1.js ${CWD}/example/test-long-running-2.js ${CWD}/example/test-failure.js"
+if [ $? -ne 2 ]; then
+ echo "2 tests should fail when running tests independently."
+ exit 1
+fi
"${CWD}/bin/whiskey" --tests "${CWD}/example/test-success.js"
Please sign in to comment.
Something went wrong with that request. Please try again.