Close #496 PR: Add a benchmark suite.

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 .nyc_output
 coverage
+bench/.results

bench/compare.js

@@ -0,0 +1,105 @@
+'use strict';
+
+var path = require('path');
+var fs = require('fs');
+var Table = require('cli-table2');
+var chalk = require('chalk');
+
+var files = fs.readdirSync(path.join(__dirname, '.results'))
+	.map(function (file) {
+		var result = JSON.parse(fs.readFileSync(path.join(__dirname, '.results', file), 'utf8'));
+		result['.file'] = path.basename(file, '.json');
+		return result;
+	})
+	// find the most recent benchmark runs
+	.sort(function (fileA, fileB) {
+		return fileB['.time'] - fileA['.time'];
+	});
+
+// Only the 3 most recent runs
+files = files.slice(0, 3);
+
+function prepStats(times) {
+	times = times
+		.map(function (time) {
+			return time.time;
+		})
+		.sort(function (timeA, timeB) {
+			return timeA - timeB;
+		});
+
+	// remove fastest and slowest
+	times = times.slice(1, times.length - 1);
+
+	var sum = times.reduce(function (a, b) {
+		return a + b;
+	}, 0);
+
+	return {
+		mean: Math.round((sum / times.length) * 1000) / 1000,
+		median: times[Math.floor(times.length / 2)],
+		min: times[0],
+		max: times[times.length - 1]
+	};
+}
+
+var results = {};
+var fileNames = files.map(function (file) {
+	return file['.file'];
+});
+var stats = ['mean', 'median', 'min', 'max'];
+
+files.forEach(function (file) {
+	Object.keys(file)
+		.filter(function (key) {
+			return !/^\./.test(key);
+		})
+		.forEach(function (key) {
+			results[key] = results[key] || {};
+			results[key][file['.file']] = prepStats(file[key]);
+		});
+});
+
+var table = new Table();
+table.push(
+	[''].concat(stats.map(function (stat) {
+		return {
+			content: stat,
+			colSpan: fileNames.length,
+			hAlign: 'center'
+		};
+	})),
+	stats.reduce(function (arr) {
+		return arr.concat(fileNames);
+	}, ['args'])
+);
+
+Object.keys(results)
+	.forEach(function (key) {
+		table.push(stats.reduce(function (arr, stat) {
+			var min = Infinity;
+			var max = -Infinity;
+
+			var statGroup = fileNames.map(function (fileName) {
+				var result = results[key][fileName];
+				result = result && result[stat];
+				if (result) {
+					min = Math.min(min, result);
+					max = Math.max(max, result);
+					return result;
+				}
+				return '';
+			});
+			return arr.concat(statGroup.map(function (stat) {
+				if (stat === min) {
+					return chalk.green(stat);
+				}
+				if (stat === max) {
+					return chalk.red(stat);
+				}
+				return stat;
+			}));
+		}, [key]));
+	});
+
+console.log(table.toString());

bench/concurrent/alternating-sync-async.js

@@ -0,0 +1,9 @@
+import test from '../../';
+
+for (var i = 0; i < 10000; i++) {
+	if (i % 2) {
+		test('test' + i, () => new Promise(resolve => setImmediate(resolve)));
+	} else {
+		test('test' + i, () => {});
+	}
+}

bench/concurrent/async-immediate.js

@@ -0,0 +1,5 @@
+import test from '../../';
+
+for (var i = 0; i < 10000; i++) {
+	test('test' + i, () => new Promise(resolve => setImmediate(resolve)));
+}

bench/concurrent/async-timeout.js

@@ -0,0 +1,5 @@
+import test from '../../';
+
+for (var i = 0; i < 10000; i++) {
+	test('test' + i, () => new Promise(resolve => setTimeout(resolve, 0)));
+}

bench/concurrent/sync.js

@@ -0,0 +1,5 @@
+import test from '../../';
+
+for (var i = 0; i < 10000; i++) {
+	test('test' + i, () => {});
+}

bench/other/failures.js

@@ -0,0 +1,7 @@
+import test from '../../';
+
+for (var i = 0; i < 1000; i++) {
+	test.serial('test' + i, t => {
+		t.is(Math.random(), Math.random());
+	});
+}

bench/run.js

@@ -0,0 +1,117 @@
+'use strict';
+
+var childProcess = require('child_process');
+var path = require('path');
+var fs = require('fs');
+var arrify = require('arrify');
+var Promise = require('bluebird');
+var mkdirp = require('mkdirp');
+var branch = require('git-branch').sync(path.join(__dirname, '..'));
+var cliPath = require.resolve('../cli');
+
+function runTests(_args) {
+	return new Promise(function (resolve) {
+		var args = [cliPath]
+			.concat(arrify(_args));
+		var start = Date.now();
+		childProcess.execFile(process.execPath, args, {
+			cwd: __dirname,
+			maxBuffer: 100000 * 200
+		}, function (err, stdout, stderr) {
+			var end = Date.now();
+			resolve({
+				args: arrify(_args),
+				time: end - start,
+				err: err,
+				stdout: stdout,
+				stderr: stderr
+			});
+		});
+	});
+}
+
+var list;
+
+if (process.argv.length === 2) {
+	list = [
+		{args: 'other/failures.js', shouldFail: true},
+		'serial/alternating-sync-async.js',
+		'serial/async-immediate.js',
+		'serial/async-timeout.js',
+		'serial/sync.js',
+		'concurrent/alternating-sync-async.js',
+		'concurrent/async-immediate.js',
+		'concurrent/async-timeout.js',
+		'concurrent/sync.js',
+		['concurrent/*.js', 'serial/*.js']
+	].map(function (definition) {
+		if (Array.isArray(definition) || typeof definition === 'string') {
+			definition = {
+				shouldFail: false,
+				args: definition
+			};
+		}
+		return definition;
+	});
+} else {
+	list = [];
+	var currentArgs = [];
+	var shouldFail = false;
+	process.argv.slice(2).forEach(function (arg) {
+		if (arg === '--') {
+			list.push({
+				args: currentArgs,
+				shouldFail: shouldFail
+			});
+			currentArgs = [];
+			shouldFail = false;
+			return;
+		}
+		if (arg === '--should-fail') {
+			shouldFail = true;
+			return;
+		}
+		currentArgs.push(arg);
+	});
+	if (currentArgs.length) {
+		list.push({
+			args: currentArgs,
+			shouldFail: shouldFail
+		});
+	}
+}
+
+list.forEach(function (definition) {
+	definition.args = ['--verbose'].concat(definition.args);
+});
+
+var combined = [];
+for (var i = 0; i < 11; i++) {
+	combined = combined.concat(list);
+}
+
+var results = {};
+
+Promise.each(combined, function (definition) {
+	var args = definition.args;
+	return runTests(args).then(function (result) {
+		var key = result.args.join(' ');
+		var passedOrFaild = result.err ? 'failed' : 'passed';
+		var seconds = result.time / 1000;
+		console.log('%s %s in %d seconds', key, passedOrFaild, seconds);
+		if (result.err && !definition.shouldFail) {
+			console.log(result.stdout);
+			console.log(result.stderr);
+			throw (result.err);
+		}
+		results[key] = results[key] || [];
+		results[key].push({passed: !results.err, shouldFail: definition.shouldFail, time: seconds});
+	});
+}).then(function () {
+	mkdirp.sync(path.join(__dirname, '.results'));
+	results['.time'] = Date.now();
+	fs.writeFileSync(
+		path.join(__dirname, '.results', branch + '.json'),
+		JSON.stringify(results, null, 4)
+	);
+});

bench/serial/alternating-sync-async.js

@@ -0,0 +1,9 @@
+import test from '../../';
+
+for (var i = 0; i < 10000; i++) {
+	if (i % 2) {
+		test.serial('test' + i, () => new Promise(resolve => setImmediate(resolve)));
+	} else {
+		test.serial('test' + i, () => {});
+	}
+}

bench/serial/async-immediate.js

@@ -0,0 +1,5 @@
+import test from '../../';
+
+for (var i = 0; i < 10000; i++) {
+	test.serial('test' + i, () => new Promise(resolve => setImmediate(resolve)));
+}

bench/serial/async-timeout.js

@@ -0,0 +1,5 @@
+import test from '../../';
+
+for (var i = 0; i < 3000; i++) {
+	test.serial('test' + i, () => new Promise(resolve => setTimeout(resolve, 0)));
+}

bench/serial/sync.js

@@ -0,0 +1,5 @@
+import test from '../../';
+
+for (var i = 0; i < 10000; i++) {
+	test.serial('test' + i, () => {});
+}

package.json

@@ -124,10 +124,13 @@
     "update-notifier": "^0.6.0"
   },
   "devDependencies": {
+    "cli-table2": "^0.1.9",
     "coveralls": "^2.11.4",
     "delay": "^1.3.0",
     "get-stream": "^1.1.0",
+    "git-branch": "^0.3.0",
     "inquirer": "^0.11.1",
+    "mkdirp": "^0.5.1",
     "nyc": "^5.1.0",
     "pify": "^2.3.0",
     "rimraf": "^2.5.0",
@@ -140,7 +143,8 @@
   },
   "xo": {
     "ignore": [
-      "cli.js"
+      "cli.js",
+      "bench/*/*.js"
     ]
   }
 }