Permalink
Browse files

add spion benchmark suite

  • Loading branch information...
1 parent e0ada6e commit 79ac6802f2850724b0221d968f9bf4733bc9f24f @petkaantonov committed Sep 11, 2013
Showing with 2,449 additions and 32 deletions.
  1. +14 −0 .editorconfig
  2. +1 −0 .gitignore
  3. +15 −0 .npmignore
  4. +30 −23 Gruntfile.js
  5. +116 −0 benchmark/async-compare/README.md
  6. +42 −0 benchmark/async-compare/complexity.js
  7. +112 −0 benchmark/async-compare/debuggability.js
  8. +49 −0 benchmark/async-compare/examples-extra/fibby.js
  9. +71 −0 benchmark/async-compare/examples/async.js
  10. +53 −0 benchmark/async-compare/examples/catcher.js
  11. +105 −0 benchmark/async-compare/examples/flattened-class.js
  12. +89 −0 benchmark/async-compare/examples/flattened-noclosure.js
  13. +74 −0 benchmark/async-compare/examples/flattened.js
  14. +60 −0 benchmark/async-compare/examples/original.js
  15. +59 −0 benchmark/async-compare/examples/promiseish.js
  16. +59 −0 benchmark/async-compare/examples/promiseishBluebird.js
  17. +59 −0 benchmark/async-compare/examples/promiseishQ.js
  18. +70 −0 benchmark/async-compare/examples/promises-compose.js
  19. +62 −0 benchmark/async-compare/examples/promises.js
  20. +205 −0 benchmark/async-compare/examples/rx.js
  21. +34 −0 benchmark/async-compare/lib/catcher.js
  22. +30 −0 benchmark/async-compare/lib/dummy.js
  23. +88 −0 benchmark/async-compare/lib/fakemaker.js
  24. +38 −0 benchmark/async-compare/lib/fakes-ctx.js
  25. +8 −0 benchmark/async-compare/lib/fakes.js
  26. +18 −0 benchmark/async-compare/lib/fakesC.js
  27. +29 −0 benchmark/async-compare/lib/fakesO.js
  28. +28 −0 benchmark/async-compare/lib/fakesP.js
  29. +4 −0 benchmark/async-compare/lib/fakesSJS-dst.js
  30. +56 −0 benchmark/async-compare/lib/fakesSJS-src.sjs
  31. +51 −0 benchmark/async-compare/lib/promiseSupport.js
  32. +479 −0 benchmark/async-compare/lib/timers-ctx.js
  33. +39 −0 benchmark/async-compare/package.json
  34. +165 −0 benchmark/async-compare/performance.js
  35. +31 −0 benchmark/spion.js
  36. +2 −3 js/bluebird.js
  37. +2 −3 js/bluebird_sync.js
  38. +2 −3 src/settled_promise_array.js
View
@@ -0,0 +1,14 @@
+# This file is for unifying the coding style for different editors and IDEs
+# editorconfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[**.js]
+indent_style = space
+indent_size = 4
View
@@ -1,6 +1,7 @@
node_modules/*
todo.txt
npm-debug.log
+benchmark/async-compare/node_modules/*
benchmark/promises-benchmark/node_modules/*
benchmark/promises-benchmark/fixtures/*
benchmark/perf-promises/node_modules/*
View
@@ -0,0 +1,15 @@
+node_modules/*
+todo.txt
+npm-debug.log
+benchmark/*
+benchmark/async-compare/node_modules/*
+benchmark/promises-benchmark/node_modules/*
+benchmark/promises-benchmark/fixtures/*
+benchmark/perf-promises/node_modules/*
+async
+sync
+mixed
+bench.json
+js/bluebird.min.js
+js/bluebird_sync.js
+js/bluebird_sync.min.js
View
@@ -158,29 +158,29 @@ module.exports = function( grunt ) {
grunt.loadNpmTasks('grunt-contrib-concat');
grunt.loadNpmTasks('grunt-bump');
- function runIndependent( file, cb ) {
- var fs = require("fs");
- var sys = require('sys');
- var spawn = require('child_process').spawn;
- var node = spawn('node', ["./"+file]);
- node.stdout.on('data', function( data ) {
- process.stdout.write(data);
- });
+ function runIndependentTest( file, cb ) {
+ var fs = require("fs");
+ var sys = require('sys');
+ var spawn = require('child_process').spawn;
+ var node = spawn('node', ["./"+file]);
+ node.stdout.on('data', function( data ) {
+ process.stdout.write(data);
+ });
- node.stderr.on('data', function( data ) {
- process.stderr.write(data);
- });
+ node.stderr.on('data', function( data ) {
+ process.stderr.write(data);
+ });
- function exit( code ) {
- if( code !== 0 ) {
- cb(new Error("process didn't exit normally"));
- }
- else {
- cb(null);
+ function exit( code ) {
+ if( code !== 0 ) {
+ throw new Error("process didn't exit normally");
+ }
+ else {
+ cb(null);
+ }
}
- }
- node.on('exit', exit );
+ node.on('exit', exit );
}
function fixStrict( code ) {
@@ -309,7 +309,7 @@ module.exports = function( grunt ) {
var suite = mochas[i].suite;
if( suite.suites.length === 0 &&
suite.tests.length === 0 ) {
- runIndependent(mochas[i].files[0], function(err) {
+ runIndependentTest(mochas[i].files[0], function(err) {
if( err ) throw err;
setTimeout(function(){
runner( mochas, i + 1 );
@@ -347,10 +347,17 @@ module.exports = function( grunt ) {
done();
}
else {
- grunt.log.writeln("Running benchmark " + files[i] );
- require(files[i])(function(){
+ var file = files[i];
+ grunt.log.writeln("Running benchmark " + file );
+ if( file.indexOf( "matcha" ) !== -1 ) {
+ grunt.log.writeln("Run matcha benchmarks with match");
runner(files, i + 1 );
- });
+ }
+ else {
+ require(file)(function(){
+ runner(files, i + 1 );
+ });
+ }
}
})(files, 0);
}
@@ -0,0 +1,116 @@
+# async-compare
+
+This project aims compare various node.js async patterns by their
+
+- complexity (number of necessary tokens)
+- performance when executing in parallel (time and memory)
+- debuggability
+
+The resulting analysis is available at
+[this blog post](http://spion.github.io/posts/analysis-generators-and-other-async-patterns-node.html)
+
+## example problem
+
+The problem is directly extracted from a DoxBee project. Its a typical if
+somewhat complex CRUD method executed when a user uploads a new document
+to the database. It involves multiple queries to the database, a couple of
+selects, some inserts and one update. Lots of mixed sync/async action.
+
+## files
+
+Example solutions for all patterns are located in the `examples` directory
+
+Non-js sorce files begin with `src-` (they're not checked for performance)
+
+Compiled files are prefixed with `dst-` (they're not checked for complexity)
+
+All other files are checked for both performance and complexity
+
+## complexity
+
+Complexity is measured by the number of tokens in the source code found by
+Esprima's lexer (comments excluded)
+
+Run `node complexity.js` to get complexity reports for all files.
+
+
+## fakes.js
+
+Wrappers can be added in `lib/fakes.js`
+
+For examples, look at the promise and thunk wrappers for query methods.
+
+Things that are specific to the upload function are not allowed here.
+
+
+## performance
+
+All external methods are mocked with setTimeout, to simulate waiting for I/O
+operations.
+
+Performance is measured by performance.js
+
+ node performance.js --n <parallel> --t <miliseconds> ./examples/*.js --harmony
+
+where `n` is the number of parallel executions of the method, while `t` is the
+time each simulated I/O operation should take, and `--harmony` enables
+all features hidden behind the v8 flag.
+
+There is an optional parameter `--file <file>` which will only test a single
+file and report any encountered errors in detail:
+
+ node --harmony performance.js --n 10000 --t 10 --file ./examples/genny.js
+
+Also, this variant doesn't spawn a new process so which means additional
+(v8) options can be passed to node.
+
+If you omit `--n`, tests will be made with 100, 500, 1000 and 2000 parallel
+requests and a giant JSON report (suitable for charts) will be generated.
+
+ node performance.js --t 1 ./examples/*.js --harmony
+
+If you omit `--n` *and* replace `--t` with `--dt`, I/O time `t` will grow with
+`n` by the formula `t = n * dt`
+
+ node performance.js --dt 0.1 ./examples/*.js --harmony
+
+Execution time and peak memory usage are reported.
+
+
+## debuggability
+
+
+`debuggability.js` measures the distance between the function that creates the
+error and the actual error in the stack trace. Reports "-" at places where
+the stack trace is completely missing the original file.
+
+To check all examples for async errors:
+
+```
+node debuggability.js --harmony --error
+```
+
+and for exceptions:
+
+```
+node debuggability.js --harmony --throw
+```
+
+and finally for exceptions inside async calls (most things can't handle this):
+
+
+```
+node debuggability.js --harmony --athrow
+```
+
+
+## misc
+
+These are factors potentially important for collaboration which could
+be added as points to arrive at a final score:
+
+- does it require native modules (-2)
+- does it require code transformation (-2)
+- will it eventually become available without code transformation (+1)
+
+
@@ -0,0 +1,42 @@
+
+var fs = require('fs');
+var table = require('text-table');
+
+var stats = module.exports = function stats() {
+ return fs.readdirSync(__dirname + '/examples').filter(function(f){
+ return !/^dst-/.test(f);
+ }).map(function(f) {
+ var file = fs.readFileSync('./examples/'+f).toString();
+ file = file.replace(/function\s*\*/g, 'function')
+ .replace(/yield/g, 'void');
+ try {
+ var tree = require('esprima').parse(file, {
+ tolerant: true,
+ tokens: true
+ });
+ } catch (e) {
+ console.log("In file", f, ":");
+ console.log(e);
+ }
+ return {name: f, tokens: tree.tokens.length}
+ });
+}
+
+var s = stats();
+
+var mintokens = s.reduce(function(acc, f) {
+ return Math.min(acc, f.tokens);
+}, Number.POSITIVE_INFINITY);
+
+s = s.sort(function(s1, s2) {
+ return s1.tokens - s2.tokens;
+});
+
+s.forEach(function(f) {
+ f.complexity = f.tokens / mintokens;
+});
+
+console.log(table([['name', 'tokens', 'complexity']].concat(
+ s.map(function(f) { return [f.name, f.tokens, f.complexity.toFixed(2)] })
+), {align: ['l','r','r']}));
+
@@ -0,0 +1,112 @@
+Error.stackTraceLimit = Infinity;
+
+var args = require('optimist').argv;
+
+global.longStackSupport = require('q').longStackSupport = true;
+
+var perf = module.exports = function(args, done) {
+ global.asyncTime = args.t || 1;
+ global.testThrow = args.throw;
+ global.testThrowAsync = args.athrow;
+ global.testError = args.error;
+
+ var fn = require(args.file);
+ fn('a','b','c', done);
+}
+
+
+if (args.file) {
+ perf(args, function(err) {
+ if (err) {
+ //throw err;
+ console.log(err);
+ // for browser-compatibility, stratifiedjs reports errors
+ // on __oni_stack (or toString()), rather than 'stack'.
+ console.error(err.__oni_stack ? err.__oni_stack.map(function(x) { return x.join(':') }).join('\n') : new Error(err.stack).stack);
+ }
+ });
+} else {
+ var cp = require('child_process')
+ var async = require('async');
+ var fs = require('fs');
+ var dir = __dirname + '/examples';
+
+ var table = require('text-table');
+
+
+ var allfiles = fs.readdirSync(dir);
+
+ var files = allfiles.filter(function(f) {
+ return !/^src-/.test(f);
+ });
+
+ var sources = allfiles.filter(function(f) {
+ return !/^dst-/.test(f);
+ });
+
+ var sourceOf = function(f) {
+ var parts = f.split('-');
+ var name = parts[1];
+ return sources.filter(function(s) {
+ return s.indexOf(name) >= 0;
+ })[0] || f;
+ }
+
+ async.mapSeries(files, function(f, done) {
+ console.error("testing", f);
+
+ var argsFork = [__filename,
+ '--file', dir + '/' + f];
+ if (args.error) argsFork.push('--error')
+ if (args.throw) argsFork.push('--throw');
+ if (args.athrow) argsFork.push('--athrow');
+
+ if (args.harmony) argsFork.unshift('--harmony');
+
+ var p = cp.spawn(process.execPath, argsFork);
+
+
+ var lineNumber = fs.readFileSync(dir + '/' + sourceOf(f), 'utf8')
+ .split('\n')
+ .map(function(l, k) {
+ return {
+ contained: l.indexOf('FileVersion.insert') >= 0,
+ line: k + 2
+ };
+ }).filter(function(l) { return l.contained; })[0].line;
+
+ var r = { file: f, data: [], line: lineNumber };
+
+ p.stderr.pipe(process.stderr);
+ p.stderr.on('data', function(d) { r.data.push(d.toString()); });
+ p.stderr.on('end', function(code) {
+ r.data = r.data.join('').split('\n').filter(function(line) {
+ // match lines reporting either compiled or source files:
+ return line.match('examples/' + f) || line.match('examples/' + sourceOf(f))
+ }).map(function(l) {
+ return {content: l,
+ line: l.split(':')[1],
+ distance: Math.abs(l.split(':')[1] - r.line)};
+ }).sort(function(l1, l2) {
+ return l1.distance - l2.distance;
+ })[0];
+ done(null, r);
+ });
+ }, function(err, res) {
+ console.log("");
+ console.log("error reporting");
+ console.log("");
+ res = res.sort(function(r1, r2) {
+ return parseFloat(r1.data ? r1.data.distance : Infinity)
+ - parseFloat(r2.data ? r2.data.distance : Infinity)
+ });
+ res = res.map(function(r) {
+ return [r.file, r.line,
+ r.data ? r.data.line : '-',
+ r.data ? r.data.distance : '-'];
+ //r.data ? 'yes ' + r.data.content :'no'];
+ })
+ res = [['file', 'actual-line', 'rep-line', 'distance']].concat(res)
+ console.log(table(res, {align: ['l','r','r','r']}));
+ });
+}
Oops, something went wrong.

0 comments on commit 79ac680

Please sign in to comment.