Further work on parallel suite execution

Handle load errors of files, and make web-runner use new system.

lib/child.js

@@ -1,12 +1,8 @@
-var testing = require('./testing')
-  , stackReplaceRegExp = new RegExp(process.cwd(),'g')
-  , frame = '~m~'
-  ;
+var testing = require('./testing');
 
 var opts =
-  { name: JSON.parse(process.ARGV[3])
-  , parallel: JSON.parse(process.ARGV[4])
-  , testName: JSON.parse(process.ARGV[5])
+  { parallel: JSON.parse(process.ARGV[3])
+  , testName: JSON.parse(process.ARGV[4])
   , onTestStart: function testStart(name) {
       postMessage('onTestStart', name);
     }
@@ -28,29 +24,19 @@ var opts =
     }
   };
 
-try {
-  var s = require(process.ARGV[2]);
-}
-catch(err) {
-  postMessage('onRequireError', {message: err.message, stack: err.stack});
-  return;
-}
+var s = require(process.ARGV[2]);
 
 testing.runSuite(s, opts);
 
-// this encode function inspired by socket.io's
 function postMessage() {
-  var r = '';
-  for (var i = 0; i < arguments.length; i++) {
-    var json = JSON.stringify(arguments[i]);
-    r += frame + json.length + frame + json;
-  }
-
-  console.log(r);
+  console.log(testing.messageEncode.apply(null, arguments));
 }
 
 function makeErrorJsonable(err) {
-  return  { message: err.message ? err.message : null
-          , stack: err.stack.replace(stackReplaceRegExp, '.')
-          };
+  var r = new RegExp(process.cwd(),'g')
+
+  return {
+      message: err.message ? err.message : null
+    , stack: err.stack.replace(r, '.')
+    }
 }

lib/console-runner.js

@@ -91,6 +91,12 @@ exports.run = function(list, options, callback) {
     delete options.all;
   }
 
+  if (typeof options['suites-parallel'] == 'undefined') {
+    options['suites-parallel'] = false;
+  }
+  options.suitesParallel = options['suites-parallel'];
+  delete options['suites-parallel'];
+
   var suites
     , startTime
     , index = 0
@@ -127,10 +133,12 @@ exports.run = function(list, options, callback) {
     var opts =
       { parallel: options.parallel
       , testName: options.testName
-      , name: suite.name
       , onTestDone: function(result) {
           testFinished(suite, result);
         }
+      , onSuiteLoadError: function() {
+          suiteFinished(suite, 'suiteLoadError', Array.prototype.slice.call(arguments));
+        }
       , onSuiteCompleted: function(results) {
           suiteFinished(suite, 'suiteCompleted', Array.prototype.slice.call(arguments));
         }
@@ -145,7 +153,9 @@ exports.run = function(list, options, callback) {
     suite.startTime = new Date();
     testing.runFile(suite.path, opts);
 
-    startNextSuite();
+    if (options.suitesParallel) {
+      startNextSuite();
+    }
   }
 
   function suiteFinished(suite, status, results) {
@@ -156,7 +166,6 @@ exports.run = function(list, options, callback) {
     delete suite.startTime;
 
     if (suite.index == finishedIndex) {
-
       for (var i = finishedIndex; i < suites.length; i++) {
         if (suites[i].finished) {
           while(suites[i].queuedTestResults.length) {
@@ -205,6 +214,27 @@ exports.run = function(list, options, callback) {
           console.log('  ✔ ' + result.name);
         }
       }
+    , suiteLoadError: function(suite) {
+        var err = suite.results[0];
+
+        if (!suite.printedName) {
+          console.log(bold(suite.name));
+        }
+
+        if (err.stack) {
+          var lines = err.stack.split('\n');
+        }
+        else {
+          var lines = err.split('\n');
+        }
+
+        // todo check output level here
+        for (var i = 0; i < lines.length; i++) {
+          console.log('  ' + lines[i]);
+        }
+
+        console.log('');
+      }
     , suiteCompleted: function(suite) {
         var suiteResults = suite.results[0];
         var tests = suiteResults.tests;
@@ -336,29 +366,23 @@ exports.run = function(list, options, callback) {
 
         for(var i = 0; i < suites.length; i++) {
           total++;
-          switch(suites[i].status) {
-            case 'suiteCompleted':
-              if (suites[i].results[0].numFailures == 0) {
-                successes++;
-              }
-              break;
+          if (suites[i].status == 'suiteCompleted' && suites[i].results[0].numFailures == 0) {
+            successes++;
           }
         }
 
         var last = '';
         if (successes != total) {
           last += bold(red('PROBLEMS:'));
-          if (total > 1) {
-            last += bold(' '+(total-successes)+'/'+total+' suites had problems.');
-          }
+          last += bold(' '+(total-successes)+'/'+total+' suites had problems.');
         }
         else {
           last += bold(green('SUCCESS:'));
           if (total > 1) {
             last += bold(' '+total+'/'+total+' suites passed successfully.');
           }
         }
-        console.log(last + ' ' + ((new Date() - startTime)/1000)+' seconds.');
+        console.log(last + ' ' + bold(((new Date() - startTime)/1000)+' seconds.'));
 
         if (callback) {
           callback(total - successes);

lib/running.js

@@ -73,7 +73,11 @@ exports.run = function(list, args, cb) {
       }
     , { longFlag: 'parallel'
       , shortFlag: 'p'
-      , description: 'run the suites in parallel mode'
+      , description: 'run the tests in parallel'
+      }
+    , { longFlag: 'suites-parallel'
+      , shortFlag: 'P'
+      , description: 'run the suites in parallel'
       }
     , { longFlag: 'help'
       , shortFlag: 'h'

lib/testing.js

@@ -1,7 +1,7 @@
 var assert = require('assert')
   , path = require('path')
   , fs = require('fs')
-  , spawn = require('child_process').spawn;
+  , spawn = require('child_process').spawn
   ;
 
 /* Runs an object of tests.  Each property in the object should be a
@@ -52,7 +52,7 @@ exports.runSuite = function(obj, options) {
 
     suite.started.push(test);
 
-    // make sure all tests are an array (flow):
+    // make sure all tests are an array:
     test.func = Array.isArray(test.func) ? test.func : [test.func];
     // TODO make sure test length is odd?
 
@@ -241,8 +241,7 @@ exports.runSuite = function(obj, options) {
 
     if (options.onSuiteCompleted) {
       var result =
-        { name: options.name
-        , tests: suite.results
+        { tests: suite.results
         , numFailures: 0
         , numSuccesses: 0
         };
@@ -257,7 +256,6 @@ exports.runSuite = function(obj, options) {
   }
 }
 
-var frame = "~m~";
 exports.runFile = function(modulepath, options) {
   options = options || {};
   if (options.testName && !Array.isArray(options.testName)) {
@@ -266,7 +264,6 @@ exports.runFile = function(modulepath, options) {
 
   var child = spawn(process.execPath, [ __dirname+'/child.js'
                                       , modulepath
-                                      , JSON.stringify(options.name || null)
                                       , JSON.stringify(options.parallel || false)
                                       , JSON.stringify(options.testName || null)
                                       ]);
@@ -280,46 +277,30 @@ exports.runFile = function(modulepath, options) {
     lines[0] = buffer + lines[0];
     buffer = lines.pop();
 
+    lines = exports.messageDecode(lines);
+
     for (var i = 0; i < lines.length; i++) {
-      if (lines[i].substr(0,3) != frame) {
+      if (typeof lines[i] === 'string') {
         console.log(lines[i]);
-        lines.splice(i,0);
-      }
-      else {
-        lines[i] = decode(lines[i]);
       }
-    }
-
-    for (var i = 0; i < lines.length; i++) {
-      if (options[lines[i][0]]) {
+      else if (options[lines[i][0]]) {
         options[lines[i][0]].apply(null, lines[i].slice(1));
       }
     }
   });
 
+  var errorBuffer = '';
   child.stderr.on('data', function(data) {
-    console.log('err: ' + data);
+    errorBuffer += data.toString();
   });
-}
 
-// this decode function inspired by socket.io's
-function decode(str) {
-  var msg = [];
-
-  for (var i = 3, number = '', l = str.length; i < l; i++){
-    var n = Number(str.substr(i, 1));
-    if (str.substr(i, 1) == n){
-      number += n;
-    } else {  
-      number = Number(number);
-      var m = str.substr(i+frame.length, number);
-      msg.push(JSON.parse(m));
-      i += frame.length*2 + number - 1;
-      number = '';
-    } 
-  }
+  child.stderr.on('close', function() {
+    if (errorBuffer && options.onSuiteLoadError) {
+      options.onSuiteLoadError(errorBuffer.trim());
+    }
+  });
 
-  return msg;
+  return child;
 }
 
 // expandFiles takes a file name, directory name or an array composed of any
@@ -488,3 +469,38 @@ exports.getTestsFromObject = function(o, filter, namespace) {
 
   return tests;
 }
+
+var messageFrame = "~m~";
+// these encode/decode functions inspired by socket.io's
+exports.messageDecode = function(lines) {
+  return lines.map(function(str) {
+    if (str.substr(0,3) !== messageFrame) {
+      return str;
+    }
+
+    var msg = [];
+    for (var i = 3, number = '', l = str.length; i < l; i++){
+      var n = Number(str.substr(i, 1));
+      if (str.substr(i, 1) == n){
+        number += n;
+      } else {  
+        number = Number(number);
+        var m = str.substr(i+messageFrame.length, number);
+        msg.push(JSON.parse(m));
+        i += messageFrame.length*2 + number - 1;
+        number = '';
+      } 
+    }
+    return msg;
+  });
+}
+exports.messageEncode = function() {
+  var r = '';
+
+  for (var i = 0; i < arguments.length; i++) {
+    var json = JSON.stringify(arguments[i]);
+    r += messageFrame + json.length + messageFrame + json;
+  }
+
+  return r;
+}