feat: add browser_output_timeout and browser_retries option

HISTORY.md

@@ -1,5 +1,11 @@
 # UNRELEASED
 
+* add `browser_retries` option (default to 6)
+* add `browser_output_timeout` option (default to no timeout)
+* properly shutdown all browsers when an uncaught exception is thrown on main `zuul` instance
+
+# 3.1.0 (2015-06-23)
+
 * improve speed of test reporting (ecb1aa)
 * expand file globs from CLI args (517bf0)
 * bump opensauce concurrency to 5 (da6e78c)

bin/zuul

@@ -32,6 +32,8 @@ program
 .option('--browser-name <browser name>', 'specficy the browser name to test an individual browser')
 .option('--browser-version <browser version>', 'specficy the browser version to test an individual browser')
 .option('--browser-platform <browser platform>', 'specficy the browser platform to test an individual browser')
+.option('--browser-retries <retries>', 'number of retries allowed when trying to start a cloud browser, default to 6')
+.option('--browser-output-timeout <timeout>', 'how much time to wait between two test results, default to -1 (no timeout)')
 .option('--concurrency <n>', 'specify the number of concurrent browsers to test')
 .option('--no-coverage', 'disable code coverage analysis with istanbul')
 .option('--open', 'open a browser automatically. only used when --local is specified')
@@ -50,7 +52,9 @@ var config = {
     server: program.server,
     concurrency: program.concurrency,
     coverage: program.coverage,
-    open: program.open
+    open: program.open,
+    browser_retries: program.browserRetries && parseInt(program.browserRetries, 10),
+    browser_output_timeout: program.browserRetries && parseInt(program.browserOutputTimeout, 10)
 };
 
 // Remove unspecified flags
@@ -196,6 +200,8 @@ if (!config.username || !config.key) {
 }
 
 scout_browser(function(err, all_browsers) {
+    var browsers = [];
+
     if (err) {
         console.error('Unable to get available browsers for saucelabs'.red);
         console.error(err.stack);
@@ -237,6 +243,8 @@ scout_browser(function(err, all_browsers) {
     var failed_browsers_count = 0;
 
     zuul.on('browser', function(browser) {
+        browsers.push(browser);
+
         var name = browser.toString();
         var wait_interval;
 
@@ -296,6 +304,12 @@ scout_browser(function(err, all_browsers) {
         console.log('- restarting: %s'.red, name);
     });
 
+    zuul.on('error', function(err) {
+        shutdownAllBrowsers(function() {
+            throw err.message;
+        });
+    });
+
     zuul.run(function(passed) {
         if (passed instanceof Error) {
             throw passed;
@@ -313,6 +327,20 @@ scout_browser(function(err, all_browsers) {
 
         process.exit((passed_tests_count > 0 && failed_browsers_count == 0) ? 0 : 1);
     });
+
+    function shutdownAllBrowsers(done) {
+        var Batch = require('batch')
+        var batch = new Batch;
+
+        browsers.forEach(function(browser) {
+            batch.push(function(done) {
+                browser.shutdown();
+                browser.once('done', done);
+            });
+        });
+
+        batch.end(done);
+    }
 });
 
 function readLocalConfig(config) {

lib/SauceBrowser.js

@@ -2,6 +2,7 @@ var wd = require('wd');
 var EventEmitter = require('events').EventEmitter;
 var FirefoxProfile = require('firefox-profile');
 var debug = require('debug');
+var omit = require('lodash').omit;
 var xtend = require('xtend');
 
 var setup_test_instance = require('./setup');
@@ -20,6 +21,7 @@ function SauceBrowser(conf, opt) {
         failed: 0
     };
     self.debug = debug('zuul:sauce:' + conf.browser + ':' + conf.version);
+    self.debug('browser conf: %j', omit(conf, ['username', 'key']));
 }
 
 SauceBrowser.prototype.__proto__ = EventEmitter.prototype;
@@ -113,6 +115,7 @@ SauceBrowser.prototype.start = function() {
                 });
 
                 reporter.on('done', function(results) {
+                    clearTimeout(self.noOutputTimeout);
                     self.debug('done');
                     var passed = results.passed;
                     var called = false;
@@ -144,6 +147,8 @@ SauceBrowser.prototype.start = function() {
                 }, 120 * 1000);
 
                 browser.get(url, function(err) {
+                    self.debug('browser opened url');
+
                     if (timeout) {
                         return;
                     }
@@ -153,6 +158,21 @@ SauceBrowser.prototype.start = function() {
                         return self.shutdown(err);
                     }
 
+                    // no new output for 30s => error
+                    watchOutput();
+
+                    function watchOutput() {
+                        if (self._opt.browser_output_timeout === -1) {
+                            return;
+                        }
+
+                        clearTimeout(self.noOutputTimeout);
+
+                        self.noOutputTimeout = setTimeout(function() {
+                            self.shutdown(new Error('Did not receive any new output from browser for 30s, shutting down'));
+                        }, self._opt.browser_output_timeout);
+                    }
+
                     (function wait() {
                         if (self.stopped) {
                             return;
@@ -173,6 +193,11 @@ SauceBrowser.prototype.start = function() {
 
                             self.debug('res.length: %s', res.length);
 
+                            // if we received some data, reset the no output watch timeout
+                            if (res.length > 0) {
+                                watchOutput();
+                            }
+
                             var has_done = false;
                             res = res || [];
                             res.filter(Boolean).forEach(function(msg) {
@@ -210,6 +235,8 @@ SauceBrowser.prototype.start = function() {
 SauceBrowser.prototype.shutdown = function(err) {
     var self = this;
 
+    clearTimeout(self.noOutputTimeout);
+
     self.stopped = true;
 
     var finish_shutdown = function () {
@@ -220,6 +247,8 @@ SauceBrowser.prototype.shutdown = function(err) {
         }
 
         if (err) {
+            // prefix browser err message with browser version
+            err.message = self._conf.browser + '@' + self._conf.version + ': ' + err.message;
             self.emit('error', err);
             return;
         }

lib/zuul.js

@@ -1,4 +1,5 @@
 var debug = require('debug')('zuul');
+var omit = require('lodash').omit;
 var open = require('opener');
 var Batch = require('batch');
 var EventEmitter = require('events').EventEmitter;
@@ -17,6 +18,14 @@ function Zuul(config) {
         return new Zuul(config);
     }
 
+    if (config.browser_retries === undefined) {
+        config.browser_retries = 6;
+    }
+
+    if (config.browser_output_timeout === undefined) {
+        config.browser_output_timeout = -1;
+    }
+
     var self = this;
 
     var ui = config.ui;
@@ -28,6 +37,8 @@ function Zuul(config) {
     config.framework_dir = framework_dir;
     self._config = config;
 
+    debug('config: %j', omit(config, ['sauce_username', 'sauce_key', 'username', 'key']));
+
     // list of browsers to test
     self._browsers = [];
 
@@ -119,10 +130,10 @@ Zuul.prototype.run = function(done) {
         self._browsers.forEach(function(browser) {
             self.emit('browser', browser);
 
-            var retry = 6;
+            var retries = config.browser_retries;
 
             browser.on('error', function(err) {
-                if (--retry >= 0) {
+                if (--retries >= 0) {
                     debug('browser error (%s), restarting', err.message)
                     self.emit('restart', browser);
                     return browser.start();