Allow to retry failed test from test context for #1773

- make retries run proper hooks
- allow retries override at different levels
- expose currentRetry to reporters

bin/_mocha

@@ -92,6 +92,7 @@ program
   .option('--prof', 'log statistical profiling information')
   .option('--recursive', 'include sub directories')
   .option('--reporters', 'display available reporters')
+  .option('--retries <times>', 'set numbers of time to retry a failed test case')
   .option('--throw-deprecation', 'throw an exception anytime a deprecated function is used')
   .option('--trace', 'trace function calls')
   .option('--trace-deprecation', 'show stack traces on deprecations')
@@ -284,6 +285,10 @@ if (program.delay) mocha.delay();
 
 mocha.globals(globals);
 
+// --retries
+
+if (program.retries) mocha.suite.retries(program.retries);
+
 // custom compiler support
 
 var extensions = ['js'];

lib/context.js

@@ -76,6 +76,21 @@ Context.prototype.skip = function() {
   return this;
 };
 
+/**
+ * Allow a number of retries on failed tests
+ *
+ * @api private
+ * @param {number} n
+ * @return {Context} self
+ */
+Context.prototype.retries = function(n) {
+  if (!arguments.length) {
+    return this.runnable().retries();
+  }
+  this.runnable().retries(n);
+  return this;
+};
+
 /**
  * Inspect the context void of `._runnable`.
  *

lib/interfaces/bdd.js

@@ -106,5 +106,12 @@ module.exports = function(suite) {
     context.xit = context.xspecify = context.it.skip = function(title) {
       context.it(title);
     };
+
+    /**
+     * Number of attempts to retry.
+     */
+    context.it.retries = function(n) {
+      context.retries(n);
+    };
   });
 };

lib/interfaces/common.js

@@ -70,6 +70,15 @@ module.exports = function(suites, context) {
        */
       skip: function(title) {
         context.test(title);
+      },
+
+      /**
+       * Number of retry attempts
+       *
+       * @param {string} n
+       */
+      retries: function(n) {
+        context.retries(n);
       }
     }
   };

lib/interfaces/qunit.js

@@ -89,5 +89,6 @@ module.exports = function(suite) {
     };
 
     context.test.skip = common.test.skip;
+    context.test.retries = common.test.retries;
   });
 };

lib/interfaces/tdd.js

@@ -101,5 +101,6 @@ module.exports = function(suite) {
     };
 
     context.test.skip = common.test.skip;
+    context.test.retries = common.test.retries;
   });
 };

lib/mocha.js

@@ -62,6 +62,7 @@ function image(name) {
  *   - `reporter` reporter instance, defaults to `mocha.reporters.spec`
  *   - `globals` array of accepted globals
  *   - `timeout` timeout in milliseconds
+ *   - `retries` number of times to retry failed tests
  *   - `bail` bail on the first test failure
  *   - `slow` milliseconds to wait before considering a test slow
  *   - `ignoreLeaks` ignore global leaks
@@ -88,6 +89,9 @@ function Mocha(options) {
   if (typeof options.timeout !== 'undefined' && options.timeout !== null) {
     this.timeout(options.timeout);
   }
+  if (typeof options.retries !== 'undefined' && options.retries !== null) {
+    this.retries(options.retries);
+  }
   this.useColors(options.useColors);
   if (options.enableTimeouts !== null) {
     this.enableTimeouts(options.enableTimeouts);
@@ -372,6 +376,20 @@ Mocha.prototype.timeout = function(timeout) {
   return this;
 };
 
+/**
+ * Set the number of times to retry failed tests.
+ *
+ * @param {Number} retry times
+ * @return {Mocha}
+ * @api public
+ * @param {number} retry times
+ * @return {Mocha}
+ */
+Mocha.prototype.retries = function(n) {
+  this.suite.retries(n);
+  return this;
+};
+
 /**
  * Set slowness threshold in milliseconds.
  *

lib/reporters/json-cov.js

@@ -144,6 +144,7 @@ function coverage(filename, data) {
 function clean(test) {
   return {
     duration: test.duration,
+    currentRetry: test.currentRetry(),
     fullTitle: test.fullTitle(),
     title: test.title
   };

lib/reporters/json-stream.js

@@ -54,6 +54,7 @@ function clean(test) {
   return {
     title: test.title,
     fullTitle: test.fullTitle(),
-    duration: test.duration
+    duration: test.duration,
+    currentRetry: test.currentRetry()
   };
 }

lib/reporters/json.js

@@ -69,6 +69,7 @@ function clean(test) {
     title: test.title,
     fullTitle: test.fullTitle(),
     duration: test.duration,
+    currentRetry: test.currentRetry(),
     err: errorJSON(test.err || {})
   };
 }

lib/runnable.js

@@ -52,6 +52,8 @@ function Runnable(title, fn) {
   this._enableTimeouts = true;
   this.timedOut = false;
   this._trace = new Error('done() called multiple times');
+  this._retries = -1;
+  this._currentRetry = 0;
 }
 
 /**
@@ -128,6 +130,30 @@ Runnable.prototype.skip = function() {
   throw new Pending();
 };
 
+/**
+ * Set number of retries.
+ *
+ * @api private
+ */
+Runnable.prototype.retries = function(n) {
+  if (!arguments.length) {
+    return this._retries;
+  }
+  this._retries = n;
+};
+
+/**
+ * Get current retry
+ *
+ * @api private
+ */
+Runnable.prototype.currentRetry = function(n) {
+  if (!arguments.length) {
+    return this._currentRetry;
+  }
+  this._currentRetry = n;
+};
+
 /**
  * Return the full title generated by recursively concatenating the parent's
  * full title.

lib/runner.js

@@ -529,8 +529,16 @@ Runner.prototype.runTests = function(suite, fn) {
         test = self.test;
 
         if (err) {
+          var retry = test.currentRetry();
           if (err instanceof Pending) {
             self.emit('pending', test);
+          } else if (retry < test.retries()) {
+            test.currentRetry(retry + 1);
+            tests.unshift(test);
+
+            // Early return + hook trigger so that it doesn't
+            // increment the count wrong
+            return self.hookUp('afterEach', next);
           } else {
             self.fail(test, err);
           }

lib/suite.js

@@ -60,6 +60,7 @@ function Suite(title, parentContext) {
   this._enableTimeouts = true;
   this._slow = 75;
   this._bail = false;
+  this._retries = -1;
   this.delayed = false;
 }
 
@@ -79,6 +80,7 @@ Suite.prototype.clone = function() {
   debug('clone');
   suite.ctx = this.ctx;
   suite.timeout(this.timeout());
+  suite.retries(this.retries());
   suite.enableTimeouts(this.enableTimeouts());
   suite.slow(this.slow());
   suite.bail(this.bail());
@@ -107,6 +109,22 @@ Suite.prototype.timeout = function(ms) {
   return this;
 };
 
+/**
+ * Set number of times to retry a failed test.
+ *
+ * @api private
+ * @param {number|string} n
+ * @return {Suite|number} for chaining
+ */
+Suite.prototype.retries = function(n) {
+  if (!arguments.length) {
+    return this._retries;
+  }
+  debug('retries %d', n);
+  this._retries = parseInt(n, 10) || 0;
+  return this;
+};
+
 /**
   * Set timeout to `enabled`.
   *
@@ -179,6 +197,7 @@ Suite.prototype.beforeAll = function(title, fn) {
   var hook = new Hook(title, fn);
   hook.parent = this;
   hook.timeout(this.timeout());
+  hook.retries(this.retries());
   hook.enableTimeouts(this.enableTimeouts());
   hook.slow(this.slow());
   hook.ctx = this.ctx;
@@ -208,6 +227,7 @@ Suite.prototype.afterAll = function(title, fn) {
   var hook = new Hook(title, fn);
   hook.parent = this;
   hook.timeout(this.timeout());
+  hook.retries(this.retries());
   hook.enableTimeouts(this.enableTimeouts());
   hook.slow(this.slow());
   hook.ctx = this.ctx;
@@ -237,6 +257,7 @@ Suite.prototype.beforeEach = function(title, fn) {
   var hook = new Hook(title, fn);
   hook.parent = this;
   hook.timeout(this.timeout());
+  hook.retries(this.retries());
   hook.enableTimeouts(this.enableTimeouts());
   hook.slow(this.slow());
   hook.ctx = this.ctx;
@@ -266,6 +287,7 @@ Suite.prototype.afterEach = function(title, fn) {
   var hook = new Hook(title, fn);
   hook.parent = this;
   hook.timeout(this.timeout());
+  hook.retries(this.retries());
   hook.enableTimeouts(this.enableTimeouts());
   hook.slow(this.slow());
   hook.ctx = this.ctx;
@@ -284,6 +306,7 @@ Suite.prototype.afterEach = function(title, fn) {
 Suite.prototype.addSuite = function(suite) {
   suite.parent = this;
   suite.timeout(this.timeout());
+  suite.retries(this.retries());
   suite.enableTimeouts(this.enableTimeouts());
   suite.slow(this.slow());
   suite.bail(this.bail());
@@ -302,6 +325,7 @@ Suite.prototype.addSuite = function(suite) {
 Suite.prototype.addTest = function(test) {
   test.parent = this;
   test.timeout(this.timeout());
+  test.retries(this.retries());
   test.enableTimeouts(this.enableTimeouts());
   test.slow(this.slow());
   test.ctx = this.ctx;

test/integration/fixtures/options/retries.js

@@ -0,0 +1,5 @@
+describe('retries', function() {
+  it('should fail', function () {
+    throw new Error('retry failure');
+  });
+});

test/integration/fixtures/retries/early-pass.js

@@ -0,0 +1,11 @@
+describe('retries', function() {
+  this.retries(1);
+  var times = 0;
+
+  it('should quit early', function() {
+    times++;
+    if (times !== 2) {
+      throw new Error('retry error ' + times);
+    }
+  });
+});

test/integration/fixtures/retries/hooks.js

@@ -0,0 +1,25 @@
+describe('retries', function() {
+  var times = 0;
+  before(function () {
+    console.log('before');
+  });
+
+  after(function () {
+    console.log('after');
+  });
+
+  beforeEach(function() {
+    console.log('before each', times);
+  });
+
+  afterEach(function () {
+    console.log('after each', times);
+  });
+
+  it('should allow override and run appropriate hooks', function(){
+    this.retries(4);
+    console.log('TEST', times);
+    times++;
+    throw new Error('retry error');
+  });
+});

test/integration/fixtures/retries/nested.js

@@ -0,0 +1,9 @@
+describe('retries', function() {
+  this.retries(3);
+  describe('nested', function () {
+    it('should retry on test 3', function(){
+      this.retries(1);
+      throw new Error('retry error');
+    });
+  });
+});

test/integration/options.js

@@ -151,4 +151,20 @@ describe('options', function() {
       });
     });
   });
+
+  describe('--retries', function() {
+    it('retries after a certain threshold', function (done) {
+      args = ['--retries', '3'];
+      run('options/retries.js', args, function(err, res) {
+        assert(!err);
+        assert.equal(res.stats.pending, 0);
+        assert.equal(res.stats.passes, 0);
+        assert.equal(res.stats.tests, 1);
+        assert.equal(res.tests[0].currentRetry, 3);
+        assert.equal(res.stats.failures, 1);
+        assert.equal(res.code, 1);
+        done();
+      });
+    })
+  });
 });