adds (optional) automatic retries with exponential backoff to jobs

Define a job with your intended maximum number of retries and agenda will take care of automatically rerunning the job in case of a failure.

`agenda.define('job with retries', { maxRetries: 2 },`

The job is retried with an exponentially increasing delay to avoid too high load on your queue.

The formula for the backoff is copied from [Sidekiq](https://github.com/mperham/sidekiq/wiki/Error-Handling#automatic-job-retry) and includes a random element.

These would be some possible example values for the delay:

|retry #|delay in s|
|---| --- |
| 1 | 27 |
| 2 | 66 |
| 3 | 118 |
| 4 | 346 |
| 6 | 727 |
| 7 | 1366 |
| 8 | 2460 |
| 9 | 4379 |
| 10 | 6613 |
| 11 | 10288 |
| 12 | 14977 |
| 13 | 20811 |
| 14 | 28636 |
| 15 | 38554 |
| 16 | 50830 |
| 17 | 65803 |
| 18 | 83625 |

Fixes agenda#123

.gitignore

@@ -3,3 +3,4 @@ coverage.html
 .idea
 .DS_Store
 docs
+package-lock.json

lib/agenda/create.js

@@ -13,6 +13,7 @@ const Job = require('../job');
 module.exports = function(name, data) {
   debug('Agenda.create(%s, [Object])', name);
   const priority = this._definitions[name] ? this._definitions[name].priority : 0;
-  const job = new Job({name, data, type: 'normal', priority, agenda: this});
+  const maxRetries = this._definitions[name] ? this._definitions[name].maxRetries : 0;
+  const job = new Job({name, data, type: 'normal', priority, maxRetries, agenda: this});
   return job;
 };

lib/agenda/define.js

@@ -23,7 +23,8 @@ module.exports = function(name, options, processor) {
     priority: options.priority || 0,
     lockLifetime: options.lockLifetime || this._defaultLockLifetime,
     running: 0,
-    locked: 0
+    locked: 0,
+    maxRetries: options.maxRetries || 0
   };
   debug('job [%s] defined with following options: \n%O', name, this._definitions[name]);
 };

lib/job/fail.js

@@ -1,5 +1,6 @@
 'use strict';
 const debug = require('debug')('agenda:job');
+const moment = require('moment-timezone');
 
 /**
  * Fails the job with a reason (error) specified
@@ -18,5 +19,15 @@ module.exports = function(reason) {
   this.attrs.failedAt = now;
   this.attrs.lastFinishedAt = now;
   debug('[%s:%s] fail() called [%d] times so far', this.attrs.name, this.attrs._id, this.attrs.failCount);
+  if (this.attrs.failCount <= this.attrs.maxRetries) {
+    const retryCount = this.attrs.failCount - 1
+    // exponential backoff formula inspired by Sidekiq
+    // see:
+    // https://github.com/mperham/sidekiq/wiki/Error-Handling#automatic-job-retry
+    // https://github.com/mperham/sidekiq/blob/47028ef8b7cb998df6d7d72eb8af731bc6bbc341/lib/sidekiq/job_retry.rb#L225
+    const waitInSeconds = Math.pow(retryCount, 4) + 15 + ((Math.random() * 30) * (retryCount + 1));
+    debug('[%s:%s] retrying again in %d seconds - retry %d of %d', this.attrs.name, this.attrs._id, parseInt(waitInSeconds, 10), retryCount + 1, this.attrs.maxRetries);
+    this.attrs.nextRunAt = moment().add(waitInSeconds, 'seconds').toDate();
+  }
   return this;
 };

lib/job/index.js

@@ -36,7 +36,7 @@ class Job {
     // Process args
     args.priority = parsePriority(args.priority) || 0;
 
-    // Set attrs to args
+    // Set attrs based on args
     const attrs = {};
     for (const key in args) {
       if ({}.hasOwnProperty.call(args, key)) {

test/retry.js

@@ -60,21 +60,14 @@ describe('Retry', () => {
     let shouldFail = true;
 
     agenda.processEvery(100); // Shave 5s off test runtime :grin:
-    agenda.define('a job', (job, done) => {
+    agenda.define('a job', { maxRetries: 2 }, (job, done) => {
       if (shouldFail) {
         shouldFail = false;
         return done(new Error('test failure'));
       }
       done();
     });
 
-    agenda.on('fail:a job', (err, job) => {
-      if (err) {
-        // Do nothing as this is expected to fail.
-      }
-      job.schedule('now').save();
-    });
-
     const successPromise = new Promise(resolve =>
       agenda.on('success:a job', resolve)
     );
@@ -83,5 +76,5 @@ describe('Retry', () => {
 
     await agenda.start();
     await successPromise;
-  });
+  }).timeout((15 + 30) * 1000);
 });