From 82ac23ac37d018d75884c323cf25c906fd402d84 Mon Sep 17 00:00:00 2001
From: Jakob Hilden <jakobhilden@gmail.com>
Date: Thu, 28 Feb 2019 15:33:42 +0100
Subject: [PATCH] adds (optional) automatic retries with exponential backoff to
 jobs

Define a job with your intended maximum number of retries and agenda will take care of automatically rerunning the job in case of a failure.

`agenda.define('job with retries', { maxRetries: 2 },`

The job is retried with an exponentially increasing delay to avoid too high load on your queue.

The formula for the backoff is copied from [Sidekiq](https://github.com/mperham/sidekiq/wiki/Error-Handling#automatic-job-retry) and includes a random element.

These would be some possible example values for the delay:

|retry #|delay in s|
|---| --- |
| 1 | 27 |
| 2 | 66 |
| 3 | 118 |
| 4 | 346 |
| 6 | 727 |
| 7 | 1366 |
| 8 | 2460 |
| 9 | 4379 |
| 10 | 6613 |
| 11 | 10288 |
| 12 | 14977 |
| 13 | 20811 |
| 14 | 28636 |
| 15 | 38554 |
| 16 | 50830 |
| 17 | 65803 |
| 18 | 83625 |

Fixes #123
---
 .gitignore           |  1 +
 lib/agenda/create.js |  3 ++-
 lib/agenda/define.js |  3 ++-
 lib/job/fail.js      | 11 +++++++++++
 lib/job/index.js     |  2 +-
 test/retry.js        | 11 ++---------
 6 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index 496696473..278641866 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@ coverage.html
 .idea
 .DS_Store
 docs
+package-lock.json
diff --git a/lib/agenda/create.js b/lib/agenda/create.js
index 2d20b5384..c7bfcb8d6 100644
--- a/lib/agenda/create.js
+++ b/lib/agenda/create.js
@@ -13,6 +13,7 @@ const Job = require('../job');
 module.exports = function(name, data) {
   debug('Agenda.create(%s, [Object])', name);
   const priority = this._definitions[name] ? this._definitions[name].priority : 0;
-  const job = new Job({name, data, type: 'normal', priority, agenda: this});
+  const maxRetries = this._definitions[name] ? this._definitions[name].maxRetries : 0;
+  const job = new Job({name, data, type: 'normal', priority, maxRetries, agenda: this});
   return job;
 };
diff --git a/lib/agenda/define.js b/lib/agenda/define.js
index cea0ea926..8f397d47a 100644
--- a/lib/agenda/define.js
+++ b/lib/agenda/define.js
@@ -23,7 +23,8 @@ module.exports = function(name, options, processor) {
     priority: options.priority || 0,
     lockLifetime: options.lockLifetime || this._defaultLockLifetime,
     running: 0,
-    locked: 0
+    locked: 0,
+    maxRetries: options.maxRetries || 0
   };
   debug('job [%s] defined with following options: \n%O', name, this._definitions[name]);
 };
diff --git a/lib/job/fail.js b/lib/job/fail.js
index 7fc682eb6..d07564cbb 100644
--- a/lib/job/fail.js
+++ b/lib/job/fail.js
@@ -1,5 +1,6 @@
 'use strict';
 const debug = require('debug')('agenda:job');
+const moment = require('moment-timezone');
 
 /**
  * Fails the job with a reason (error) specified
@@ -18,5 +19,15 @@ module.exports = function(reason) {
   this.attrs.failedAt = now;
   this.attrs.lastFinishedAt = now;
   debug('[%s:%s] fail() called [%d] times so far', this.attrs.name, this.attrs._id, this.attrs.failCount);
+  if (this.attrs.failCount <= this.attrs.maxRetries) {
+    const retryCount = this.attrs.failCount - 1
+    // exponential backoff formula inspired by Sidekiq
+    // see:
+    // https://github.com/mperham/sidekiq/wiki/Error-Handling#automatic-job-retry
+    // https://github.com/mperham/sidekiq/blob/47028ef8b7cb998df6d7d72eb8af731bc6bbc341/lib/sidekiq/job_retry.rb#L225
+    const waitInSeconds = Math.pow(retryCount, 4) + 15 + ((Math.random() * 30) * (retryCount + 1));
+    debug('[%s:%s] retrying again in %d seconds - retry %d of %d', this.attrs.name, this.attrs._id, parseInt(waitInSeconds, 10), retryCount + 1, this.attrs.maxRetries);
+    this.attrs.nextRunAt = moment().add(waitInSeconds, 'seconds').toDate();
+  }
   return this;
 };
diff --git a/lib/job/index.js b/lib/job/index.js
index f11f005a5..b3174ceab 100644
--- a/lib/job/index.js
+++ b/lib/job/index.js
@@ -36,7 +36,7 @@ class Job {
     // Process args
     args.priority = parsePriority(args.priority) || 0;
 
-    // Set attrs to args
+    // Set attrs based on args
     const attrs = {};
     for (const key in args) {
       if ({}.hasOwnProperty.call(args, key)) {
diff --git a/test/retry.js b/test/retry.js
index 6df1cd3c5..b96f50a1a 100644
--- a/test/retry.js
+++ b/test/retry.js
@@ -60,7 +60,7 @@ describe('Retry', () => {
     let shouldFail = true;
 
     agenda.processEvery(100); // Shave 5s off test runtime :grin:
-    agenda.define('a job', (job, done) => {
+    agenda.define('a job', { maxRetries: 2 }, (job, done) => {
       if (shouldFail) {
         shouldFail = false;
         return done(new Error('test failure'));
@@ -68,13 +68,6 @@ describe('Retry', () => {
       done();
     });
 
-    agenda.on('fail:a job', (err, job) => {
-      if (err) {
-        // Do nothing as this is expected to fail.
-      }
-      job.schedule('now').save();
-    });
-
     const successPromise = new Promise(resolve =>
       agenda.on('success:a job', resolve)
     );
@@ -83,5 +76,5 @@ describe('Retry', () => {
 
     await agenda.start();
     await successPromise;
-  });
+  }).timeout((15 + 30) * 1000);
 });