Retry email jobs

Resolves #953
Adds 5 retires for each email which appear in a tab in the web resque UI.
Adding rety on mailers and all other jobs
Removing resque-rollbar as we don't need to send errors async to Rollbar.

.foreman

@@ -1 +1 @@
-concurrency: medium=1, mailers=1, resque_web=1, sunspot=1
+concurrency: medium=1, mailers=1, resque_web=1, sunspot=1, resque_scheduler=1

Gemfile

@@ -59,7 +59,7 @@ gem "pghero"
 gem "rack-cache", require: "rack/cache"
 gem "redis-rails"
 gem "resque"
-gem "resque-rollbar"
+gem "resque-retry"
 gem "thin"
 gem "thumbs_up", "~> 0.4.6"
 gem "whenever"

Gemfile.lock

@@ -171,6 +171,8 @@ GEM
       mail (~> 2.7)
     erubi (1.9.0)
     erubis (2.7.0)
+    et-orbi (1.2.4)
+      tzinfo
     eventmachine (1.2.7)
     excon (0.71.0)
     execjs (2.7.0)
@@ -188,6 +190,9 @@ GEM
       thor (~> 0.19.1)
     formtastic (3.1.5)
       actionpack (>= 3.2.13)
+    fugit (1.3.5)
+      et-orbi (~> 1.1, >= 1.1.8)
+      raabro (~> 1.1)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
     grape (1.3.0)
@@ -340,6 +345,7 @@ GEM
     pry-rails (0.3.9)
       pry (>= 0.10.4)
     public_suffix (4.0.1)
+    raabro (1.3.1)
     rack (2.2.2)
     rack-accept (0.4.5)
       rack (>= 0.4)
@@ -425,9 +431,14 @@ GEM
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
-    resque-rollbar (0.4.0)
-      resque
-      rollbar (>= 1.4)
+    resque-retry (1.7.3)
+      resque (>= 1.25, < 3.0)
+      resque-scheduler (~> 4.0)
+    resque-scheduler (4.4.0)
+      mono_logger (~> 1.0)
+      redis (>= 3.3)
+      resque (>= 1.26)
+      rufus-scheduler (~> 3.2)
     retryable (3.0.4)
     rgeo (2.1.1)
     rgeo-activerecord (6.2.1)
@@ -476,6 +487,8 @@ GEM
     ruby_parser (3.14.1)
       sexp_processor (~> 4.9)
     rubyzip (1.3.0)
+    rufus-scheduler (3.6.0)
+      fugit (~> 1.1, >= 1.1.6)
     safe_yaml (1.0.5)
     sanitize_email (2.0.3)
     sass (3.4.25)
@@ -633,7 +646,7 @@ DEPENDENCIES
   ratelimit
   redis-rails
   resque
-  resque-rollbar
+  resque-retry
   retryable
   rgeo-geojson
   rollbar

Procfile

@@ -2,3 +2,4 @@ medium: env QUEUE=medium bundle exec rake environment resque:work
 mailers: env QUEUE=mailers bundle exec rake environment resque:work
 resque_web: bundle exec resque-web -F -L --namespace resque:Cyclescape --app-dir ./ --pid-file /var/run/resque_web.pid --log-file /var/log/resque_web.log --url-file ./resque_web.url
 sunspot: bundle exec rake sunspot:solr:run
+resque_scheduler: bundle exec rake resque:scheduler

Rakefile

@@ -7,3 +7,6 @@
 require File.expand_path("config/application", __dir__)
 
 Rails.application.load_tasks
+
+require "resque/tasks"
+require "resque/scheduler/tasks"

config/initializers/resque.rb

@@ -1,13 +1,32 @@
 # frozen_string_literal: true
 
-require "resque/server"
-require "resque/failure/multiple"
+require "resque-retry"
+require "resque-retry/server"
 require "resque/failure/redis"
-require "resque/rollbar"
 
-Resque::Failure::Multiple.classes = [Resque::Failure::Redis, Resque::Failure::Rollbar]
-Resque::Failure.backend = Resque::Failure::Multiple
+# Enable resque-retry failure backend.
+Resque::Failure::MultipleWithRetrySuppression.classes = [Resque::Failure::Redis]
+Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
 
 Resque.redis.namespace = "resque:Cyclescape"
 Resque.inline = true if Rails.env.test?
-Resque.before_fork = proc { ActiveRecord::Base.establish_connection }
+
+Resque.after_fork do
+  defined?(ActiveRecord::Base) && ActiveRecord::Base.establish_connection
+end
+
+Resque.before_fork do
+  defined?(ActiveRecord::Base) && ActiveRecord::Base.connection.disconnect!
+end
+
+module ActionMailer
+  class DeliveryJob
+    retry_on StandardError, wait: :exponentially_longer, attempts: 5
+  end
+end
+
+module ActiveJob
+  class Base
+    retry_on StandardError, wait: :exponentially_longer, attempts: 5
+  end
+end