Merge pull request #687 from 37signals/retry_reconnection

Worker tries to reconnect to Redis up to 3 times with increasing delay before giving up

lib/resque/worker.rb

@@ -151,7 +151,7 @@ def work(interval = 5.0, &block)
           else
             unregister_signal_handlers if !@cant_fork
             procline "Processing #{job.queue} since #{Time.now.to_i}"
-            redis.client.reconnect # Don't share connection with parent
+            reconnect
             perform(job, &block)
             exit! unless @cant_fork
           end
@@ -224,6 +224,24 @@ def reserve(interval = 5.0)
       Job.new(queue.name, job) if queue && job
     end
 
+    # Reconnect to Redis to avoid sharing a connection with the parent,
+    # retry up to 3 times with increasing delay before giving up.
+    def reconnect
+      tries = 0
+      begin
+        redis.client.reconnect
+      rescue Redis::BaseConnectionError
+        if (tries += 1) <= 3
+          log "Error reconnecting to Redis; retrying"
+          sleep(tries)
+          retry
+        else
+          log "Error reconnecting to Redis; quitting"
+          raise
+        end
+      end
+    end
+
     # Returns a list of queues to use when searching for a job.
     # A splat ("*") means you want every queue (in alpha order) - this
     # can be useful for dynamically adding new queues. Low priority queues

test/worker_test.rb

@@ -504,6 +504,40 @@ def self.perform
     refute_equal original_connection, Resque.redis.client.connection.instance_variable_get("@sock")
   end
 
+  it "tries to reconnect three times before giving up" do
+    begin
+      class Redis::Client
+        alias_method :original_reconnect, :reconnect
+
+        def reconnect
+          raise Redis::BaseConnectionError
+        end
+      end
+
+      class Resque::Worker
+        alias_method :original_sleep, :sleep
+
+        def sleep(duration = nil)
+          # noop
+        end
+      end
+
+      @worker.very_verbose = true
+      stdout, stderr = capture_io { @worker.work(0) }
+
+      assert_equal 3, stdout.scan(/retrying/).count
+      assert_equal 1, stdout.scan(/quitting/).count
+    ensure
+      class Redis::Client
+        alias_method :reconnect, :original_reconnect
+      end
+
+      class Resque::Worker
+        alias_method :sleep, :original_sleep
+      end
+    end
+  end
+
   it "will call before_pause before it is paused" do
     before_pause_called = false
     captured_worker = nil