Permalink
Browse files

Merge pull request #90 from seomoz/harden_worker

Harden worker and improve script caching
  • Loading branch information...
2 parents 8e2917b + 57d264f commit 7502f0e0777ad860ff1672e568e70da64d648873 @myronmarston myronmarston committed Apr 5, 2013
View
@@ -14,11 +14,11 @@ module Qless
USING_LEGACY_REDIS_VERSION = ::Redis::VERSION.to_f < 3.0
end
-require "qless/lua_script_cache"
require "qless/version"
require "qless/config"
require "qless/queue"
require "qless/job"
+require "qless/lua_script"
module Qless
extend self
@@ -40,10 +40,6 @@ def worker_name
@worker_name ||= [Socket.gethostname, Process.pid.to_s].join('-')
end
- def lua_script_cache
- @lua_script_cache ||= LuaScriptCache.new
- end
-
class ClientJobs
def initialize(client)
@client = client
@@ -163,7 +159,7 @@ def initialize(options = {})
['cancel', 'config', 'complete', 'depends', 'fail', 'failed', 'get', 'heartbeat', 'jobs', 'peek', 'pop',
'priority', 'put', 'queues', 'recur', 'retry', 'stats', 'tag', 'track', 'workers', 'pause', 'unpause',
'deregister_workers'].each do |f|
- self.instance_variable_set("@_#{f}", Qless.lua_script_cache.script_for(f, @redis))
+ self.instance_variable_set("@_#{f}", Qless::LuaScript.new(f, @redis))
end
@jobs = ClientJobs.new(self)
@@ -1,18 +1,19 @@
+require 'digest/sha1'
+
module Qless
class LuaScript
LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
- def initialize(name, redis, sha = nil)
- @sha = sha
+ def initialize(name, redis)
@name = name
@redis = redis
- reload() unless sha
+ @sha = Digest::SHA1.hexdigest(script_contents)
end
attr_reader :name, :redis, :sha
def reload()
- @sha = @redis.script(:load, File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua")))
+ @sha = @redis.script(:load, script_contents)
end
def call(keys, argv)
@@ -33,5 +34,9 @@ def _call(keys, argv)
@redis.evalsha(@sha, keys: keys, argv: argv)
end
end
+
+ def script_contents
+ @script_contents ||= File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua"))
+ end
end
end
@@ -1,22 +0,0 @@
-require 'qless/lua_script'
-
-module Qless
- class LuaScriptCache
- def initialize
- @sha_cache = {}
- end
-
- def script_for(script_name, redis_connection)
- key = CacheKey.new(script_name, redis_connection.id)
-
- sha = @sha_cache.fetch(key) do
- @sha_cache[key] = LuaScript.new(script_name, redis_connection).sha
- end
-
- LuaScript.new(script_name, redis_connection, sha)
- end
-
- CacheKey = Struct.new(:script_name, :redis_server_url)
- end
-end
-
View
@@ -78,7 +78,7 @@ def work(interval = 5.0)
next
end
- unless job = @job_reserver.reserve
+ unless job = reserve_job
break if interval.zero?
procline "Waiting for #{@job_reserver.description}"
log! "Sleeping for #{interval} seconds"
@@ -117,6 +117,15 @@ def perform(job)
try_complete(job)
end
+ def reserve_job
+ @job_reserver.reserve
+ rescue Exception => error
+ # We want workers to durably stay up, so we don't want errors
+ # during job reserving (e.g. network timeouts, etc) to kill
+ # the worker.
+ log "Got an error while reserving a job: #{error.class}: #{error.message}"
+ end
+
def shutdown
@shutdown = true
end
@@ -1,35 +0,0 @@
-require 'spec_helper'
-require "qless"
-require 'yaml'
-
-module Qless
- describe LuaScriptCache, :integration do
- def ensure_put_script_loaded(client)
- client.queues["q1"].put(Qless::Job, {})
- end
-
- let(:redis_1) { Redis.new(url: redis_url) }
- let(:redis_2) { Redis.new(url: redis_url) }
- let(:client_1) { Qless::Client.new(redis: redis_1) }
- let(:client_2) { Qless::Client.new(redis: redis_2) }
-
- before { stub_const("SomeJob", Class.new) }
-
- it 'does not prevent watch-multi-exec blocks from working properly' do
- ensure_put_script_loaded(client_1)
- ensure_put_script_loaded(client_2)
-
- redis_2.watch("some_key") do
- response = redis_2.multi do
- redis_1.set("some_key", "some_value") # force the multi block to no-op
- client_2.queues["some_job"].put(SomeJob, {})
- end
-
- expect(response).to be_nil
- end
-
- expect(client_2.queues["some_job"].length).to eq(0)
- end
- end
-end
-
@@ -0,0 +1,39 @@
+require 'spec_helper'
+require "qless"
+require 'yaml'
+
+module Qless
+ describe LuaScript, :integration do
+ let(:redis) { client.redis }
+
+ it 'does not make any redis requests upon initialization' do
+ redis = double("Redis")
+
+ expect {
+ LuaScript.new("config", redis)
+ }.not_to raise_error # e.g. MockExpectationError
+ end
+
+ it 'can issue the command without loading the script if it is already loaded' do
+ script = LuaScript.new("config", redis)
+ redis.script(:load, script.send(:script_contents)) # to ensure its loaded
+ redis.should_not_receive(:script)
+
+ expect {
+ script.call([], ['set', 'key', 3])
+ }.to change { redis.keys.size }.by(1)
+ end
+
+ it 'loads the script as needed if the command fails' do
+ script = LuaScript.new("config", redis)
+ redis.script(:flush)
+
+ redis.should_receive(:script).and_call_original
+
+ expect {
+ script.call([], ['set', 'key', 3])
+ }.to change { redis.keys.size }.by(1)
+ end
+ end
+end
+
@@ -1,56 +0,0 @@
-require 'spec_helper'
-require 'qless'
-require 'qless/lua_script_cache'
-
-module Qless
- describe LuaScriptCache do
- def redis_double(db_num)
- fire_double("Redis", id: "redis://foo:1234/#{db_num}").tap do |redis|
- redis.stub(:script) do |script_contents|
- script_contents
- end
- end
- end
-
- let(:redis_1a) { redis_double(1) }
- let(:redis_1b) { redis_double(1) }
- let(:redis_2) { redis_double(2) }
- let(:cache) { LuaScriptCache.new }
-
- before do
- File.stub(:read) do |file_name|
- file_name.split('/').last
- end
- end
-
- it 'loads each different script' do
- redis_1a.should_receive(:script).twice
-
- script_1 = cache.script_for("foo", redis_1a)
- script_2 = cache.script_for("bar", redis_1a)
-
- expect(script_1).to be_a(LuaScript)
- expect(script_2).to be_a(LuaScript)
-
- expect(script_1.name).to eq("foo")
- expect(script_2.name).to eq("bar")
- end
-
- it 'loads the same script each time it is needed in a different redis server' do
- redis_1a.should_receive(:script).once
- redis_2.should_receive(:script).once
-
- cache.script_for("foo", redis_1a)
- cache.script_for("foo", redis_2)
- end
-
- it 'loads a script only once when it is needed by multiple connections to the same redis server' do
- redis_1a.should_receive(:script).once
- redis_1b.should_not_receive(:script)
-
- cache.script_for("foo", redis_1a)
- cache.script_for("foo", redis_1b)
- end
- end
-end
-
@@ -18,13 +18,6 @@
end
end
- describe ".lua_script_cache" do
- it 'returns a memoized script cache instance' do
- expect(Qless.lua_script_cache).to be_a(Qless::LuaScriptCache)
- expect(Qless.lua_script_cache).to be(Qless.lua_script_cache)
- end
- end
-
context 'when instantiated' do
let(:redis) { fire_double("Redis", id: "redis://foo:1/1", info: { "redis_version" => "2.6.0" }) }
let(:redis_class) { fire_replaced_class_double("Redis") }
@@ -55,17 +48,6 @@
client = Qless::Client.new(redis: redis)
client.redis.should be(redis)
end
-
- it 'loads the lua scripts from the cache so that the scripts are not unnecessarily loaded multiple times' do
- cache = fire_double("Qless::LuaScriptCache")
- Qless.stub(lua_script_cache: cache)
-
- loaded_scripts = []
- cache.stub(:script_for) { |name, redis| loaded_scripts << name }
-
- client = Qless::Client.new(redis: redis)
- expect(loaded_scripts).to include("put", "complete")
- end
end
end
@@ -164,6 +164,19 @@ def around_perform(job)
worker.work(0)
paused_checks.should be >= 20
end
+
+ context 'when an error occurs while reserving a job' do
+ before { reserver.stub(:reserve) { raise "redis error" } }
+
+ it 'does not kill the worker' do
+ expect { worker.work(0) }.not_to raise_error
+ end
+
+ it 'logs the error' do
+ worker.work(0)
+ expect(log_output.string).to include("redis error")
+ end
+ end
end
end

0 comments on commit 7502f0e

Please sign in to comment.