Permalink
Browse files

Merge pull request #79 from seomoz/lua_caching

Lua caching
  • Loading branch information...
2 parents e7cb4db + 8741333 commit ad585cc5521552b90e09efc991bb5a8884092657 @myronmarston myronmarston committed Feb 11, 2013
View
@@ -7,24 +7,24 @@ module Qless
# Define our error base class before requiring the other
# files so they can define subclasses.
Error = Class.new(StandardError)
+
+ # to maintain backwards compatibility with v2.x of that gem we need this constant because:
+ # * (lua.rb) the #evalsha method signature changed between v2.x and v3.x of the redis ruby gem
+ # * (worker.rb) in v3.x you have to reconnect to the redis server after forking the process
+ USING_LEGACY_REDIS_VERSION = ::Redis::VERSION.to_f < 3.0
end
+require "qless/lua_script_cache"
require "qless/version"
require "qless/config"
require "qless/queue"
require "qless/job"
-require "qless/lua"
module Qless
extend self
UnsupportedRedisVersionError = Class.new(Error)
- # to maintain backwards compatibility with v2.x of that gem we need this constant because:
- # * (lua.rb) the #evalsha method signature changed between v2.x and v3.x of the redis ruby gem
- # * (worker.rb) in v3.x you have to reconnect to the redis server after forking the process
- USING_LEGACY_REDIS_VERSION = ::Redis::VERSION.to_f < 3.0
-
def generate_jid
SecureRandom.uuid.gsub('-', '')
end
@@ -40,6 +40,10 @@ def worker_name
@worker_name ||= [Socket.gethostname, Process.pid.to_s].join('-')
end
+ def lua_script_cache
+ @lua_script_cache ||= LuaScriptCache.new
+ end
+
class ClientJobs
def initialize(client)
@client = client
@@ -158,7 +162,7 @@ def initialize(options = {})
@config = Config.new(self)
['cancel', 'config', 'complete', 'depends', 'fail', 'failed', 'get', 'heartbeat', 'jobs', 'peek', 'pop',
'priority', 'put', 'queues', 'recur', 'retry', 'stats', 'tag', 'track', 'workers', 'pause', 'unpause'].each do |f|
- self.instance_variable_set("@_#{f}", Lua.new(f, @redis))
+ self.instance_variable_set("@_#{f}", Qless.lua_script_cache.script_for(f, @redis))
end
@jobs = ClientJobs.new(self)
View
@@ -1,5 +1,3 @@
-require "qless/lua"
-require "redis"
require "json"
module Qless
@@ -28,4 +26,4 @@ def clear(option)
@client._config.call([], ['unset', option])
end
end
-end
+end
View
@@ -1,6 +1,5 @@
require "qless"
require "qless/queue"
-require "qless/lua"
require "redis"
require "json"
@@ -1,5 +1,5 @@
module Qless
- class Lua
+ class LuaScript
LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
def initialize(name, redis)
@@ -9,18 +9,28 @@ def initialize(name, redis)
reload()
end
+ attr_reader :name, :redis
+
def reload()
@sha = @redis.script(:load, File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua")))
end
def call(keys, argv)
- begin
- return @redis.evalsha(@sha, keys.length, *(keys + argv)) if USING_LEGACY_REDIS_VERSION
- return @redis.evalsha(@sha, keys: keys, argv: argv)
- rescue
- reload
- return @redis.evalsha(@sha, keys.length, *(keys + argv)) if USING_LEGACY_REDIS_VERSION
- return @redis.evalsha(@sha, keys: keys, argv: argv)
+ _call(keys, argv)
+ rescue
+ reload
+ _call(keys, argv)
+ end
+
+ private
+
+ if USING_LEGACY_REDIS_VERSION
+ def _call(keys, argv)
+ @redis.evalsha(@sha, keys.length, *(keys + argv))
+ end
+ else
+ def _call(keys, argv)
+ @redis.evalsha(@sha, keys: keys, argv: argv)
end
end
end
@@ -0,0 +1,20 @@
+require 'qless/lua_script'
+
+module Qless
+ class LuaScriptCache
+ def initialize
+ @cache = {}
+ end
+
+ def script_for(script_name, redis_connection)
+ key = CacheKey.new(script_name, redis_connection.id)
+
+ @cache.fetch(key) do
+ @cache[key] = LuaScript.new(script_name, redis_connection)
+ end
+ end
+
+ CacheKey = Struct.new(:script_name, :redis_server_url)
+ end
+end
+
View
@@ -1,4 +1,3 @@
-require "qless/lua"
require "qless/job"
require "redis"
require "json"
@@ -2169,15 +2169,15 @@ def should_only_have_tracked_jid_for(type)
describe "#lua" do
it "checks cancel's arguments" do
- cancel = Qless::Lua.new("cancel", @redis)
+ cancel = Qless::LuaScript.new("cancel", @redis)
# Providing in keys
lambda { cancel(["foo"], ["deadbeef"]) }.should raise_error
# Missing an id
lambda { cancel([], []) }.should raise_error
end
it "checks complete's arguments" do
- complete = Qless::Lua.new("complete", @redis)
+ complete = Qless::LuaScript.new("complete", @redis)
[
# Not enough args
[[], []],
@@ -2199,7 +2199,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks config's arguments" do
- config = Qless::Lua.new("config", @redis)
+ config = Qless::LuaScript.new("config", @redis)
[
# Passing in keys
[["foo"], []],
@@ -2211,7 +2211,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks fail's arguments" do
- fail = Qless::Lua.new("fail", @redis)
+ fail = Qless::LuaScript.new("fail", @redis)
[
# Passing in keys
[["foo"], ["deadbeef", "worker1", "foo", "bar", 12345]],
@@ -2233,7 +2233,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks failed's arguments" do
- failed = Qless::Lua.new("failed", @redis)
+ failed = Qless::LuaScript.new("failed", @redis)
[
# Passing in keys
[["foo"], []],
@@ -2245,7 +2245,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks get's arguments" do
- get = Qless::Lua.new("get", @redis)
+ get = Qless::LuaScript.new("get", @redis)
[
# Passing in keys
[["foo"], ["deadbeef"]],
@@ -2255,7 +2255,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks heartbeat's arguments" do
- heartbeat = Qless::Lua.new("heartbeat", @redis)
+ heartbeat = Qless::LuaScript.new("heartbeat", @redis)
[
# Passing in keys
[["foo"], ["deadbeef", "foo", 12345]],
@@ -2273,7 +2273,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks jobs' arguments" do
- jobs = Qless::Lua.new('jobs', @redis)
+ jobs = Qless::LuaScript.new('jobs', @redis)
[
# Providing keys
[['foo'], []],
@@ -2289,7 +2289,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks peek's arguments" do
- peek = Qless::Lua.new("peek", @redis)
+ peek = Qless::LuaScript.new("peek", @redis)
[
# Passing in no keys
[[], [1, 12345]],
@@ -2307,7 +2307,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks pop's arguments" do
- pop = Qless::Lua.new("pop", @redis)
+ pop = Qless::LuaScript.new("pop", @redis)
[
# Passing in no keys
[[], ["worker1", 1, 12345, 12346]],
@@ -2331,7 +2331,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks priority's arguments" do
- priority = Qless::Lua.new("pop", @redis)
+ priority = Qless::LuaScript.new("pop", @redis)
[
# Passing in keys
[['foo'], ['12345', 1]],
@@ -2345,7 +2345,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks put's arguments" do
- put = Qless::Lua.new("put", @redis)
+ put = Qless::LuaScript.new("put", @redis)
[
# Passing in no keys
[[], ["deadbeef", "{}", 12345]],
@@ -2375,7 +2375,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks queues' arguments" do
- queues = Qless::Lua.new("queues", @redis)
+ queues = Qless::LuaScript.new("queues", @redis)
[
# Passing in keys
[["foo"], [12345]],
@@ -2387,7 +2387,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks recur's arguments" do
- recur = Qless::Lua.new("recur", @redis)
+ recur = Qless::LuaScript.new("recur", @redis)
[
# Passing in keys
[['foo'], [12345]],
@@ -2422,7 +2422,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks retry's arguments" do
- rtry = Qless::Lua.new("queues", @redis)
+ rtry = Qless::LuaScript.new("queues", @redis)
[
# Passing in keys
[['foo'], ['12345', 'testing', 'worker', 12345, 0]],
@@ -2442,7 +2442,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks stats' arguments" do
- stats = Qless::Lua.new("stats", @redis)
+ stats = Qless::LuaScript.new("stats", @redis)
[
# Passing in keys
[["foo"], ["foo", "bar"]],
@@ -2454,7 +2454,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks tags' arguments" do
- tag = Qless::Lua.new("tag", @redis)
+ tag = Qless::LuaScript.new("tag", @redis)
[
# Passing in keys
[['foo'], ['add', '12345', 12345, 'foo']],
@@ -2481,7 +2481,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks track's arguments" do
- track = Qless::Lua.new("track", @redis)
+ track = Qless::LuaScript.new("track", @redis)
[
# Passing in keys
[["foo"], []],
@@ -0,0 +1,55 @@
+require 'spec_helper'
+require 'qless'
+require 'qless/lua_script_cache'
+
+module Qless
+ describe LuaScriptCache do
+ let(:redis_1a) { fire_double("Redis", id: "redis://foo:1234/1", script: "sha") }
+ let(:redis_1b) { fire_double("Redis", id: "redis://foo:1234/1", script: "sha") }
+ let(:redis_2) { fire_double("Redis", id: "redis://foo:1234/2", script: "sha") }
+ let(:cache) { LuaScriptCache.new }
+
+ before { File.stub(read: "script content") }
+
+ it 'returns different lua script objects when the script name is different' do
+ script_1 = cache.script_for("foo", redis_1a)
+ script_2 = cache.script_for("bar", redis_1a)
+
+ expect(script_1).to be_a(LuaScript)
+ expect(script_2).to be_a(LuaScript)
+
+ expect(script_1).not_to be(script_2)
+
+ expect(script_1.name).to eq("foo")
+ expect(script_2.name).to eq("bar")
+ end
+
+ it 'returns different lua script objects when the redis connection is to a different server' do
+ script_1 = cache.script_for("foo", redis_1a)
+ script_2 = cache.script_for("foo", redis_2)
+
+ expect(script_1).to be_a(LuaScript)
+ expect(script_2).to be_a(LuaScript)
+
+ expect(script_1).not_to be(script_2)
+
+ expect(script_1.redis).to be(redis_1a)
+ expect(script_2.redis).to be(redis_2)
+ end
+
+ it 'returns the same lua script object when the script name and redis connection are the same' do
+ script_1 = cache.script_for("foo", redis_1a)
+ script_2 = cache.script_for("foo", redis_1a)
+
+ expect(script_1).to be(script_2)
+ end
+
+ it 'returns the same lua script object when the script name and redis conneciton URL are the same' do
+ script_1 = cache.script_for("foo", redis_1a)
+ script_2 = cache.script_for("foo", redis_1b)
+
+ expect(script_1).to be(script_2)
+ end
+ end
+end
+
@@ -18,12 +18,19 @@
end
end
+ describe ".lua_script_cache" do
+ it 'returns a memoized script cache instance' do
+ expect(Qless.lua_script_cache).to be_a(Qless::LuaScriptCache)
+ expect(Qless.lua_script_cache).to be(Qless.lua_script_cache)
+ end
+ end
+
context 'when instantiated' do
- let(:redis) { fire_double("Redis", info: { "redis_version" => "2.6.0" }) }
+ let(:redis) { fire_double("Redis", id: "redis://foo:1/1", info: { "redis_version" => "2.6.0" }) }
let(:redis_class) { fire_replaced_class_double("Redis") }
before do
- Qless::Lua.stub(:new) # so no scripts get loaded
+ Qless::LuaScript.stub(:new) # so no scripts get loaded
redis_class.stub(connect: redis)
end
@@ -48,6 +55,17 @@
client = Qless::Client.new(redis: redis)
client.redis.should be(redis)
end
+
+ it 'loads the lua scripts from the cache so that the scripts are not unnecessarily loaded multiple times' do
+ cache = fire_double("Qless::LuaScriptCache")
+ Qless.stub(lua_script_cache: cache)
+
+ loaded_scripts = []
+ cache.stub(:script_for) { |name, redis| loaded_scripts << name }
+
+ client = Qless::Client.new(redis: redis)
+ expect(loaded_scripts).to include("put", "complete")
+ end
end
end

0 comments on commit ad585cc

Please sign in to comment.