Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Lua caching #79

Merged
merged 2 commits into from

2 participants

Myron Marston proby
Myron Marston
Implement a lua script cache.

It's unnecessary (and slows things down) to load the same script into the same
redis server multiple times in the same process.
myronmarston added some commits
Myron Marston myronmarston Lua => LuaScript.
"LuaScript" is a better name for this class, as it represents a specific lua script, and not the lua language as a whole.

While I was at it, I moved the USING_LEGACY_REDIS_VERSION conditional out of the method and into the class body -- might as well evaulate that branch once at load time rather than once per script call.
a662978
Myron Marston myronmarston Implement a lua script cache.
It's unnecessary (and slows things down) to load the same script into the same
redis server multiple times in the same process.
8741333
proby

looks good

Myron Marston myronmarston merged commit ad585cc into from
Myron Marston myronmarston deleted the branch
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Feb 11, 2013
  1. Myron Marston

    Lua => LuaScript.

    myronmarston authored
    "LuaScript" is a better name for this class, as it represents a specific lua script, and not the lua language as a whole.
    
    While I was at it, I moved the USING_LEGACY_REDIS_VERSION conditional out of the method and into the class body -- might as well evaulate that branch once at load time rather than once per script call.
  2. Myron Marston

    Implement a lua script cache.

    myronmarston authored
    It's unnecessary (and slows things down) to load the same script into the same
    redis server multiple times in the same process.
This page is out of date. Refresh to see the latest.
18 lib/qless.rb
View
@@ -7,24 +7,24 @@ module Qless
# Define our error base class before requiring the other
# files so they can define subclasses.
Error = Class.new(StandardError)
+
+ # to maintain backwards compatibility with v2.x of that gem we need this constant because:
+ # * (lua.rb) the #evalsha method signature changed between v2.x and v3.x of the redis ruby gem
+ # * (worker.rb) in v3.x you have to reconnect to the redis server after forking the process
+ USING_LEGACY_REDIS_VERSION = ::Redis::VERSION.to_f < 3.0
end
+require "qless/lua_script_cache"
require "qless/version"
require "qless/config"
require "qless/queue"
require "qless/job"
-require "qless/lua"
module Qless
extend self
UnsupportedRedisVersionError = Class.new(Error)
- # to maintain backwards compatibility with v2.x of that gem we need this constant because:
- # * (lua.rb) the #evalsha method signature changed between v2.x and v3.x of the redis ruby gem
- # * (worker.rb) in v3.x you have to reconnect to the redis server after forking the process
- USING_LEGACY_REDIS_VERSION = ::Redis::VERSION.to_f < 3.0
-
def generate_jid
SecureRandom.uuid.gsub('-', '')
end
@@ -40,6 +40,10 @@ def worker_name
@worker_name ||= [Socket.gethostname, Process.pid.to_s].join('-')
end
+ def lua_script_cache
+ @lua_script_cache ||= LuaScriptCache.new
+ end
+
class ClientJobs
def initialize(client)
@client = client
@@ -158,7 +162,7 @@ def initialize(options = {})
@config = Config.new(self)
['cancel', 'config', 'complete', 'depends', 'fail', 'failed', 'get', 'heartbeat', 'jobs', 'peek', 'pop',
'priority', 'put', 'queues', 'recur', 'retry', 'stats', 'tag', 'track', 'workers', 'pause', 'unpause'].each do |f|
- self.instance_variable_set("@_#{f}", Lua.new(f, @redis))
+ self.instance_variable_set("@_#{f}", Qless.lua_script_cache.script_for(f, @redis))
end
@jobs = ClientJobs.new(self)
4 lib/qless/config.rb
View
@@ -1,5 +1,3 @@
-require "qless/lua"
-require "redis"
require "json"
module Qless
@@ -28,4 +26,4 @@ def clear(option)
@client._config.call([], ['unset', option])
end
end
-end
+end
1  lib/qless/job.rb
View
@@ -1,6 +1,5 @@
require "qless"
require "qless/queue"
-require "qless/lua"
require "redis"
require "json"
26 lib/qless/lua.rb → lib/qless/lua_script.rb
View
@@ -1,5 +1,5 @@
module Qless
- class Lua
+ class LuaScript
LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
def initialize(name, redis)
@@ -9,18 +9,28 @@ def initialize(name, redis)
reload()
end
+ attr_reader :name, :redis
+
def reload()
@sha = @redis.script(:load, File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua")))
end
def call(keys, argv)
- begin
- return @redis.evalsha(@sha, keys.length, *(keys + argv)) if USING_LEGACY_REDIS_VERSION
- return @redis.evalsha(@sha, keys: keys, argv: argv)
- rescue
- reload
- return @redis.evalsha(@sha, keys.length, *(keys + argv)) if USING_LEGACY_REDIS_VERSION
- return @redis.evalsha(@sha, keys: keys, argv: argv)
+ _call(keys, argv)
+ rescue
+ reload
+ _call(keys, argv)
+ end
+
+ private
+
+ if USING_LEGACY_REDIS_VERSION
+ def _call(keys, argv)
+ @redis.evalsha(@sha, keys.length, *(keys + argv))
+ end
+ else
+ def _call(keys, argv)
+ @redis.evalsha(@sha, keys: keys, argv: argv)
end
end
end
20 lib/qless/lua_script_cache.rb
View
@@ -0,0 +1,20 @@
+require 'qless/lua_script'
+
+module Qless
+ class LuaScriptCache
+ def initialize
+ @cache = {}
+ end
+
+ def script_for(script_name, redis_connection)
+ key = CacheKey.new(script_name, redis_connection.id)
+
+ @cache.fetch(key) do
+ @cache[key] = LuaScript.new(script_name, redis_connection)
+ end
+ end
+
+ CacheKey = Struct.new(:script_name, :redis_server_url)
+ end
+end
+
1  lib/qless/queue.rb
View
@@ -1,4 +1,3 @@
-require "qless/lua"
require "qless/job"
require "redis"
require "json"
36 spec/integration/qless_spec.rb
View
@@ -2169,7 +2169,7 @@ def should_only_have_tracked_jid_for(type)
describe "#lua" do
it "checks cancel's arguments" do
- cancel = Qless::Lua.new("cancel", @redis)
+ cancel = Qless::LuaScript.new("cancel", @redis)
# Providing in keys
lambda { cancel(["foo"], ["deadbeef"]) }.should raise_error
# Missing an id
@@ -2177,7 +2177,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks complete's arguments" do
- complete = Qless::Lua.new("complete", @redis)
+ complete = Qless::LuaScript.new("complete", @redis)
[
# Not enough args
[[], []],
@@ -2199,7 +2199,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks config's arguments" do
- config = Qless::Lua.new("config", @redis)
+ config = Qless::LuaScript.new("config", @redis)
[
# Passing in keys
[["foo"], []],
@@ -2211,7 +2211,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks fail's arguments" do
- fail = Qless::Lua.new("fail", @redis)
+ fail = Qless::LuaScript.new("fail", @redis)
[
# Passing in keys
[["foo"], ["deadbeef", "worker1", "foo", "bar", 12345]],
@@ -2233,7 +2233,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks failed's arguments" do
- failed = Qless::Lua.new("failed", @redis)
+ failed = Qless::LuaScript.new("failed", @redis)
[
# Passing in keys
[["foo"], []],
@@ -2245,7 +2245,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks get's arguments" do
- get = Qless::Lua.new("get", @redis)
+ get = Qless::LuaScript.new("get", @redis)
[
# Passing in keys
[["foo"], ["deadbeef"]],
@@ -2255,7 +2255,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks heartbeat's arguments" do
- heartbeat = Qless::Lua.new("heartbeat", @redis)
+ heartbeat = Qless::LuaScript.new("heartbeat", @redis)
[
# Passing in keys
[["foo"], ["deadbeef", "foo", 12345]],
@@ -2273,7 +2273,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks jobs' arguments" do
- jobs = Qless::Lua.new('jobs', @redis)
+ jobs = Qless::LuaScript.new('jobs', @redis)
[
# Providing keys
[['foo'], []],
@@ -2289,7 +2289,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks peek's arguments" do
- peek = Qless::Lua.new("peek", @redis)
+ peek = Qless::LuaScript.new("peek", @redis)
[
# Passing in no keys
[[], [1, 12345]],
@@ -2307,7 +2307,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks pop's arguments" do
- pop = Qless::Lua.new("pop", @redis)
+ pop = Qless::LuaScript.new("pop", @redis)
[
# Passing in no keys
[[], ["worker1", 1, 12345, 12346]],
@@ -2331,7 +2331,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks priority's arguments" do
- priority = Qless::Lua.new("pop", @redis)
+ priority = Qless::LuaScript.new("pop", @redis)
[
# Passing in keys
[['foo'], ['12345', 1]],
@@ -2345,7 +2345,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks put's arguments" do
- put = Qless::Lua.new("put", @redis)
+ put = Qless::LuaScript.new("put", @redis)
[
# Passing in no keys
[[], ["deadbeef", "{}", 12345]],
@@ -2375,7 +2375,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks queues' arguments" do
- queues = Qless::Lua.new("queues", @redis)
+ queues = Qless::LuaScript.new("queues", @redis)
[
# Passing in keys
[["foo"], [12345]],
@@ -2387,7 +2387,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks recur's arguments" do
- recur = Qless::Lua.new("recur", @redis)
+ recur = Qless::LuaScript.new("recur", @redis)
[
# Passing in keys
[['foo'], [12345]],
@@ -2422,7 +2422,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks retry's arguments" do
- rtry = Qless::Lua.new("queues", @redis)
+ rtry = Qless::LuaScript.new("queues", @redis)
[
# Passing in keys
[['foo'], ['12345', 'testing', 'worker', 12345, 0]],
@@ -2442,7 +2442,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks stats' arguments" do
- stats = Qless::Lua.new("stats", @redis)
+ stats = Qless::LuaScript.new("stats", @redis)
[
# Passing in keys
[["foo"], ["foo", "bar"]],
@@ -2454,7 +2454,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks tags' arguments" do
- tag = Qless::Lua.new("tag", @redis)
+ tag = Qless::LuaScript.new("tag", @redis)
[
# Passing in keys
[['foo'], ['add', '12345', 12345, 'foo']],
@@ -2481,7 +2481,7 @@ def should_only_have_tracked_jid_for(type)
end
it "checks track's arguments" do
- track = Qless::Lua.new("track", @redis)
+ track = Qless::LuaScript.new("track", @redis)
[
# Passing in keys
[["foo"], []],
55 spec/unit/lua_script_cache_spec.rb
View
@@ -0,0 +1,55 @@
+require 'spec_helper'
+require 'qless'
+require 'qless/lua_script_cache'
+
+module Qless
+ describe LuaScriptCache do
+ let(:redis_1a) { fire_double("Redis", id: "redis://foo:1234/1", script: "sha") }
+ let(:redis_1b) { fire_double("Redis", id: "redis://foo:1234/1", script: "sha") }
+ let(:redis_2) { fire_double("Redis", id: "redis://foo:1234/2", script: "sha") }
+ let(:cache) { LuaScriptCache.new }
+
+ before { File.stub(read: "script content") }
+
+ it 'returns different lua script objects when the script name is different' do
+ script_1 = cache.script_for("foo", redis_1a)
+ script_2 = cache.script_for("bar", redis_1a)
+
+ expect(script_1).to be_a(LuaScript)
+ expect(script_2).to be_a(LuaScript)
+
+ expect(script_1).not_to be(script_2)
+
+ expect(script_1.name).to eq("foo")
+ expect(script_2.name).to eq("bar")
+ end
+
+ it 'returns different lua script objects when the redis connection is to a different server' do
+ script_1 = cache.script_for("foo", redis_1a)
+ script_2 = cache.script_for("foo", redis_2)
+
+ expect(script_1).to be_a(LuaScript)
+ expect(script_2).to be_a(LuaScript)
+
+ expect(script_1).not_to be(script_2)
+
+ expect(script_1.redis).to be(redis_1a)
+ expect(script_2.redis).to be(redis_2)
+ end
+
+ it 'returns the same lua script object when the script name and redis connection are the same' do
+ script_1 = cache.script_for("foo", redis_1a)
+ script_2 = cache.script_for("foo", redis_1a)
+
+ expect(script_1).to be(script_2)
+ end
+
+ it 'returns the same lua script object when the script name and redis conneciton URL are the same' do
+ script_1 = cache.script_for("foo", redis_1a)
+ script_2 = cache.script_for("foo", redis_1b)
+
+ expect(script_1).to be(script_2)
+ end
+ end
+end
+
22 spec/unit/qless_spec.rb
View
@@ -18,12 +18,19 @@
end
end
+ describe ".lua_script_cache" do
+ it 'returns a memoized script cache instance' do
+ expect(Qless.lua_script_cache).to be_a(Qless::LuaScriptCache)
+ expect(Qless.lua_script_cache).to be(Qless.lua_script_cache)
+ end
+ end
+
context 'when instantiated' do
- let(:redis) { fire_double("Redis", info: { "redis_version" => "2.6.0" }) }
+ let(:redis) { fire_double("Redis", id: "redis://foo:1/1", info: { "redis_version" => "2.6.0" }) }
let(:redis_class) { fire_replaced_class_double("Redis") }
before do
- Qless::Lua.stub(:new) # so no scripts get loaded
+ Qless::LuaScript.stub(:new) # so no scripts get loaded
redis_class.stub(connect: redis)
end
@@ -48,6 +55,17 @@
client = Qless::Client.new(redis: redis)
client.redis.should be(redis)
end
+
+ it 'loads the lua scripts from the cache so that the scripts are not unnecessarily loaded multiple times' do
+ cache = fire_double("Qless::LuaScriptCache")
+ Qless.stub(lua_script_cache: cache)
+
+ loaded_scripts = []
+ cache.stub(:script_for) { |name, redis| loaded_scripts << name }
+
+ client = Qless::Client.new(redis: redis)
+ expect(loaded_scripts).to include("put", "complete")
+ end
end
end
Something went wrong with that request. Please try again.