Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Added the Redis Promises proxy + specs

  • Loading branch information...
commit 9d0d3a0bfaf507e92ca8d8f1fcf8095d6b0c8270 1 parent 4a3a78b
Mathieu Ravaux authored
1  lib/redis-auto-batches.rb
... ... @@ -1,4 +1,5 @@
1 1 require "redis-auto-batches/promise"
  2 +require "redis-auto-batches/redis_promise_proxy"
2 3 require "redis-auto-batches/version"
3 4
4 5 module RedisAutoBatches
173 lib/redis-auto-batches/redis_promise_proxy.rb
... ... @@ -0,0 +1,173 @@
  1 +# TODO: think about and manage the use of the multi / exec commands, so that
  2 +# we have a transparent behavior between a multi and an exec command.
  3 +
  4 +module RedisAutoBatches
  5 + class RedisPromiseProxy
  6 + class PromiseWithIndex < RedisAutoBatches::Promise
  7 + attr_accessor :promise_index
  8 + def initialize(index, &block)
  9 + promise_index = index
  10 + super(&block)
  11 + end
  12 +
  13 + # def respond_to?(method)
  14 + # method.equal?(:promise_index) || super(method)
  15 + # end
  16 + end
  17 +
  18 + MAX_BUFFERED_PROMISES = 1_000
  19 +
  20 + READ_COMMANDS = ::Set.new(%w[
  21 + ttl sort
  22 + randomkey keys srandmember type
  23 + get mget mapped_mget [] []=
  24 + exists hexists
  25 + hget hmget hkeys hgetall hvals hlen
  26 + lindex llen lrange
  27 + zscore zcard zcount zrange zrank zrangebyscore zrevrange zrevrangebyscore zrevrank
  28 + smembers sismember sdiff sunion sinter scard
  29 + dbsize debug
  30 + ].map(&:to_sym))
  31 +
  32 +
  33 + WRITE_COMMANDS = ::Set.new(%w[
  34 + incr decr incrby decrby hincrby zincrby
  35 + del expireat getset hdel hmset hset hsetnx info lastsave lpop lpush lrem lset ltrim
  36 + mapped_hmset move mset msetnx rename
  37 + renamenx rpop rpoplpush rpush sadd sdiffstore select set setnx
  38 + sinterstore smove spop srem subscribe sunionstore zadd
  39 + zinterstore zrem zremrangebyrank zremrangebyscore zunionstore
  40 + ].map(&:to_sym))
  41 +
  42 + IMMEDIATE_COMMANDS = Set.new(%w[
  43 + auth bgrewtriteaof bgsave blpop brpop brpoplpush config debug flushall flushdb monitor
  44 + persist expire setex psubscribe publish punsubscribe quit save shutdown slaveof unwatch watch
  45 + ].map(&:to_sym))
  46 +
  47 + attr_accessor :redis
  48 + attr_accessor :buffered_promises
  49 + attr_accessor :in_unit_of_work
  50 +
  51 + ##
  52 + # Creates a new proxy that will be usable just like a normal
  53 + # Redis object, but will buffer Redis commands in transactions
  54 + # to minimize the number of necessary round-trips
  55 + #
  56 + # @example
  57 + # redis = Statistics::RedisPromiseProxy.new(Redis.new)
  58 + # user_ids = [<user_1>, <user_2>, <user_3>, <user_4>, ]
  59 + # friend_counts = user_ids.map { |id| redis.scard("users:#{id}:friendships") }
  60 + # friend_counts.count # => 4
  61 + # friend_counts[0] # => 174 # the Redis request is made right here
  62 + #
  63 + # @param Redis
  64 + # @see Redis.new
  65 + def initialize(redis)
  66 + @redis = redis
  67 + @queue_mutex = ::Mutex.new
  68 + @flush_mutex = ::Mutex.new
  69 + @buffered_promises = []
  70 + end
  71 +
  72 +
  73 + ##
  74 + # if _command_ is a read command, return a promise of the result of calling this command
  75 + # else, realize the buffered promises and then call the command, directly returning its result
  76 + # TODO: maybe we should wrap the result of the command in a promise in any case
  77 + def method_missing(command, *args, &block)
  78 + promise_redis_result(command, *args, &block)
  79 + end
  80 +
  81 + def promise_redis_result(command, *args, &block)
  82 + @queue_mutex.synchronize do
  83 + if IMMEDIATE_COMMANDS.include?(command)
  84 + # puts "Will flush non-read command called on Redis (#{command.inspect})."
  85 + flush
  86 + @redis.send(command, *args, &block)
  87 + else
  88 + if outside_unit_of_work?
  89 + # if Rails.env.test?
  90 + # # unless caller.any? { |line| line.include?('/rspec/core/') && line.include?('run_before_each') }
  91 + # # puts "\t#{caller.join("\n\t")}"
  92 + # raise RuntimeError.new("Accessing to redis outside a unit of work")
  93 + # # end
  94 + # end
  95 + # puts "Forcing the creation of a unit of work..."
  96 + start_unit_of_work
  97 + end
  98 +
  99 + #delay with a promise
  100 + raise NotImplementedError.new("Passing a block when calling a Redis command is not currently supported.") if block_given?
  101 + index = @buffered_promises.count
  102 +
  103 + promise = PromiseWithIndex.new(index) do
  104 + # puts "Will flush since value accessed for promise ##{index} (#{command}, #{args.inspect})"
  105 + flush
  106 + promise.force
  107 + end
  108 + @buffered_promises << [promise, command, args]
  109 + flush if flush_needed?
  110 + promise
  111 + end
  112 + end
  113 + end
  114 +
  115 + #TODO: add this to monitor_with_new_relic and log the executions times
  116 + def flush
  117 + @flush_mutex.synchronize do
  118 + return if @buffered_promises.empty?
  119 +
  120 + results = if @buffered_promises.length == 1
  121 + (prom, command, args) = @buffered_promises.first
  122 + # puts "command: #{command}(#{args.inspect})"
  123 + res = @redis.send(command, *args)
  124 + [res]
  125 + else
  126 + @redis.multi
  127 + @buffered_promises.each do |promise, command, args|
  128 + @redis.send(command, *args)
  129 + end
  130 + @redis.exec
  131 + end
  132 +
  133 + results.each.with_index do |result, index|
  134 + # puts " ==> #{result}, #{index}"
  135 + @buffered_promises[index][0].fulfill(result)
  136 + end
  137 +
  138 + @buffered_promises = []
  139 + end
  140 + end
  141 +
  142 + # Redis.unit_of_work method like mongoid that will flush the previous unit_of_work and ensure the termination of the current unit_of_work
  143 + def unit_of_work
  144 + previous_state = @in_unit_of_work
  145 + begin
  146 + start_unit_of_work
  147 + yield if block_given?
  148 + ensure
  149 + end_unit_of_work(previous_state)
  150 + end
  151 + end
  152 +
  153 + def start_unit_of_work
  154 + flush
  155 + @in_unit_of_work = true
  156 + end
  157 +
  158 + def end_unit_of_work(previous_state)
  159 + flush
  160 + @in_unit_of_work = previous_state
  161 + end
  162 +
  163 + def outside_unit_of_work?
  164 + ! @in_unit_of_work
  165 + end
  166 +
  167 + # we flush every 1000 promises (at least)
  168 + def flush_needed?
  169 + @buffered_promises.length >= MAX_BUFFERED_PROMISES
  170 + end
  171 +
  172 + end
  173 +end
63 spec/acceptance_spec.rb
... ... @@ -0,0 +1,63 @@
  1 +require 'spec_helper'
  2 +
  3 +describe 'Acceptance criteria' do
  4 + let(:redis) { Redis.connect }
  5 + subject { RedisAutoBatches::RedisPromiseProxy.new(redis) }
  6 +
  7 + before do
  8 + redis.set("key1", 10)
  9 + redis.set("key2", 20)
  10 + redis.set("key3", 30)
  11 + end
  12 +
  13 + include RedisMonitoring
  14 +
  15 + context "when used inside a unit of work" do
  16 + context "when executing several read Redis commands" do
  17 + it "does only one round-trip to Redis" do
  18 + subject.unit_of_work do
  19 + [ subject.get("key1"),
  20 + subject.get("key2"),
  21 + subject.get("key3")
  22 + ]
  23 + end
  24 +
  25 + actual_redis_commands.should == [
  26 + 'multi',
  27 + 'get key1',
  28 + 'get key2',
  29 + 'get key3',
  30 + 'exec'
  31 + ]
  32 + end
  33 + end
  34 +
  35 + context "when issuing read commands after write commands" do
  36 + def issue_operations
  37 + subject.unit_of_work do
  38 + [
  39 + subject.get("key1"),
  40 + subject.set("key1", "1000"),
  41 + subject.get("key1")
  42 + ]
  43 + end
  44 + end
  45 +
  46 + it "still does only one round-trip to redis" do
  47 + pending "Threading issues"
  48 + issue_operations
  49 +
  50 + commands = actual_redis_commands
  51 + # puts commands.inspect
  52 + nb_round_trips_to_redis(commands).should == 1
  53 + end
  54 +
  55 + it "reads the newly written value, as expected" do
  56 + values = issue_operations
  57 + values.should == %w(10 OK 1000)
  58 + end
  59 + end
  60 +
  61 +
  62 + end
  63 +end
191 spec/redis_promise_proxy_spec.rb
... ... @@ -0,0 +1,191 @@
  1 +require 'spec_helper'
  2 +
  3 +describe RedisAutoBatches::RedisPromiseProxy do
  4 + let(:redis) { stub(:redis) }
  5 + subject { RedisAutoBatches::RedisPromiseProxy.new(redis) }
  6 +
  7 + # around(:each) do |example|
  8 + # subject.unit_of_work do
  9 + # example.run
  10 + # redis.should_receive(:multi)
  11 + # end
  12 + # end
  13 +
  14 + def quacks_like_a_promise?(object)
  15 + object.respond_to?(:__force__) &&
  16 + object.respond_to?(:__chain__) &&
  17 + object.respond_to?(:__pending__?) &&
  18 + object.respond_to?(:__fulfilled__?) &&
  19 + object.respond_to?(:__failed__?)
  20 + end
  21 +
  22 + let(:keys) { %w[ key_1 key_2 key_3 key_4 key_5 ] }
  23 +
  24 + describe "#get" do
  25 + it "returns a promise" do
  26 + subject.stub(:outside_unit_of_work?, false)
  27 + quacks_like_a_promise?(subject.get("key_1")).should be_true
  28 + end
  29 +
  30 + context "when the result is used" do
  31 + it "hits redis" do
  32 + subject.unit_of_work do
  33 + redis.should_receive(:get).with("key_1").and_return("fine")
  34 + result = subject.get("key_1")
  35 + result.length.should == 4
  36 + end
  37 + end
  38 + end
  39 +
  40 + context "when the result isn't used" do
  41 + it "doesn't hit redis" do
  42 + subject.stub(:outside_unit_of_work?, false)
  43 + redis.should_not_receive(:get)
  44 + subject.get("key_1")
  45 + end
  46 + end
  47 +
  48 + end
  49 +
  50 + describe "succession of 5 gets" do
  51 + it "returns 5 promises and fulfill it" do
  52 + subject.unit_of_work do
  53 + keys.each do |key|
  54 + quacks_like_a_promise?(subject.get(key)).should be_true
  55 + end
  56 + redis.should_receive(:multi).exactly(1).times.and_return(nil)
  57 + redis.should_receive(:get).exactly(5).times.and_return(nil)
  58 + redis.should_receive(:exec).and_return(['1', '2', '3', '4', '5'])
  59 + end
  60 + end
  61 +
  62 + context "when the result is not used" do
  63 + it "doesn't hit redis" do
  64 + subject.stub(:outside_unit_of_work?, false)
  65 + redis.should_not_receive(:multi)
  66 + redis.should_not_receive(:get)
  67 + redis.should_not_receive(:exec)
  68 + keys.each { |key| subject.get(key) }
  69 + end
  70 + end
  71 +
  72 + context "when the result of one of the promises is used" do
  73 + before do
  74 + redis.should_receive(:multi).exactly(1).times.and_return(nil)
  75 + redis.should_receive(:get).exactly(5).times.and_return(nil)
  76 + redis.should_receive(:exec).and_return(['1', '2', '3', '4', '5'])
  77 + end
  78 +
  79 + it "hits redis in a transaction" do
  80 + subject.unit_of_work do
  81 + results = keys.map { |key| subject.get(key) }
  82 + results[3] + results[4]
  83 + end
  84 + end
  85 +
  86 + it "fulfills each promise with the respective correct value" do
  87 + subject.unit_of_work do
  88 + results = keys.map { |key| subject.get(key) }
  89 + results.should == ['1', '2', '3', '4', '5']
  90 + end
  91 + end
  92 + end
  93 +
  94 + end
  95 +
  96 + # get and set now behaves exactly the same
  97 + # describe "#set" do
  98 + # it "hits redis immediately" do
  99 + # redis.should_not_receive(:multi)
  100 + # redis.should_not_receive(:exec)
  101 + # redis.should_receive(:set).with("key_1", "value_1").and_return('1')
  102 + # subject.set('key_1', 'value_1')
  103 + # end
  104 + #
  105 + # it "flushes pending buffered reads" do
  106 + # redis.should_receive(:multi).exactly(1).times.and_return(nil)
  107 + # redis.should_receive(:get).exactly(5).times.and_return(nil)
  108 + # redis.should_receive(:exec).and_return(['1', '2', '3', '4', '5'])
  109 + # redis.should_receive(:set).with("key_1", "value_1").and_return('1')
  110 + # keys.each { |key| subject.get(key) }
  111 + # subject.set('key_1', 'value_1')
  112 + # end
  113 + #
  114 + # end
  115 +
  116 + context "repeated usage" do
  117 + it "reinitializes correctly its data structures" do
  118 + redis.should_receive(:multi).exactly(2).times.and_return(nil)
  119 + redis.should_receive(:get).exactly(5).times.and_return(nil)
  120 + redis.should_receive(:exec).and_return(['1', '2', '3'], ['4', '5'])
  121 + subject.unit_of_work do
  122 + one, two, three = subject.get('key_1'), subject.get('key_2'), subject.get('key_3')
  123 + one.pending?.should be_true
  124 + two.pending?.should be_true
  125 + three.pending?.should be_true
  126 +
  127 + three.to_i.should == 3
  128 + one.fulfilled?.should be_true
  129 + two.fulfilled?.should be_true
  130 + three.fulfilled?.should be_true
  131 +
  132 + four, five = subject.get('key_4'), subject.get('key_5')
  133 + four.pending?.should be_true
  134 + five.pending?.should be_true
  135 + four.to_i.should == 4
  136 + five.fulfilled?.should be_true
  137 + five.to_i.should == 5
  138 + end
  139 +
  140 + end
  141 + end
  142 +
  143 + context "with chaining of operations on the promise" do
  144 + it "stays lazy" do
  145 + subject.stub(:outside_unit_of_work?, false)
  146 + redis.should_not_receive(:multi)
  147 + redis.should_not_receive(:get)
  148 + subject.get('key_1').chain {|v| v.to_i }
  149 + end
  150 +
  151 + it "applies the computation in the expected way" do
  152 + redis.should_receive(:get).and_return('12')
  153 + subject.unit_of_work do
  154 + subject.get('key_1').chain {|v| v.to_i }.should == 12
  155 + end
  156 +
  157 + end
  158 +
  159 + end
  160 +
  161 + describe "#unit_of_work" do
  162 + it "flush everything on the start of a new unit of work and restore correctly after" do
  163 + subject.unit_of_work do
  164 + subject.get('key_1')
  165 + redis.should_receive(:get).with("key_1").and_return("fine")
  166 + subject.unit_of_work do
  167 + subject.get('key_2')
  168 + redis.should_receive(:get).with("key_2").and_return("fine")
  169 + end
  170 + subject.get('key_3')
  171 + redis.should_receive(:get).with("key_3").and_return("fine")
  172 + end
  173 + end
  174 + it "execute immediately immediate commands (and flush waiting commands before)" do
  175 + subject.unit_of_work do
  176 + redis.should_receive(:expire).with("key_1", 14)
  177 + subject.expire('key_1', 14)
  178 + end
  179 + end
  180 + it "flush after MAX_BUFFERED_PROMISES" do
  181 + subject.unit_of_work do
  182 + redis.should_receive(:multi)
  183 + redis.should_receive(:get).with("key_1").exactly(RedisAutoBatches::RedisPromiseProxy::MAX_BUFFERED_PROMISES).times.and_return(nil)
  184 + redis.should_receive(:exec).and_return(["fine"])
  185 + RedisAutoBatches::RedisPromiseProxy::MAX_BUFFERED_PROMISES.times { subject.get('key_1') }
  186 + subject.get('key_1')
  187 + redis.should_receive(:get).with("key_1").exactly(1).times.and_return("fine")
  188 + end
  189 + end
  190 + end
  191 +end
80 spec/support/redis_monitoring.rb
... ... @@ -0,0 +1,80 @@
  1 +module RedisMonitoring
  2 +
  3 + def self.included(base)
  4 + base.class_eval do
  5 + before do
  6 + @called_commands = []
  7 + start_redis_monitoring
  8 + end
  9 +
  10 + after do
  11 + @verif_thread.kill
  12 + end
  13 + end
  14 + end
  15 +
  16 + def start_redis_monitoring
  17 + mutex_for_redis_verification_startup = Mutex.new
  18 +
  19 + @verif_thread = Thread.new do
  20 + mutex_for_redis_verification_startup.lock
  21 + Redis.connect.monitor do |command|
  22 + if command.include? 'monitor'
  23 + mutex_for_redis_verification_startup.unlock
  24 + end
  25 +
  26 + if command.include? 'ping'
  27 + # puts "Killing verif thread"
  28 + Thread.current.kill
  29 + end
  30 +
  31 + unless /"monitor"|OK|ping$/ =~ command
  32 + command_without_timestamp = command.gsub(/^\d+\.\d+ /, '').gsub('"', '')
  33 + @called_commands << command_without_timestamp
  34 + end
  35 + end
  36 + end
  37 +
  38 + # let the verif thread acquire the mutex and wait on it while the verif starts up
  39 + @verif_thread.run
  40 + mutex_for_redis_verification_startup.lock
  41 + # puts "Rocknroll !"
  42 + end
  43 +
  44 +
  45 +
  46 + def nb_round_trips_to_redis(commands)
  47 + # puts "\n\nCounting nb_round_trips_to_redis"
  48 + round_trips = 0
  49 + in_a_transaction = false
  50 +
  51 + # puts commands.inspect
  52 + commands.each do |command|
  53 + # puts command.inspect
  54 + if command == 'multi'
  55 + round_trips += 1
  56 + in_a_transaction = true
  57 + elsif command == 'exec'
  58 + in_a_transaction = false
  59 + elsif !in_a_transaction
  60 + round_trips += 1
  61 + end
  62 + end
  63 +
  64 + # puts "Counted #{round_trips} round_trips !\n\n\n"
  65 + round_trips
  66 + end
  67 +
  68 +
  69 + def actual_redis_commands
  70 + # wait for the redis monitor thread to receive feedback on the sent commands
  71 + # puts "sending a ping ! #{@called_commands.inspect}"
  72 +
  73 + redis.ping
  74 +
  75 + @verif_thread.join
  76 +
  77 + @called_commands
  78 + end
  79 +
  80 +end

0 comments on commit 9d0d3a0

Please sign in to comment.
Something went wrong with that request. Please try again.