Add latency to sidekiqload

Gemfile

@@ -4,6 +4,7 @@ gemspec
 gem 'rails', '~> 4.2'
 gem 'simplecov'
 gem 'minitest'
+gem 'toxiproxy'
 
 platforms :rbx do
   gem 'rubysl', '~> 2.0'         # if using anything in the ruby standard library

bin/sidekiqload

@@ -14,12 +14,11 @@ Celluloid.logger = nil
 include Sidekiq::Util
 
 Sidekiq.configure_server do |config|
-  config.redis = { db: 13 }
+  config.redis = { db: 13, port: 6380 }
   config.options[:queues] << 'default'
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
 end
-Sidekiq.redis {|c| c.flushdb}
 
 class LoadWorker
   include Sidekiq::Worker
@@ -29,10 +28,22 @@ class LoadWorker
   end
 
   def perform(idx)
-    raise idx.to_s if idx % 100 == 1
+    #raise idx.to_s if idx % 100 == 1
   end
 end
 
+# brew tap shopify/shopify
+# brew install toxiproxy
+# gem install toxiproxy
+require 'toxiproxy'
+# simulate a non-localhost network for realer-world conditions.
+# adding 1ms of network latency has an ENORMOUS impact on benchmarks
+Toxiproxy.populate([{
+    "name": "redis",
+    "listen": "127.0.0.1:6380",
+    "upstream": "127.0.0.1:6379"
+}])
+
 self_read, self_write = IO.pipe
 %w(INT TERM USR1 USR2 TTIN).each do |sig|
   begin
@@ -44,6 +55,7 @@ self_read, self_write = IO.pipe
   end
 end
 
+Sidekiq.redis {|c| c.flushdb}
 def handle_signal(launcher, sig)
   Sidekiq.logger.debug "Got #{sig} signal"
   case sig
@@ -82,20 +94,16 @@ end
 iter = 10
 count = 10_000
 
-Client = Thread.new do
-  watchdog("client thread") do
-    iter.times do
-      arr = Array.new(count) do
-        []
-      end
-      count.times do |idx|
-        arr[idx][0] = idx
-      end
-      Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
-    end
-    Sidekiq.logger.error "Created #{count*iter} jobs"
+iter.times do
+  arr = Array.new(count) do
+    []
   end
+  count.times do |idx|
+    arr[idx][0] = idx
+  end
+  Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
 end
+Sidekiq.logger.error "Created #{count*iter} jobs"
 
 Monitoring = Thread.new do
   watchdog("monitor thread") do
@@ -108,26 +116,34 @@ Monitoring = Thread.new do
         end
       end.map(&:to_i)
       total = qsize + retries
-      GC.start
+      #GC.start
       Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
       if total == 0
         Sidekiq.logger.error("Done")
-        Celluloid.shutdown
         exit(0)
       end
     end
   end
 end
 
 begin
-  launcher = Sidekiq::Launcher.new(Sidekiq.options)
-  launcher.run
+  #RubyProf::exclude_threads = [ Monitoring ]
+  #RubyProf.start
+  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+    launcher = Sidekiq::Launcher.new(Sidekiq.options)
+    launcher.run
 
-  while readable_io = IO.select([self_read])
-    signal = readable_io.first[0].gets.strip
-    handle_signal(launcher, signal)
+    while readable_io = IO.select([self_read])
+      signal = readable_io.first[0].gets.strip
+      handle_signal(launcher, signal)
+    end
   end
 rescue SystemExit => e
+  #Sidekiq.logger.error("Profiling...")
+  #result = RubyProf.stop
+  #printer = RubyProf::GraphHtmlPrinter.new(result)
+  #printer.print(File.new("output.html", "w"), :min_percent => 1)
   # normal
 rescue => e
   raise e if $DEBUG