namespacing the sidekiq connection

Sidekiq does not support `Proc` namespaces in its redis connection.

Therefore, we need to use (1) a `Redis::Namespace` wrapper class and
(2) a `Redis::DynamicNamespace` wrapper class, which allows Proc
namespaces by using `method_missing` to wrap the redis commands in a
namespace when calling the commands.

This is discussed here:
resque/redis-namespace#75

The `Redis::DynamicNamespace` wrapper is suggested here:
resque/redis-namespace#75 (comment)
resque/redis-namespace#75 (comment)

See also this stackoverflow question:
http://stackoverflow.com/q/40638628/2066546

Please note, this is just an intermediate step. Eventually, we will
move to separate redis server instances.

Internal trello cards:
https://trello.com/c/6xYBRb09/1075-redis-bottleneck
https://trello.com/c/cypH3vmW/896-multitenancy

Gemfile.lock

@@ -71,6 +71,7 @@ PATH
       rails-i18n
       rails-settings-cached (>= 0.6.5)
       redcarpet (>= 3.3.2)
+      redis-namespace
       redis-rails
       redis_analytics
       refile (~> 0.5.5)
@@ -459,16 +460,18 @@ GEM
     rdoc (4.2.0)
     redcarpet (3.3.4)
     redis (3.3.1)
-    redis-actionpack (5.0.0)
-      actionpack (>= 4.0.0, < 6)
-      redis-rack (~> 2.0.0.pre)
-      redis-store (~> 1.2.0.pre)
+    redis-actionpack (5.0.1)
+      actionpack (>= 4.0, < 6)
+      redis-rack (>= 1, < 3)
+      redis-store (>= 1.1.0, < 1.4.0)
     redis-activesupport (5.0.1)
       activesupport (>= 3, < 6)
       redis-store (~> 1.2.0)
-    redis-rack (2.0.0.pre)
-      rack (> 1.5, < 3)
-      redis-store (~> 1.2.0.pre)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
+    redis-rack (1.6.0)
+      rack (~> 1.5)
+      redis-store (~> 1.2.0)
     redis-rails (5.0.1)
       redis-actionpack (~> 5.0.0)
       redis-activesupport (~> 5.0.0)

config/initializers/redis.rb

@@ -11,6 +11,34 @@
 ENV['REDIS_HOST'] ||= 'redis' if (Resolv.getaddress "redis" rescue false)
 ENV['REDIS_HOST'] ||= 'localhost'
 
+class Redis
+  class DynamicNamespace < BasicObject
+    # Uses the given block to generate a redis namespace dynamically
+    # @param options [Hash{Symbol=>Object}]
+    # @option options [Redis] :redis (Redis.current)
+    def initialize(namespace_proc, options = {})
+      @namespace_proc = namespace_proc
+      @options = options.dup
+    end
+
+    # @return [String]
+    def current_namespace
+      @namespace_proc.call
+    end
+
+    # @api private
+    def method_missing(*a,&b)
+      Namespace.new(current_namespace, @options).public_send(*a,&b)
+    end
+  end
+
+  # monkey-patch Redis::Namespace to make it think that
+  # DynamicNamespace is one of it
+  def Namespace.===(other)
+    super or other.kind_of?(DynamicNamespace)
+  end
+end
+
 # Use this class to provide a configuration for a `:redis_store`:
 #
 # Where you normally write
@@ -26,6 +54,20 @@
 # This way, we can collect the common configuration options in the
 # class below.
 #
+# Where you normally write
+#
+#     configuration.redis_connection = Redis.new({...})
+#
+# you may write
+#
+#     configuration.redis_connection =
+#       RedisConnectionConfiguration.new(:redis_analytics).to_redis
+#
+# And if you want to apply the dynamic namespace automatically, write:
+#
+#     configuration.redis_connection =
+#       RedisConnectionConfiguration.new(:redis_analytics).to_namespaced_redis
+#
 class RedisConnectionConfiguration
 
   # Arguments:
@@ -59,5 +101,13 @@ def namespace
   def to_hash
     default_options.merge(@options)
   end
+
+  def to_redis
+    Redis.new self.to_hash
+  end
+
+  def to_namespaced_redis
+    Redis::DynamicNamespace.new(namespace, redis: to_redis)
+  end
 end
 

config/initializers/sidekiq.rb

@@ -13,10 +13,15 @@
 
 # http://stackoverflow.com/questions/14825565/sidekiq-deploy-to-multiple-environments
 #
+# Sidekiq does not support Proc namespacing. Thus, we have to use our own
+# `Redis` object and wrap it with a redis namespace class.
+#
+# https://github.com/mperham/sidekiq/wiki/Using-Redis#complete-control
+# http://stackoverflow.com/q/40638628/2066546
+#
 Sidekiq.configure_server do |config|
-  config.redis = {host: ENV['REDIS_HOST'], port: '6379', namespace: "#{::STAGE}_sidekiq", timeout: 15.0 }
+  config.redis = ConnectionPool.new(size: 25) { RedisConnectionConfiguration.new(:sidekiq).to_namespaced_redis }
 end
-
 Sidekiq.configure_client do |config|
-  config.redis = {host: ENV['REDIS_HOST'], port: '6379', namespace: "#{::STAGE}_sidekiq", timeout: 15.0 }
-end
+  config.redis = ConnectionPool.new(size: 5) { RedisConnectionConfiguration.new(:sidekiq).to_namespaced_redis }
+end

demo_app/my_platform/Gemfile.lock

@@ -71,6 +71,7 @@ PATH
       rails-i18n
       rails-settings-cached (>= 0.6.5)
       redcarpet (>= 3.3.2)
+      redis-namespace
       redis-rails
       redis_analytics
       refile (~> 0.5.5)
@@ -204,7 +205,7 @@ GEM
     coffee-script-source (1.10.0)
     colored (1.2)
     concurrent-ruby (1.0.2)
-    connection_pool (2.2.0)
+    connection_pool (2.2.1)
     coveralls (0.8.10)
       json (~> 1.8)
       rest-client (>= 1.6.8, < 2)
@@ -458,17 +459,19 @@ GEM
       ffi (>= 0.5.0)
     rdoc (4.2.0)
     redcarpet (3.3.4)
-    redis (3.3.1)
-    redis-actionpack (5.0.0)
-      actionpack (>= 4.0.0, < 6)
-      redis-rack (~> 2.0.0.pre)
-      redis-store (~> 1.2.0.pre)
+    redis (3.3.2)
+    redis-actionpack (5.0.1)
+      actionpack (>= 4.0, < 6)
+      redis-rack (>= 1, < 3)
+      redis-store (>= 1.1.0, < 1.4.0)
     redis-activesupport (5.0.1)
       activesupport (>= 3, < 6)
       redis-store (~> 1.2.0)
-    redis-rack (2.0.0.pre)
-      rack (> 1.5, < 3)
-      redis-store (~> 1.2.0.pre)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
+    redis-rack (1.6.0)
+      rack (~> 1.5)
+      redis-store (~> 1.2.0)
     redis-rails (5.0.1)
       redis-actionpack (~> 5.0.0)
       redis-activesupport (~> 5.0.0)

lib/your_platform/engine.rb

@@ -18,6 +18,7 @@
 
 # Caching
 require 'redis-rails'
+require 'redis-namespace'
 
 # Workers
 require 'sidekiq'

your_platform.gemspec

@@ -63,6 +63,7 @@ Gem::Specification.new do |s|
 
   # Caching
   s.add_dependency 'redis-rails'
+  s.add_dependency 'redis-namespace'
 
   # Workers
   s.add_dependency 'foreman'