Merge pull request #43 from igor-pavlenko/master

Moved Redis connection to the class & horizontal partitioning
martinrusev · Feb 27, 2017 · 8b19cf8 · 8b19cf8
2 parents 6248e59 + 996b97b
commit 8b19cf8
Show file tree

Hide file tree

Showing 5 changed files with 250 additions and 66 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -2,6 +2,7 @@ language: python
 services:
   - redis-server
 python:
+  - "3.6"
   - "3.5"
   - "3.4"
   - "2.7"
@@ -12,13 +13,15 @@ env:
   - DJANGO="django>=1.5"
 matrix:
   exclude:
+    - python: "3.6"
+      env: DJANGO="django>=1.4,<1.5"
     - python: "3.5"
       env: DJANGO="django>=1.4,<1.5"
     - python: "3.4"
       env: DJANGO="django>=1.4,<1.5"
     - python: "2.6"
       env: DJANGO="django>=1.5"
 install:
-  - pip install $DJANGO --use-mirrors
-  - pip install -q redis --use-mirrors
+  - pip install $DJANGO
+  - pip install -q redis
 script: python setup.py nosetests
diff --git a/README.md b/README.md
@@ -39,6 +39,41 @@ SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH = '/var/run/redis/redis.sock'
 SESSION_REDIS_SENTINEL_LIST = [(host, port), (host, port), (host, port)]
 SESSION_REDIS_SENTINEL_MASTER_ALIAS = 'sentinel-master'
 
+# Redis Pool (Horizontal partitioning)
+# Divide sessions between Redis instances based on session key.
+# You can choose connection type for each Redis from pool (host/port, unix socket, redis url). 
+SESSION_REDIS_PREFIX = 'session'
+SESSION_REDIS_SOCKET_TIMEOUT = 1
+SESSION_REDIS_RETRY_ON_TIMEOUT = False
+SESSION_REDIS_POOL = [
+    {
+        'SESSION_REDIS_HOST': 'localhost3',
+        'SESSION_REDIS_PORT': 6379,
+        'SESSION_REDIS_DB': 0,
+        'SESSION_REDIS_PASSWORD': None,
+        'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH': None,
+        'SESSION_REDIS_URL': None,
+        'SESSION_REDIS_WEIGHT': 1,
+    },
+    {
+        'SESSION_REDIS_HOST': 'localhost2',
+        'SESSION_REDIS_PORT': 6379,
+        'SESSION_REDIS_DB': 0,
+        'SESSION_REDIS_PASSWORD': None,
+        'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH': None,
+        'SESSION_REDIS_URL': None,
+        'SESSION_REDIS_WEIGHT': 1,
+    },
+    {
+        'SESSION_REDIS_HOST': 'localhost1',
+        'SESSION_REDIS_PORT': 6379,
+        'SESSION_REDIS_DB': 0,
+        'SESSION_REDIS_PASSWORD': None,
+        'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH': None,
+        'SESSION_REDIS_URL': None,
+        'SESSION_REDIS_WEIGHT': 1,
+    },
+]
 ```
 
 

diff --git a/redis_sessions/session.py b/redis_sessions/session.py
@@ -8,44 +8,94 @@
 from redis_sessions import settings
 
 
-# Avoid new redis connection on each request
-
-if settings.SESSION_REDIS_SENTINEL_LIST is not None:
-    from redis.sentinel import Sentinel
-
-    redis_server = Sentinel(
-        settings.SESSION_REDIS_SENTINEL_LIST,
-        socket_timeout=settings.SESSION_REDIS_SOCKET_TIMEOUT,
-        retry_on_timeout=settings.SESSION_REDIS_RETRY_ON_TIMEOUT,
-        db=getattr(settings, 'SESSION_REDIS_DB', 0),
-        password=getattr(settings, 'SESSION_REDIS_PASSWORD', None)
-    ).master_for(settings.SESSION_REDIS_SENTINEL_MASTER_ALIAS)
-
-elif settings.SESSION_REDIS_URL is not None:
-
-    redis_server = redis.StrictRedis.from_url(
-        settings.SESSION_REDIS_URL,
-        socket_timeout=settings.SESSION_REDIS_SOCKET_TIMEOUT
-    )
-elif settings.SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH is None:
-
-    redis_server = redis.StrictRedis(
-        host=settings.SESSION_REDIS_HOST,
-        port=settings.SESSION_REDIS_PORT,
-        socket_timeout=settings.SESSION_REDIS_SOCKET_TIMEOUT,
-        retry_on_timeout=settings.SESSION_REDIS_RETRY_ON_TIMEOUT,
-        db=settings.SESSION_REDIS_DB,
-        password=settings.SESSION_REDIS_PASSWORD
-    )
-else:
-
-    redis_server = redis.StrictRedis(
-        unix_socket_path=settings.SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH,
-        socket_timeout=settings.SESSION_REDIS_SOCKET_TIMEOUT,
-        retry_on_timeout=settings.SESSION_REDIS_RETRY_ON_TIMEOUT,
-        db=settings.SESSION_REDIS_DB,
-        password=settings.SESSION_REDIS_PASSWORD,
-    )
+class RedisServer():
+    __redis = {}
+
+    def __init__(self, session_key):
+        self.session_key = session_key
+        self.connection_key = ''
+
+        if settings.SESSION_REDIS_SENTINEL_LIST is not None:
+            self.connection_type = 'sentinel'
+        else:
+            if settings.SESSION_REDIS_POOL is not None:
+                server_key, server = self.get_server(session_key, settings.SESSION_REDIS_POOL)
+                self.connection_key = str(server_key)
+                settings.SESSION_REDIS_HOST = getattr(server, 'SESSION_REDIS_HOST', 'localhost')
+                settings.SESSION_REDIS_PORT = getattr(server, 'SESSION_REDIS_PORT', 6379)
+                settings.SESSION_REDIS_DB = getattr(server, 'SESSION_REDIS_DB', 0)
+                settings.SESSION_REDIS_PASSWORD = getattr(server, 'SESSION_REDIS_PASSWORD', None)
+                settings.SESSION_REDIS_URL = getattr(server, 'SESSION_REDIS_URL', None)
+                settings.SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH = getattr(server,
+                                                                         'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', None)
+
+            if settings.SESSION_REDIS_URL is not None:
+                self.connection_type = 'redis_url'
+            elif settings.SESSION_REDIS_HOST is not None:
+                self.connection_type = 'redis_host'
+            elif settings.SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH is not None:
+                self.connection_type = 'redis_unix_url'
+
+        self.connection_key += self.connection_type
+
+    def get_server(self, key, servers_pool):
+        total_weight = sum([row.get('SESSION_REDIS_WEIGHT', 1) for row in servers_pool])
+        pos = 0
+        for i in range(3, -1, -1):
+            pos = pos * 2 ** 8 + ord(key[i])
+        pos = pos % total_weight
+
+        pool = iter(servers_pool)
+        server = next(pool)
+        server_key = 0
+        i = 0
+        while i < total_weight:
+            if i <= pos < (i + server.get('SESSION_REDIS_WEIGHT', 1)):
+                return server_key, server
+            i += server.get('SESSION_REDIS_WEIGHT', 1)
+            server = next(pool)
+            server_key += 1
+
+        return
+
+    def get(self):
+        if self.connection_key in self.__redis:
+            return self.__redis[self.connection_key]
+
+        if self.connection_type == 'sentinel':
+            from redis.sentinel import Sentinel
+            self.__redis[self.connection_key] = Sentinel(
+                settings.SESSION_REDIS_SENTINEL_LIST,
+                socket_timeout=settings.SESSION_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=settings.SESSION_REDIS_RETRY_ON_TIMEOUT,
+                db=getattr(settings, 'SESSION_REDIS_DB', 0),
+                password=getattr(settings, 'SESSION_REDIS_PASSWORD', None)
+            ).master_for(settings.SESSION_REDIS_SENTINEL_MASTER_ALIAS)
+
+        elif self.connection_type == 'redis_url':
+            self.__redis[self.connection_key] = redis.StrictRedis.from_url(
+                settings.SESSION_REDIS_URL,
+                socket_timeout=settings.SESSION_REDIS_SOCKET_TIMEOUT
+            )
+        elif self.connection_type == 'unix_url':
+            self.__redis[self.connection_key] = redis.StrictRedis(
+                host=settings.SESSION_REDIS_HOST,
+                port=settings.SESSION_REDIS_PORT,
+                socket_timeout=settings.SESSION_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=settings.SESSION_REDIS_RETRY_ON_TIMEOUT,
+                db=settings.SESSION_REDIS_DB,
+                password=settings.SESSION_REDIS_PASSWORD
+            )
+        else:
+            self.__redis[self.connection_key] = redis.StrictRedis(
+                unix_socket_path=settings.SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH,
+                socket_timeout=settings.SESSION_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=settings.SESSION_REDIS_RETRY_ON_TIMEOUT,
+                db=settings.SESSION_REDIS_DB,
+                password=settings.SESSION_REDIS_PASSWORD,
+            )
+
+        return self.__redis[self.connection_key]
 
 
 class SessionStore(SessionBase):
@@ -54,8 +104,7 @@ class SessionStore(SessionBase):
     """
     def __init__(self, session_key=None):
         super(SessionStore, self).__init__(session_key)
-
-        self.server = redis_server
+        self.server = RedisServer(session_key).get()
 
     def load(self):
         try:

diff --git a/redis_sessions/settings.py b/redis_sessions/settings.py
@@ -8,15 +8,34 @@
 SESSION_REDIS_DB = getattr(settings, 'SESSION_REDIS_DB', 0)
 SESSION_REDIS_PREFIX = getattr(settings, 'SESSION_REDIS_PREFIX', '')
 SESSION_REDIS_PASSWORD = getattr(settings, 'SESSION_REDIS_PASSWORD', None)
-SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH = getattr(
-    settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', None
-)
+SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH = getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', None)
 SESSION_REDIS_URL = getattr(settings, 'SESSION_REDIS_URL', None)
 
+"""
+Should be on the format:
+[
+    {
+        'SESSION_REDIS_HOST': 'localhost2',
+        'SESSION_REDIS_PORT': 6379,
+        'SESSION_REDIS_DB': 0,
+        'SESSION_REDIS_PASSWORD': None,
+        'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH': None,
+        'SESSION_REDIS_URL': None,
+        'SESSION_REDIS_WEIGHT': 1,
+    },
+    {
+        'SESSION_REDIS_HOST': 'localhost1',
+        'SESSION_REDIS_PORT': 6379,
+        'SESSION_REDIS_DB': 0,
+        'SESSION_REDIS_PASSWORD': None,
+        'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH': None,
+        'SESSION_REDIS_URL': None,
+        'SESSION_REDIS_WEIGHT': 1,
+    },
+]
+"""
+SESSION_REDIS_POOL = getattr(settings, 'SESSION_REDIS_POOL', None)
+
 # should be on the format [(host, port), (host, port), (host, port)]
-SESSION_REDIS_SENTINEL_LIST = getattr(
-	settings, 'SESSION_REDIS_SENTINEL_LIST', None
-)
-SESSION_REDIS_SENTINEL_MASTER_ALIAS = getattr(
-	settings, 'SESSION_REDIS_SENTINEL_MASTER_ALIAS', None
-)
+SESSION_REDIS_SENTINEL_LIST = getattr(settings, 'SESSION_REDIS_SENTINEL_LIST', None)
+SESSION_REDIS_SENTINEL_MASTER_ALIAS = getattr(settings, 'SESSION_REDIS_SENTINEL_MASTER_ALIAS', None)