OpenConceptLab/ocl_issues#927 Redis clustering, adding cache and cele…

…ry_once retries

core/common/backends.py

@@ -1,22 +1,23 @@
 from celery_once.backends import Redis
 from django.conf import settings
 from django.contrib.auth.backends import ModelBackend
+from django_redis import get_redis_connection
 from mozilla_django_oidc.auth import OIDCAuthenticationBackend
 from pydash import get
-from redis import Sentinel
 
 
-class QueueOnceRedisSentinelBackend(Redis):
+class QueueOnceRedisBackend(Redis):
+    """
+    Calls get_redis_connection from django_redis so that it re-uses django redis cache config.
+    """
     def __init__(self, backend_settings):
         # pylint: disable=super-init-not-called
-        self._sentinel = Sentinel(backend_settings['sentinels'])
-        self._sentinel_master = backend_settings['sentinels_master']
         self.blocking_timeout = backend_settings.get("blocking_timeout", 1)
         self.blocking = backend_settings.get("blocking", False)
 
     @property
     def redis(self):
-        return self._sentinel.master_for(self._sentinel_master)
+        return get_redis_connection('default')
 
 
 class OCLOIDCAuthenticationBackend(OIDCAuthenticationBackend):

core/settings.py

@@ -17,6 +17,8 @@
 from celery.schedules import crontab
 from corsheaders.defaults import default_headers
 from kombu import Queue, Exchange
+from redis.backoff import ExponentialBackoff
+from redis.retry import Retry
 
 from core import __version__
 
@@ -308,11 +310,20 @@
 REDIS_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}"
 
 REDIS_SENTINELS = os.environ.get('REDIS_SENTINELS', None)
-REDIS_SENTINELS_MASTER = os.environ.get('REDIS_SENTINELS_MASTER', 'default')
-REDIS_SENTINELS_LIST = []
+if REDIS_SENTINELS:
+    REDIS_SENTINELS_MASTER = os.environ.get('REDIS_SENTINELS_MASTER', 'default')
+    REDIS_SENTINELS_LIST = []
 
 # django cache
-OPTIONS = {}
+OPTIONS = {
+    'SOCKET_CONNECT_TIMEOUT': 5,
+    'SOCKET_TIMEOUT': 5,
+    'CONNECTION_POOL_KWARGS': {
+        'max_connections': 100,
+        'retry': Retry(ExponentialBackoff(cap=10, base=0.5), 10),
+        'health_check_interval': 5
+    }
+}
 if REDIS_SENTINELS:
     DJANGO_REDIS_CONNECTION_FACTORY = 'django_redis.pool.SentinelConnectionFactory'
 
@@ -328,7 +339,7 @@
 CACHES = {
     'default': {
         'BACKEND': 'django_redis.cache.RedisCache',
-        'LOCATION': f'redis://{REDIS_SENTINELS_MASTER}/{REDIS_DB}' if REDIS_SENTINELS_MASTER else REDIS_URL,
+        'LOCATION': f'redis://{REDIS_SENTINELS_MASTER}/{REDIS_DB}' if REDIS_SENTINELS else REDIS_URL,
         'OPTIONS': OPTIONS
     }
 }
@@ -367,7 +378,6 @@
     'redis_socket_timeout': 5,
     'redis_backend_health_check_interval': 5,
     'redis_retry_on_timeout': True,
-    'redis_socket_keepalive': True,
 }
 
 if REDIS_SENTINELS:
@@ -401,21 +411,9 @@
 }
 
 CELERY_ACCEPT_CONTENT = ['application/json']
-if REDIS_SENTINELS:
-    CELERY_ONCE = {
-        'backend': 'core.common.backends.QueueOnceRedisSentinelBackend',
-        'settings': {
-            'sentinels': REDIS_SENTINELS_LIST,
-            'sentinels_master': REDIS_SENTINELS_MASTER
-        }
-    }
-else:
-    CELERY_ONCE = {
-        'backend': 'celery_once.backends.Redis',
-        'settings': {
-            'url': CELERY_RESULT_BACKEND,
-        }
-    }
+CELERY_ONCE = {
+        'backend': 'core.common.backends.QueueOnceRedisBackend',
+}
 CELERYBEAT_SCHEDULE = {
     'healthcheck-every-minute': {
         'task': 'core.common.tasks.beat_healthcheck',