From 969f67ff6d0ac047745c45a5f0716bc53068905f Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 2 Dec 2019 16:59:55 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(eucalyptus/3/wb)=20add=20missing=20su?= =?UTF-8?q?pport=20for=20redis=20sentinel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To connect celery to redis-sentinel we use the plugin celery-redis-sentinel. Also to use session in redis, the setting allowing to change the backend to use was missing and setting to configure redis were also missing. We added them, it is now possible to use redis to store sessions. --- releases/eucalyptus/3/wb/CHANGELOG.md | 1 + .../3/wb/config/cms/docker_run_production.py | 35 +++++++++++++++++++ .../3/wb/config/lms/docker_run_production.py | 35 +++++++++++++++++++ releases/eucalyptus/3/wb/requirements.txt | 1 + 4 files changed, 72 insertions(+) diff --git a/releases/eucalyptus/3/wb/CHANGELOG.md b/releases/eucalyptus/3/wb/CHANGELOG.md index 08f406b6..2220814e 100644 --- a/releases/eucalyptus/3/wb/CHANGELOG.md +++ b/releases/eucalyptus/3/wb/CHANGELOG.md @@ -15,6 +15,7 @@ release. - First experimental release of OpenEdx `eucalyptus.3` (wb flavor). - Set replicaSet and read_preference in mongodb connection +- Add missing support for redis sentinel [unreleased]: https://github.com/openfun/openedx-docker/compare/eucalyptus.3-1.0.0-wb...HEAD [eucalyptus.3-1.0.0-wb]: https://github.com/openfun/openedx-docker/releases/tag/eucalyptus.3-1.0.0-wb diff --git a/releases/eucalyptus/3/wb/config/cms/docker_run_production.py b/releases/eucalyptus/3/wb/config/cms/docker_run_production.py index 8c1de4b1..6298b45e 100644 --- a/releases/eucalyptus/3/wb/config/cms/docker_run_production.py +++ b/releases/eucalyptus/3/wb/config/cms/docker_run_production.py @@ -14,6 +14,7 @@ import os import platform +from celery_redis_sentinel import register from lms.envs.fun.utils import Configuration from openedx.core.lib.logsettings import get_logger_config from path import Path as path @@ -189,6 +190,32 @@ "SESSION_SAVE_EVERY_REQUEST", default=SESSION_SAVE_EVERY_REQUEST, formatter=bool ) +# Configuration to use session with redis +# To use redis, change SESSION_ENGINE with value redis_sessions.session +SESSION_REDIS_HOST = config("SESSION_REDIS_HOST", default="redis") +SESSION_REDIS_PORT = config("SESSION_REDIS_HOST", default=6379, formatter=int) +SESSION_REDIS_DB = config("SESSION_REDIS_DB", default=1, formatter=int) +SESSION_REDIS_PASSWORD = config("SESSION_REDIS_PASSWORD", default=None) +SESSION_REDIS_PREFIX = config("SESSION_REDIS_PREFIX", default="session") +SESSION_REDIS_SOCKET_TIMEOUT = config("SESSION_REDIS_SOCKET_TIMEOUT", default=1, formatter=int) +SESSION_REDIS_RETRY_ON_TIMEOUT = config("SESSION_REDIS_RETRY_ON_TIMEOUT", default=False, formatter=bool) + +SESSION_REDIS = config( + "SESSION_REDIS", + default={ + "host": SESSION_REDIS_HOST, + "port": SESSION_REDIS_PORT, + "db": SESSION_REDIS_DB, # db 0 is used for Celery Broker + "password": SESSION_REDIS_PASSWORD, + "prefix": SESSION_REDIS_PREFIX, + "socket_timeout": SESSION_REDIS_SOCKET_TIMEOUT, + "retry_on_timeout": SESSION_REDIS_RETRY_ON_TIMEOUT, + }, + formatter=json.loads, +) +SESSION_REDIS_SENTINEL_LIST = config("SESSION_REDIS_SENTINEL_LIST", default=None, formatter=json.loads) +SESSION_REDIS_SENTINEL_MASTER_ALIAS = config("SESSION_REDIS_SENTINEL_MASTER_ALIAS", default=None) + # social sharing settings SOCIAL_SHARING_SETTINGS = config( "SOCIAL_SHARING_SETTINGS", default=SOCIAL_SHARING_SETTINGS, formatter=json.loads @@ -519,6 +546,7 @@ DATADOG["api_key"] = config("DATADOG_API", default=None) # Celery Broker +# For redis sentinel you the transport redis-sentinel CELERY_BROKER_TRANSPORT = config("CELERY_BROKER_TRANSPORT", default="redis") CELERY_BROKER_USER = config("CELERY_BROKER_USER", default="") CELERY_BROKER_PASSWORD = config("CELERY_BROKER_PASSWORD", default="") @@ -526,6 +554,10 @@ CELERY_BROKER_PORT = config("CELERY_BROKER_PORT", default=6379, formatter=int) CELERY_BROKER_VHOST = config("CELERY_BROKER_VHOST", default=0, formatter=int) +if CELERY_BROKER_TRANSPORT == "redis-sentinel": + # register redis sentinel schema in celery + register() + BROKER_URL = "{transport}://{user}:{password}@{host}:{port}/{vhost}".format( transport=CELERY_BROKER_TRANSPORT, user=CELERY_BROKER_USER, @@ -534,6 +566,9 @@ port=CELERY_BROKER_PORT, vhost=CELERY_BROKER_VHOST, ) +# To use redis-sentinel, refer to the documenation here +# https://celery-redis-sentinel.readthedocs.io/en/latest/ +BROKER_TRANSPORT_OPTIONS = config("BROKER_TRANSPORT_OPTIONS", default={}, formatter=json.loads) # Event tracking TRACKING_BACKENDS.update(config("TRACKING_BACKENDS", default={}, formatter=json.loads)) diff --git a/releases/eucalyptus/3/wb/config/lms/docker_run_production.py b/releases/eucalyptus/3/wb/config/lms/docker_run_production.py index 8ae1befb..a9d1e6db 100644 --- a/releases/eucalyptus/3/wb/config/lms/docker_run_production.py +++ b/releases/eucalyptus/3/wb/config/lms/docker_run_production.py @@ -23,6 +23,7 @@ import platform import warnings +from celery_redis_sentinel import register from openedx.core.lib.logsettings import get_logger_config from path import Path as path from xmodule.modulestore.modulestore_settings import ( @@ -195,6 +196,32 @@ "SESSION_SAVE_EVERY_REQUEST", default=SESSION_SAVE_EVERY_REQUEST, formatter=bool ) +# Configuration to use session with redis +# To use redis, change SESSION_ENGINE with value redis_sessions.session +SESSION_REDIS_HOST = config("SESSION_REDIS_HOST", default="redis") +SESSION_REDIS_PORT = config("SESSION_REDIS_HOST", default=6379, formatter=int) +SESSION_REDIS_DB = config("SESSION_REDIS_DB", default=1, formatter=int) +SESSION_REDIS_PASSWORD = config("SESSION_REDIS_PASSWORD", default=None) +SESSION_REDIS_PREFIX = config("SESSION_REDIS_PREFIX", default="session") +SESSION_REDIS_SOCKET_TIMEOUT = config("SESSION_REDIS_SOCKET_TIMEOUT", default=1, formatter=int) +SESSION_REDIS_RETRY_ON_TIMEOUT = config("SESSION_REDIS_RETRY_ON_TIMEOUT", default=False, formatter=bool) + +SESSION_REDIS = config( + "SESSION_REDIS", + default={ + "host": SESSION_REDIS_HOST, + "port": SESSION_REDIS_PORT, + "db": SESSION_REDIS_DB, # db 0 is used for Celery Broker + "password": SESSION_REDIS_PASSWORD, + "prefix": SESSION_REDIS_PREFIX, + "socket_timeout": SESSION_REDIS_SOCKET_TIMEOUT, + "retry_on_timeout": SESSION_REDIS_RETRY_ON_TIMEOUT, + }, + formatter=json.loads, +) +SESSION_REDIS_SENTINEL_LIST = config("SESSION_REDIS_SENTINEL_LIST", default=None, formatter=json.loads) +SESSION_REDIS_SENTINEL_MASTER_ALIAS = config("SESSION_REDIS_SENTINEL_MASTER_ALIAS", default=None) + AWS_SES_REGION_NAME = config("AWS_SES_REGION_NAME", default="us-east-1") AWS_SES_REGION_ENDPOINT = config( "AWS_SES_REGION_ENDPOINT", default="email.us-east-1.amazonaws.com" @@ -782,6 +809,7 @@ EDX_API_KEY = config("EDX_API_KEY", default=None) # Celery Broker +# For redis sentinel you the transport redis-sentinel CELERY_BROKER_TRANSPORT = config("CELERY_BROKER_TRANSPORT", default="redis") CELERY_BROKER_USER = config("CELERY_BROKER_USER", default="") CELERY_BROKER_PASSWORD = config("CELERY_BROKER_PASSWORD", default="") @@ -789,6 +817,10 @@ CELERY_BROKER_PORT = config("CELERY_BROKER_PORT", default=6379, formatter=int) CELERY_BROKER_VHOST = config("CELERY_BROKER_VHOST", default=0, formatter=int) +if CELERY_BROKER_TRANSPORT == "redis-sentinel": + # register redis sentinel schema in celery + register() + BROKER_URL = "{transport}://{user}:{password}@{host}:{port}/{vhost}".format( transport=CELERY_BROKER_TRANSPORT, user=CELERY_BROKER_USER, @@ -797,6 +829,9 @@ port=CELERY_BROKER_PORT, vhost=CELERY_BROKER_VHOST, ) +# To use redis-sentinel, refer to the documenation here +# https://celery-redis-sentinel.readthedocs.io/en/latest/ +BROKER_TRANSPORT_OPTIONS = config("BROKER_TRANSPORT_OPTIONS", default={}, formatter=json.loads) # upload limits STUDENT_FILEUPLOAD_MAX_SIZE = config( diff --git a/releases/eucalyptus/3/wb/requirements.txt b/releases/eucalyptus/3/wb/requirements.txt index 802ff33a..a390c80f 100644 --- a/releases/eucalyptus/3/wb/requirements.txt +++ b/releases/eucalyptus/3/wb/requirements.txt @@ -13,3 +13,4 @@ xblock-utils2==0.3.0 # ==== third-party apps ==== raven==6.9.0 django-redis-sessions==0.6.1 +celery-redis-sentinel==0.3.0