From ab174504417a13d60eabee514ecbdeda89a2e6b0 Mon Sep 17 00:00:00 2001 From: AIweimd <42801710+AIweimd@users.noreply.github.com> Date: Tue, 2 Jun 2020 09:44:20 +0800 Subject: [PATCH 1/9] extension for redis cluster mode --- flask_caching/backends/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/flask_caching/backends/__init__.py b/flask_caching/backends/__init__.py index 23769b4b..5bde7655 100644 --- a/flask_caching/backends/__init__.py +++ b/flask_caching/backends/__init__.py @@ -18,7 +18,7 @@ from flask_caching.backends.nullcache import NullCache # TODO: Rename to "redis" when python2 support is removed -from flask_caching.backends.rediscache import RedisCache, RedisSentinelCache +from flask_caching.backends.rediscache import RedisCache, RedisSentinelCache, RedisClusterCache from flask_caching.backends.simplecache import SimpleCache try: @@ -35,6 +35,7 @@ "filesystem", "redis", "redissentinel", + "rediscluster", "uwsgi", "memcached", "gaememcached", @@ -116,6 +117,15 @@ def redissentinel(app, config, args, kwargs): return RedisSentinelCache(*args, **kwargs) +def rediscluster(app, config, args, kwargs): + kwargs.update( + dict(cluster=config.get("CACHE_REDIS_CLUSTER", ""), + password=config.get("CACHE_REDIS_PASSWORD", ""), + default_timeout=config.get("CACHE_DEFAULT_TIMEOUT", 300), + key_prefix=config.get("CACHE_KEY_PREFIX", ""))) + return RedisClusterCache(*args, **kwargs) + + def uwsgi(app, config, args, kwargs): if not has_UWSGICache: raise NotImplementedError( From 1cedc3814a9d0f84e1c95abc36c5224d7aff7d79 Mon Sep 17 00:00:00 2001 From: AIweimd <42801710+AIweimd@users.noreply.github.com> Date: Tue, 2 Jun 2020 09:48:16 +0800 Subject: [PATCH 2/9] extension for redis cluster mode --- flask_caching/backends/rediscache.py | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/flask_caching/backends/rediscache.py b/flask_caching/backends/rediscache.py index e2708503..0b718a15 100644 --- a/flask_caching/backends/rediscache.py +++ b/flask_caching/backends/rediscache.py @@ -273,3 +273,58 @@ def __init__( self._read_clients = sentinel.slave_for(master) self.key_prefix = key_prefix or "" + +class RedisClusterCache(RedisCache): + """Uses the Redis key-value store as a cache backend. + + The first argument can be either a string denoting address of the Redis + server or an object resembling an instance of a rediscluster.RedisCluster class. + + Note: Python Redis API already takes care of encoding unicode strings on + the fly. + + + :param cluster: The redis cluster nodes address separated by comma. + e.g. host1:port,host2:port2,host3:port3 . + :param password: password authentication for the Redis server. + :param default_timeout: the default timeout that is used if no timeout is + specified on :meth:`~BaseCache.set`. A timeout of + 0 indicates that the cache never expires. + :param key_prefix: A prefix that should be added to all keys. + + Any additional keyword arguments will be passed to + ``rediscluster.RedisCluster``. + """ + def __init__(self, + cluster="", + password="", + default_timeout=300, + key_prefix="", + **kwargs): + super().__init__(default_timeout=default_timeout) + + if kwargs.get("decode_responses", None): + raise ValueError("decode_responses is not supported by " + "RedisCache.") + + try: + from rediscluster import RedisCluster + except ImportError: + raise RuntimeError("no rediscluster module found") + HOST = 0 + PORT = 1 + try: + nodes = [(node.split(':')) for node in cluster.split(',')] + startup_nodes = [{ + 'host': node[HOST].strip(), + 'port': node[PORT].strip() + } for node in nodes] + except IndexError: + raise ValueError("Please give the correct cluster argument " + "e.g. host1:port,host2:port2,host3:port3") + cluster = RedisCluster(startup_nodes=startup_nodes, + password=password, + skip_full_coverage_check=True, + **kwargs) + self._write_client = self._read_clients = cluster + self.key_prefix = key_prefix From eec47d8cd798ce24eda9c61e8529a344771a8f18 Mon Sep 17 00:00:00 2001 From: AIweimd <42801710+AIweimd@users.noreply.github.com> Date: Wed, 3 Jun 2020 13:55:52 +0800 Subject: [PATCH 3/9] extension for redis cluster mode --- flask_caching/backends/rediscache.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flask_caching/backends/rediscache.py b/flask_caching/backends/rediscache.py index 0b718a15..4042c1cb 100644 --- a/flask_caching/backends/rediscache.py +++ b/flask_caching/backends/rediscache.py @@ -311,8 +311,7 @@ def __init__(self, from rediscluster import RedisCluster except ImportError: raise RuntimeError("no rediscluster module found") - HOST = 0 - PORT = 1 + HOST, PORT = 0, 1 try: nodes = [(node.split(':')) for node in cluster.split(',')] startup_nodes = [{ @@ -321,7 +320,7 @@ def __init__(self, } for node in nodes] except IndexError: raise ValueError("Please give the correct cluster argument " - "e.g. host1:port,host2:port2,host3:port3") + "e.g. host1:port1,host2:port2,host3:port3") cluster = RedisCluster(startup_nodes=startup_nodes, password=password, skip_full_coverage_check=True, From 87ffb9643fb91a3c2ce8299d28e9eefb3e284a41 Mon Sep 17 00:00:00 2001 From: AIweimd <42801710+AIweimd@users.noreply.github.com> Date: Wed, 24 Jun 2020 10:19:40 +0800 Subject: [PATCH 4/9] Update rediscache.py --- flask_caching/backends/rediscache.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/flask_caching/backends/rediscache.py b/flask_caching/backends/rediscache.py index 4042c1cb..305d750d 100644 --- a/flask_caching/backends/rediscache.py +++ b/flask_caching/backends/rediscache.py @@ -285,7 +285,7 @@ class RedisClusterCache(RedisCache): :param cluster: The redis cluster nodes address separated by comma. - e.g. host1:port,host2:port2,host3:port3 . + e.g. host1:port1,host2:port2,host3:port3 . :param password: password authentication for the Redis server. :param default_timeout: the default timeout that is used if no timeout is specified on :meth:`~BaseCache.set`. A timeout of @@ -311,19 +311,23 @@ def __init__(self, from rediscluster import RedisCluster except ImportError: raise RuntimeError("no rediscluster module found") - HOST, PORT = 0, 1 + try: nodes = [(node.split(':')) for node in cluster.split(',')] startup_nodes = [{ - 'host': node[HOST].strip(), - 'port': node[PORT].strip() + 'host': node[0].strip(), + 'port': node[1].strip() } for node in nodes] except IndexError: raise ValueError("Please give the correct cluster argument " "e.g. host1:port1,host2:port2,host3:port3") + # Skips the check of cluster-require-full-coverage config, + # useful for clusters without the CONFIG command (like aws) + skip_full_coverage_check = kwargs.pop('skip_full_coverage_check', True) + cluster = RedisCluster(startup_nodes=startup_nodes, password=password, - skip_full_coverage_check=True, + skip_full_coverage_check=skip_full_coverage_check, **kwargs) self._write_client = self._read_clients = cluster self.key_prefix = key_prefix From a499cf0717aec09be77abb39601954c28dee80b5 Mon Sep 17 00:00:00 2001 From: AIweimd <42801710+AIweimd@users.noreply.github.com> Date: Thu, 16 Jul 2020 17:33:30 +0800 Subject: [PATCH 5/9] Update index.rst Extension for redis cluster mode --- docs/index.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 938fc5fe..07b86dca 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -329,6 +329,7 @@ The following configuration values exist for Flask-Caching: * **filesystem**: FileSystemCache * **redis**: RedisCache (redis required) * **redissentinel**: RedisSentinelCache (redis required) + * **rediscluster**: RedisClusterCache (redis required) * **uwsgi**: UWSGICache (uwsgi required) * **memcached**: MemcachedCache (pylibmc or memcache required) * **gaememcached**: same as memcached (for backwards compatibility) @@ -386,9 +387,11 @@ The following configuration values exist for Flask-Caching: ``CACHE_REDIS_DB`` A Redis db (zero-based number index). Default is 0. Used only for RedisCache and RedisSentinelCache. ``CACHE_REDIS_SENTINELS`` A list or a tuple of Redis sentinel addresses. Used only for - RedisSentinelCache. + RedisClusterCache. ``CACHE_REDIS_SENTINEL_MASTER`` The name of the master server in a sentinel configuration. Used only for RedisSentinelCache. +``CACHE_REDIS_CLUSTER `` A string of comma-separated Redis cluster node addresses. + e.g. host1:port1,host2:port2,host3:port3 . Used only for RedisClusterCache. ``CACHE_DIR`` Directory to store cache. Used only for FileSystemCache. ``CACHE_REDIS_URL`` URL to connect to Redis server. @@ -472,6 +475,17 @@ Set ``CACHE_TYPE`` to ``redissentinel`` to use this type. Entries in CACHE_OPTIONS are passed to the redis client as ``**kwargs`` +RedisClusterCache +`````````````````` + +Set ``CACHE_TYPE`` to ``rediscluster`` to use this type. + +- CACHE_KEY_PREFIX +- CACHE_REDIS_CLUSTER +- CACHE_REDIS_PASSWORD + +Entries in CACHE_OPTIONS are passed to the redis client as ``**kwargs`` + MemcachedCache `````````````` From 61b2a1f0029496efe5ea3eb9b1052502085e3975 Mon Sep 17 00:00:00 2001 From: AIweimd <42801710+AIweimd@users.noreply.github.com> Date: Thu, 16 Jul 2020 17:40:09 +0800 Subject: [PATCH 6/9] Update index.rst Extension for redis cluster mode --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 07b86dca..f1b0d43e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -387,7 +387,7 @@ The following configuration values exist for Flask-Caching: ``CACHE_REDIS_DB`` A Redis db (zero-based number index). Default is 0. Used only for RedisCache and RedisSentinelCache. ``CACHE_REDIS_SENTINELS`` A list or a tuple of Redis sentinel addresses. Used only for - RedisClusterCache. + RedisSentinelCache. ``CACHE_REDIS_SENTINEL_MASTER`` The name of the master server in a sentinel configuration. Used only for RedisSentinelCache. ``CACHE_REDIS_CLUSTER `` A string of comma-separated Redis cluster node addresses. From 2180f183853ceaca72960686fd2691508d853ad7 Mon Sep 17 00:00:00 2001 From: AIweimd <42801710+AIweimd@users.noreply.github.com> Date: Thu, 16 Jul 2020 20:26:33 +0800 Subject: [PATCH 7/9] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index f1b0d43e..b4c0d63b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -390,7 +390,7 @@ The following configuration values exist for Flask-Caching: RedisSentinelCache. ``CACHE_REDIS_SENTINEL_MASTER`` The name of the master server in a sentinel configuration. Used only for RedisSentinelCache. -``CACHE_REDIS_CLUSTER `` A string of comma-separated Redis cluster node addresses. +``CACHE_REDIS_CLUSTER`` A string of comma-separated Redis cluster node addresses. e.g. host1:port1,host2:port2,host3:port3 . Used only for RedisClusterCache. ``CACHE_DIR`` Directory to store cache. Used only for FileSystemCache. From da09a86c7f2a90f780a379f2ce5fe026ca29fcc5 Mon Sep 17 00:00:00 2001 From: AIweimd <42801710+AIweimd@users.noreply.github.com> Date: Thu, 16 Jul 2020 20:35:33 +0800 Subject: [PATCH 8/9] Update index.rst Add redis cluster documention. --- docs/index.rst | 82 ++------------------------------------------------ 1 file changed, 2 insertions(+), 80 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b4c0d63b..14fb0a6d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -390,7 +390,7 @@ The following configuration values exist for Flask-Caching: RedisSentinelCache. ``CACHE_REDIS_SENTINEL_MASTER`` The name of the master server in a sentinel configuration. Used only for RedisSentinelCache. -``CACHE_REDIS_CLUSTER`` A string of comma-separated Redis cluster node addresses. +``CACHE_REDIS_CLUSTER`` A string of comma-separated Redis cluster node addresses. e.g. host1:port1,host2:port2,host3:port3 . Used only for RedisClusterCache. ``CACHE_DIR`` Directory to store cache. Used only for FileSystemCache. @@ -544,82 +544,4 @@ Relevant configuration values SpreadSASLMemcachedCache ```````````````````````` -Set ``CACHE_TYPE`` to ``spreadsaslmemcached`` to use this type. - -Same as SASLMemcachedCache however, it has the ablity to spread value across -multiple keys if it is bigger than the memcached treshold which by -default is 1M. Uses pickle. - -.. versionadded:: 0.11 - -.. versionchanged:: 1.1.0 - Renamed ``spreadsaslmemcachedcache`` to ``spreadsaslmemcached`` for - the sake of consistency. - - -Custom Cache Backends ---------------------- - -You are able to easily add your own custom cache backends by exposing a function -that can instantiate and return a cache object. ``CACHE_TYPE`` will be the -import string to your custom function. It should expect to receive three -arguments. - -* ``app`` -* ``args`` -* ``kwargs`` - -Your custom cache object must also subclass the -:class:`flask_caching.backends.cache.BaseCache` class. Flask-Caching will make sure -that ``threshold`` is already included in the kwargs options dictionary since -it is common to all BaseCache classes. - -An example Redis cache implementation:: - - #: the_app/custom.py - class RedisCache(BaseCache): - def __init__(self, servers, default_timeout=500): - pass - - def redis(app, config, args, kwargs): - args.append(app.config['REDIS_SERVERS']) - return RedisCache(*args, **kwargs) - -With this example, your ``CACHE_TYPE`` might be ``the_app.custom.redis`` - -An example PylibMC cache implementation to change binary setting and provide -username/password if SASL is enabled on the library:: - - #: the_app/custom.py - def pylibmccache(app, config, args, kwargs): - return pylibmc.Client(servers=config['CACHE_MEMCACHED_SERVERS'], - username=config['CACHE_MEMCACHED_USERNAME'], - password=config['CACHE_MEMCACHED_PASSWORD'], - binary=True) - -With this example, your ``CACHE_TYPE`` might be ``the_app.custom.pylibmccache`` - - -API ---- - -.. toctree:: - :maxdepth: 2 - - api - - -Additional Information ----------------------- - -.. toctree:: - :maxdepth: 2 - - changelog - license - -* :ref:`search` - - -.. _Flask: http://flask.pocoo.org/ -.. _werkzeug: http://werkzeug.pocoo.org/ +Set ``CACHE_TYPE`` to ``spreadsaslmemcached From 5659d6ea38727a98f18bdb09711f0a1d201f778f Mon Sep 17 00:00:00 2001 From: AIweimd <42801710+AIweimd@users.noreply.github.com> Date: Fri, 17 Jul 2020 15:40:13 +0800 Subject: [PATCH 9/9] Update index.rst Add the documentation of Redis cluster mode. --- docs/index.rst | 80 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 14fb0a6d..44e8c8b0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -544,4 +544,82 @@ Relevant configuration values SpreadSASLMemcachedCache ```````````````````````` -Set ``CACHE_TYPE`` to ``spreadsaslmemcached +Set ``CACHE_TYPE`` to ``spreadsaslmemcached`` to use this type. + +Same as SASLMemcachedCache however, it has the ablity to spread value across +multiple keys if it is bigger than the memcached treshold which by +default is 1M. Uses pickle. + +.. versionadded:: 0.11 + +.. versionchanged:: 1.1.0 + Renamed ``spreadsaslmemcachedcache`` to ``spreadsaslmemcached`` for + the sake of consistency. + + +Custom Cache Backends +--------------------- + +You are able to easily add your own custom cache backends by exposing a function +that can instantiate and return a cache object. ``CACHE_TYPE`` will be the +import string to your custom function. It should expect to receive three +arguments. + +* ``app`` +* ``args`` +* ``kwargs`` + +Your custom cache object must also subclass the +:class:`flask_caching.backends.cache.BaseCache` class. Flask-Caching will make sure +that ``threshold`` is already included in the kwargs options dictionary since +it is common to all BaseCache classes. + +An example Redis cache implementation:: + + #: the_app/custom.py + class RedisCache(BaseCache): + def __init__(self, servers, default_timeout=500): + pass + + def redis(app, config, args, kwargs): + args.append(app.config['REDIS_SERVERS']) + return RedisCache(*args, **kwargs) + +With this example, your ``CACHE_TYPE`` might be ``the_app.custom.redis`` + +An example PylibMC cache implementation to change binary setting and provide +username/password if SASL is enabled on the library:: + + #: the_app/custom.py + def pylibmccache(app, config, args, kwargs): + return pylibmc.Client(servers=config['CACHE_MEMCACHED_SERVERS'], + username=config['CACHE_MEMCACHED_USERNAME'], + password=config['CACHE_MEMCACHED_PASSWORD'], + binary=True) + +With this example, your ``CACHE_TYPE`` might be ``the_app.custom.pylibmccache`` + + +API +--- + +.. toctree:: + :maxdepth: 2 + + api + + +Additional Information +---------------------- + +.. toctree:: + :maxdepth: 2 + + changelog + license + +* :ref:`search` + + +.. _Flask: http://flask.pocoo.org/ +.. _werkzeug: http://werkzeug.pocoo.org/