diff --git a/.travis.yml b/.travis.yml index a8594fe6..14c52791 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,14 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6-dev" - "nightly" services: - redis-server install: - "if [[ $REDIS_VERSION == '3.0' ]]; then REDIS_VERSION=3.0 make redis-install; fi" - "if [[ $REDIS_VERSION == '3.2' ]]; then REDIS_VERSION=3.2 make redis-install; fi" + - "if [[ $REDIS_VERSION == '4.0' ]]; then REDIS_VERSION=4.0 make redis-install; fi" - pip install -r dev-requirements.txt - pip install -e . - "if [[ $HIREDIS == '1' ]]; then pip install hiredis; fi" @@ -23,6 +25,10 @@ env: - HIREDIS=0 REDIS_VERSION=3.2 # Redis 3.2 and HIREDIS - HIREDIS=1 REDIS_VERSION=3.2 + # Redis 4.0 + - HIREDIS=0 REDIS_VERSION=4.0 + # Redis 4.0 and HIREDIS + - HIREDIS=1 REDIS_VERSION=4.0 script: - make start - coverage erase diff --git a/README.md b/README.md index fb7534fa..d1def896 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,6 @@ Small sample script that shows how to get started with RedisCluster. It can also >>> # Requires at least one node for cluster discovery. Multiple nodes is recommended. >>> startup_nodes = [{"host": "127.0.0.1", "port": "7000"}] ->>> # Note: decode_responses must be set to True when used with python3 >>> rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) >>> rc.set("foo", "bar") @@ -55,8 +54,8 @@ True ## License & Authors -Copyright (c) 2013-2016 Johan Andersson +Copyright (c) 2013-2017 Johan Andersson MIT (See docs/License.txt file) -The license should be the same as redis-py (https://github.com/andymccurdy/redis-py) \ No newline at end of file +The license should be the same as redis-py (https://github.com/andymccurdy/redis-py) diff --git a/docs/authors.rst b/docs/authors.rst index bd99c6c4..ee4cb8d4 100644 --- a/docs/authors.rst +++ b/docs/authors.rst @@ -22,3 +22,5 @@ Authors who contributed code or testing: - monklof - https://github.com/monklof - dutradda - https://github.com/dutradda - AngusP - https://github.com/AngusP + - Doug Kent - https://github.com/dkent + - VascoVisser - https://github.com/VascoVisser diff --git a/docs/index.rst b/docs/index.rst index bba309b4..b90dc393 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,7 +44,7 @@ Small sample script that shows how to get started with RedisCluster. It can also >>> # Requires at least one node for cluster discovery. Multiple nodes is recommended. >>> startup_nodes = [{"host": "127.0.0.1", "port": "7000"}] - >>> # Note: decode_responses must be set to True when used with python3 + >>> # Note: See note on Python 3 for decode_responses behaviour >>> rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) >>> rc.set("foo", "bar") @@ -53,6 +53,9 @@ Small sample script that shows how to get started with RedisCluster. It can also 'bar' +.. note:: Python 3 + + Since Python 3 changed to Unicode strings from Python 2's ASCII, the return type of *most* commands will be binary strings, unless the class is instantiated with the option ``decode_responses=True``. In this case, the responses will be Python 3 strings (Unicode). For the init argument `decode_responses`, when set to False, redis-py-cluster will not attempt to decode the responses it receives. In Python 3, this means the responses will be of type `bytes`. In Python 2, they will be native strings (`str`). If `decode_responses` is set to True, for Python 3 responses will be `str`, for Python 2 they will be `unicode`. Dependencies & supported python versions ---------------------------------------- @@ -67,14 +70,15 @@ Dependencies & supported python versions Supported python versions ------------------------- -- 2.7.x -- 3.3.x -- 3.4.1+ -- 3.5.x +- 2.7 +- 3.3 +- 3.4.1+ (See note) +- 3.5 +- 3.6 Experimental: -- Up to 3.6.0a0 +- 3.7-dev .. note:: Python 3.4.0 diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 7af8862b..c7d56f35 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,11 +1,20 @@ Release Notes ============= -Unstable --------- +1.3.4 (Mar 5, 2017) +------------------- + * Package is now built as a wheel and source package when releases is built. + * Fixed issues with some key types in `NodeManager.keyslot()`. * Add support for `PUBSUB` subcommands `CHANNELS`, `NUMSUB [arg] [args...]` and `NUMPAT`. * Add method `set_result_callback(command, callback)` allowing the default reply callbacks to be changed, in the same way `set_response_callback(command, callback)` inherited from Redis-Py does for responses. + * Node manager now honors defined max_connections variable so connections that is emited from that class uses the same variable. + * Fixed a bug in cluster detection when running on python 3.x and decode_responses=False was used. + Data back from redis for cluster structure is now converted no matter what the data you want to set/get later is using. + * Add SSLClusterConnection for connecting over TLS/SSL to Redis Cluster + * Add new option to make the nodemanager to follow the cluster when nodes move around by avoiding to query the original list of startup nodes that was provided + when the client object was first created. This could make the client handle drifting clusters on for example AWS easier but there is a higher risk of the client talking to + the wrong group of nodes during split-brain event if the cluster is not consistent. This feature is EXPERIMENTAL and use it with care. 1.3.3 (Dec 15, 2016) -------------------- diff --git a/rediscluster/__init__.py b/rediscluster/__init__.py index 59ae5282..c7d0a6d3 100644 --- a/rediscluster/__init__.py +++ b/rediscluster/__init__.py @@ -16,7 +16,7 @@ setattr(redis, "StrictClusterPipeline", StrictClusterPipeline) # Major, Minor, Fix version -__version__ = (1, 3, 3) +__version__ = (1, 3, 4) if sys.version_info[0:3] == (3, 4, 0): raise RuntimeError("CRITICAL: rediscluster do not work with python 3.4.0. Please use 3.4.1 or higher.") diff --git a/rediscluster/client.py b/rediscluster/client.py index ccc2e134..65122ec4 100644 --- a/rediscluster/client.py +++ b/rediscluster/client.py @@ -53,7 +53,7 @@ class StrictRedisCluster(StrictRedis): "ECHO", "CONFIG GET", "CONFIG SET", "SLOWLOG GET", "CLIENT KILL", "INFO", "BGREWRITEAOF", "BGSAVE", "CLIENT LIST", "CLIENT GETNAME", "CONFIG RESETSTAT", "CONFIG REWRITE", "DBSIZE", "LASTSAVE", "PING", "SAVE", "SLOWLOG LEN", "SLOWLOG RESET", - "TIME", "KEYS", "CLUSTER INFO", "PUBSUB CHANNELS", + "TIME", "KEYS", "CLUSTER INFO", "PUBSUB CHANNELS", "PUBSUB NUMSUB", "PUBSUB NUMPAT", ], 'all-nodes'), string_keys_to_dict([ @@ -126,7 +126,7 @@ class StrictRedisCluster(StrictRedis): } def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32, max_connections_per_node=False, init_slot_cache=True, - readonly_mode=False, reinitialize_steps=None, skip_full_coverage_check=False, **kwargs): + readonly_mode=False, reinitialize_steps=None, skip_full_coverage_check=False, nodemanager_follow_cluster=False, **kwargs): """ :startup_nodes: List of nodes that initial bootstrapping can be done from @@ -141,6 +141,10 @@ def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32, :skip_full_coverage_check: Skips the check of cluster-require-full-coverage config, useful for clusters without the CONFIG command (like aws) + :nodemanager_follow_cluster: + The node manager will during initialization try the last set of nodes that + it was operating on. This will allow the client to drift along side the cluster + if the cluster nodes move around alot. :**kwargs: Extra arguments that will be sent into StrictRedis instance when created (See Official redis-py doc for supported kwargs @@ -173,6 +177,7 @@ def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32, reinitialize_steps=reinitialize_steps, max_connections_per_node=max_connections_per_node, skip_full_coverage_check=skip_full_coverage_check, + nodemanager_follow_cluster=nodemanager_follow_cluster, **kwargs ) @@ -753,7 +758,6 @@ def renamenx(self, src, dst): return False - def pubsub_channels(self, pattern='*', aggregate=True): """ Return a list of channels that have at least one subscriber. @@ -761,7 +765,6 @@ def pubsub_channels(self, pattern='*', aggregate=True): """ return self.execute_command('PUBSUB CHANNELS', pattern, aggregate=aggregate) - def pubsub_numpat(self, aggregate=True): """ Returns the number of subscriptions to patterns. @@ -769,15 +772,14 @@ def pubsub_numpat(self, aggregate=True): """ return self.execute_command('PUBSUB NUMPAT', aggregate=aggregate) - def pubsub_numsub(self, *args, **kwargs): """ Return a list of (channel, number of subscribers) tuples for each channel given in ``*args``. - + ``aggregate`` keyword argument toggles merging of response. """ - options = { 'aggregate': kwargs.get('aggregate', True) } + options = {'aggregate': kwargs.get('aggregate', True)} return self.execute_command('PUBSUB NUMSUB', *args, **options) #### diff --git a/rediscluster/connection.py b/rediscluster/connection.py index f105ef23..9f2b707c 100644 --- a/rediscluster/connection.py +++ b/rediscluster/connection.py @@ -17,7 +17,7 @@ # 3rd party imports from redis._compat import nativestr from redis.client import dict_merge -from redis.connection import ConnectionPool, Connection, DefaultParser +from redis.connection import ConnectionPool, Connection, DefaultParser, SSLConnection from redis.exceptions import ConnectionError @@ -57,6 +57,35 @@ def on_connect(self): raise ConnectionError('READONLY command failed') +class SSLClusterConnection(SSLConnection): + """ + Manages TCP communication over TLS/SSL to and from a Redis cluster + Usage: + pool = ClusterConnectionPool(connection_class=SSLClusterConnection, ...) + client = StrictRedisCluster(connection_pool=pool) + """ + description_format = "SSLClusterConnection" + + def __init__(self, **kwargs): + self.readonly = kwargs.pop('readonly', False) + kwargs['parser_class'] = ClusterParser + kwargs.pop('ssl', None) # Needs to be removed to avoid exception in redis Connection init + super(SSLClusterConnection, self).__init__(**kwargs) + + def on_connect(self): + ''' + Initialize the connection, authenticate and select a database and send READONLY if it is + set during object initialization. + ''' + super(SSLClusterConnection, self).on_connect() + + if self.readonly: + self.send_command('READONLY') + + if nativestr(self.read_response()) != 'OK': + raise ConnectionError('READONLY command failed') + + class UnixDomainSocketConnection(Connection): """ """ @@ -71,11 +100,15 @@ class ClusterConnectionPool(ConnectionPool): def __init__(self, startup_nodes=None, init_slot_cache=True, connection_class=ClusterConnection, max_connections=None, max_connections_per_node=False, reinitialize_steps=None, - skip_full_coverage_check=False, **connection_kwargs): + skip_full_coverage_check=False, nodemanager_follow_cluster=False, **connection_kwargs): """ :skip_full_coverage_check: Skips the check of cluster-require-full-coverage config, useful for clusters without the CONFIG command (like aws) + :nodemanager_follow_cluster: + The node manager will during initialization try the last set of nodes that + it was operating on. This will allow the client to drift along side the cluster + if the cluster nodes move around alot. """ super(ClusterConnectionPool, self).__init__(connection_class=connection_class, max_connections=max_connections) @@ -92,8 +125,18 @@ def __init__(self, startup_nodes=None, init_slot_cache=True, connection_class=Cl self.max_connections = max_connections or 2 ** 31 self.max_connections_per_node = max_connections_per_node - self.nodes = NodeManager(startup_nodes, reinitialize_steps=reinitialize_steps, - skip_full_coverage_check=skip_full_coverage_check, **connection_kwargs) + if connection_class == SSLClusterConnection: + connection_kwargs['ssl'] = True # needed in StrictRedis init + + self.nodes = NodeManager( + startup_nodes, + reinitialize_steps=reinitialize_steps, + skip_full_coverage_check=skip_full_coverage_check, + max_connections=self.max_connections, + nodemanager_follow_cluster=nodemanager_follow_cluster, + **connection_kwargs + ) + if init_slot_cache: self.nodes.initialize() @@ -297,7 +340,7 @@ class ClusterReadOnlyConnectionPool(ClusterConnectionPool): """ def __init__(self, startup_nodes=None, init_slot_cache=True, connection_class=ClusterConnection, - max_connections=None, **connection_kwargs): + max_connections=None, nodemanager_follow_cluster=False, **connection_kwargs): """ """ super(ClusterReadOnlyConnectionPool, self).__init__( @@ -306,6 +349,7 @@ def __init__(self, startup_nodes=None, init_slot_cache=True, connection_class=Cl connection_class=connection_class, max_connections=max_connections, readonly=True, + nodemanager_follow_cluster=nodemanager_follow_cluster, **connection_kwargs) def get_node_by_slot(self, slot): diff --git a/rediscluster/crc.py b/rediscluster/crc.py index 53ca2813..7a8208d9 100644 --- a/rediscluster/crc.py +++ b/rediscluster/crc.py @@ -43,7 +43,7 @@ def _crc16_py3(data): """ """ crc = 0 - for byte in data.encode("utf-8"): + for byte in data: crc = ((crc << 8) & 0xff00) ^ x_mode_m_crc16_lookup[((crc >> 8) & 0xff) ^ byte] return crc & 0xffff @@ -52,7 +52,7 @@ def _crc16_py2(data): """ """ crc = 0 - for byte in data.encode("utf-8"): + for byte in data: crc = ((crc << 8) & 0xff00) ^ x_mode_m_crc16_lookup[((crc >> 8) & 0xff) ^ ord(byte)] return crc & 0xffff diff --git a/rediscluster/nodemanager.py b/rediscluster/nodemanager.py index 6cf318f3..d97c35d2 100644 --- a/rediscluster/nodemanager.py +++ b/rediscluster/nodemanager.py @@ -2,7 +2,6 @@ # python std lib import random -import sys # rediscluster imports from .crc import crc16 @@ -10,7 +9,7 @@ # 3rd party imports from redis import StrictRedis -from redis._compat import unicode +from redis._compat import b, unicode, bytes, long, basestring from redis import ConnectionError @@ -19,11 +18,15 @@ class NodeManager(object): """ RedisClusterHashSlots = 16384 - def __init__(self, startup_nodes=None, reinitialize_steps=None, skip_full_coverage_check=False, **connection_kwargs): + def __init__(self, startup_nodes=None, reinitialize_steps=None, skip_full_coverage_check=False, nodemanager_follow_cluster=False, **connection_kwargs): """ :skip_full_coverage_check: Skips the check of cluster-require-full-coverage config, useful for clusters without the CONFIG command (like aws) + :nodemanager_follow_cluster: + The node manager will during initialization try the last set of nodes that + it was operating on. This will allow the client to drift along side the cluster + if the cluster nodes move around alot. """ self.connection_kwargs = connection_kwargs self.nodes = {} @@ -33,49 +36,40 @@ def __init__(self, startup_nodes=None, reinitialize_steps=None, skip_full_covera self.reinitialize_counter = 0 self.reinitialize_steps = reinitialize_steps or 25 self._skip_full_coverage_check = skip_full_coverage_check + self.nodemanager_follow_cluster = nodemanager_follow_cluster if not self.startup_nodes: raise RedisClusterException("No startup nodes provided") - # Minor performance tweak to avoid having to check inside the method - # for each call to keyslot method. - if sys.version_info[0] < 3: - self.keyslot = self.keyslot_py_2 - else: - self.keyslot = self.keyslot_py_3 - - def keyslot_py_2(self, key): + def encode(self, value): """ - Calculate keyslot for a given key. - Tuned for compatibility with python 2.7.x + Return a bytestring representation of the value. + This method is copied from Redis' connection.py:Connection.encode """ - k = unicode(key) - - start = k.find("{") + if isinstance(value, bytes): + return value + elif isinstance(value, (int, long)): + value = b(str(value)) + elif isinstance(value, float): + value = b(repr(value)) + elif not isinstance(value, basestring): + value = unicode(value) + if isinstance(value, unicode): + # The encoding should be configurable as in connection.py:Connection.encode + value = value.encode('utf-8') + return value - if start > -1: - end = k.find("}", start + 1) - if end > -1 and end != start + 1: - k = k[start + 1:end] - - return crc16(k) % self.RedisClusterHashSlots - - def keyslot_py_3(self, key): + def keyslot(self, key): """ Calculate keyslot for a given key. - Tuned for compatibility with supported python 3.x versions + Tuned for compatibility with python 2.7.x """ - try: - # Handle bytes case - k = str(key, encoding='utf-8') - except TypeError: - # Convert others to str. - k = str(key) + k = self.encode(key) - start = k.find("{") + start = k.find(b"{") if start > -1: - end = k.find("}", start + 1) + end = k.find(b"}", start + 1) if end > -1 and end != start + 1: k = k[start + 1:end] @@ -172,7 +166,13 @@ def initialize(self): disagreements = [] startup_nodes_reachable = False - for node in self.orig_startup_nodes: + nodes = self.orig_startup_nodes + + # With this option the client will attempt to connect to any of the previous set of nodes instead of the original set of nodes + if self.nodemanager_follow_cluster: + nodes = self.startup_nodes + + for node in nodes: try: r = self.get_redis_link(host=node["host"], port=node["port"], decode_responses=True) cluster_slots = r.execute_command("cluster", "slots") diff --git a/rediscluster/utils.py b/rediscluster/utils.py index 9cf5f26b..ad12fd2f 100644 --- a/rediscluster/utils.py +++ b/rediscluster/utils.py @@ -126,7 +126,7 @@ def parse_cluster_slots(resp, **options): current_host = options.get('current_host', '') def fix_server(*args): - return (args[0] or current_host, args[1]) + return (nativestr(args[0]) or current_host, args[1]) slots = {} for slot in resp: @@ -145,6 +145,7 @@ def parse_cluster_nodes(resp, **options): @see: http://redis.io/commands/cluster-nodes # string @see: http://redis.io/commands/cluster-slaves # list of string """ + resp = nativestr(resp) current_host = options.get('current_host', '') def parse_slots(s): @@ -252,4 +253,3 @@ def parse_pubsub_numsub(command, res, **options): for channel, numsub in numsub_d.items(): ret_numsub.append((channel, numsub)) return ret_numsub - diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..3c6e79cf --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index 0109830f..f1bba3fb 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="redis-py-cluster", - version="1.3.3", + version="1.3.4", description="Cluster library for redis 3.0.0 built on top of redis-py lib", long_description=readme + '\n\n' + history, author="Johan Andersson", diff --git a/tests/test_node_manager.py b/tests/test_node_manager.py index 6a9677a6..52bfb367 100644 --- a/tests/test_node_manager.py +++ b/tests/test_node_manager.py @@ -39,6 +39,13 @@ def test_keyslot(): assert n.keyslot("{foo}bar") == 12182 assert n.keyslot("{foo}") == 12182 assert n.keyslot(1337) == 4314 + + assert n.keyslot(125) == n.keyslot(b"125") + assert n.keyslot(125) == n.keyslot("\x31\x32\x35") + assert n.keyslot("大奖") == n.keyslot(b"\xe5\xa4\xa7\xe5\xa5\x96") + assert n.keyslot(u"大奖") == n.keyslot(b"\xe5\xa4\xa7\xe5\xa5\x96") + assert n.keyslot(1337.1234) == n.keyslot("1337.1234") + assert n.keyslot(1337) == n.keyslot("1337") assert n.keyslot(b"abc") == n.keyslot("abc") assert n.keyslot("abc") == n.keyslot(unicode("abc")) assert n.keyslot(unicode("abc")) == n.keyslot(b"abc") @@ -346,6 +353,12 @@ def patch_execute_command(*args, **kwargs): }] +def test_initialize_follow_cluster(): + n = NodeManager(nodemanager_follow_cluster=True, startup_nodes=[{'host': '127.0.0.1', 'port': 7000}]) + n.orig_startup_nodes = None + n.initialize() + + def test_init_with_down_node(): """ If I can't connect to one of the nodes, everything should still work. diff --git a/tests/test_utils.py b/tests/test_utils.py index 6b11f71e..7ee9278e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,7 +19,7 @@ # 3rd party imports import pytest -from redis._compat import unicode +from redis._compat import unicode, b def test_parse_cluster_slots(): @@ -70,6 +70,29 @@ def test_parse_cluster_slots(): parse_cluster_slots(extended_mock_response) + mock_binary_response = [ + [0, 5460, [b('172.17.0.2'), 7000], [b('172.17.0.2'), 7003]], + [5461, 10922, [b('172.17.0.2'), 7001], [b('172.17.0.2'), 7004]], + [10923, 16383, [b('172.17.0.2'), 7002], [b('172.17.0.2'), 7005]] + ] + parse_cluster_slots(mock_binary_response) + + extended_mock_binary_response = [ + [0, 5460, [b('172.17.0.2'), 7000, b('ffd36d8d7cb10d813f81f9662a835f6beea72677')], [b('172.17.0.2'), 7003, b('5c15b69186017ddc25ebfac81e74694fc0c1a160')]], + [5461, 10922, [b('172.17.0.2'), 7001, b('069cda388c7c41c62abe892d9e0a2d55fbf5ffd5')], [b('172.17.0.2'), 7004, b('dc152a08b4cf1f2a0baf775fb86ad0938cb907dc')]], + [10923, 16383, [b('172.17.0.2'), 7002, b('3588b4cf9fc72d57bb262a024747797ead0cf7ea')], [b('172.17.0.2'), 7005, b('a72c02c7d85f4ec3145ab2c411eefc0812aa96b0')]] + ] + + extended_mock_parsed = { + (0, 5460): {'master': ('172.17.0.2', 7000), 'slaves': [('172.17.0.2', 7003)]}, + (5461, 10922): {'master': ('172.17.0.2', 7001), + 'slaves': [('172.17.0.2', 7004)]}, + (10923, 16383): {'master': ('172.17.0.2', 7002), + 'slaves': [('172.17.0.2', 7005)]} + } + + assert parse_cluster_slots(extended_mock_binary_response) == extended_mock_parsed + def test_string_keys_to(): def mock_true():