diff --git a/README.rst b/README.rst index f6b6386..644a037 100644 --- a/README.rst +++ b/README.rst @@ -36,10 +36,13 @@ Your cache backend should look something like this:: 'default': { 'BACKEND': 'django_elasticache.memcached.ElastiCache', 'LOCATION': 'cache-c.draaaf.cfg.use1.cache.amazonaws.com:11211', + 'OPTIONS' { + 'IGNORE_CLUSTER_ERRORS': [True,False], + }, } } -By the first call to cache it connects to cluster (using LOCATION param), +By the first call to cache it connects to cluster (using ``LOCATION`` param), gets list of all nodes and setup pylibmc client using full list of nodes. As result your cache will work with all nodes in cluster and automatically detect new nodes in cluster. List of nodes are stored in class-level @@ -48,6 +51,10 @@ But if you're using gunicorn or mod_wsgi you usually have max_request settings w restart process after some count of processed requests, so auto discovery will work fine. +The ``IGNORE_CLUSTER_ERRORS`` option is useful when ``LOCATION`` doesn't have support +for ``config get cluster``. When set to ``True``, and ``config get cluster`` fails, +it returns a list of a single node with the same endpoint supplied to ``LOCATION``. + Django-elasticache changes default pylibmc params to increase performance. Another solutions diff --git a/django_elasticache/cluster_utils.py b/django_elasticache/cluster_utils.py index 7d4a11d..740e4cf 100644 --- a/django_elasticache/cluster_utils.py +++ b/django_elasticache/cluster_utils.py @@ -17,7 +17,7 @@ def __init__(self, cmd, response): 'Unexpected response {} for command {}'.format(response, cmd)) -def get_cluster_info(host, port): +def get_cluster_info(host, port, ignore_cluster_errors=False): """ return dict with info about nodes in cluster and current version { @@ -40,8 +40,21 @@ def get_cluster_info(host, port): else: cmd = b'get AmazonElastiCache:cluster\n' client.write(cmd) - res = client.read_until(b'\n\r\nEND\r\n') + regex_index, match_object, res = client.expect([ + re.compile(b'\n\r\nEND\r\n'), + re.compile(b'ERROR\r\n') + ]) client.close() + + if res == b'ERROR\r\n' and ignore_cluster_errors: + return { + 'version': version, + 'nodes': [ + '{}:{}'.format(smart_text(host), + smart_text(port)) + ] + } + ls = list(filter(None, re.compile(br'\r?\n').split(res))) if len(ls) != 4: raise WrongProtocolData(cmd, res) diff --git a/django_elasticache/memcached.py b/django_elasticache/memcached.py index 0f8a8be..f7c19fc 100644 --- a/django_elasticache/memcached.py +++ b/django_elasticache/memcached.py @@ -38,6 +38,9 @@ def __init__(self, server, params): raise InvalidCacheBackendError( 'Server configuration should be in format IP:port') + self._ignore_cluster_errors = self._options.get( + 'IGNORE_CLUSTER_ERRORS', False) + def update_params(self, params): """ update connection params to maximize performance @@ -67,7 +70,8 @@ def get_cluster_nodes(self): server, port = self._servers[0].split(':') try: self._cluster_nodes_cache = ( - get_cluster_info(server, port)['nodes']) + get_cluster_info(server, port, + self._ignore_cluster_errors)['nodes']) except (socket.gaierror, socket.timeout) as err: raise Exception('Cannot connect to cluster {} ({})'.format( self._servers[0], err diff --git a/tests/test_backend.py b/tests/test_backend.py index 3006652..9b6f425 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -47,7 +47,7 @@ def test_split_servers(get_cluster_info): } backend._lib.Client = Mock() assert backend._cache - get_cluster_info.assert_called_once_with('h', '0') + get_cluster_info.assert_called_once_with('h', '0', False) backend._lib.Client.assert_called_once_with(servers) @@ -70,7 +70,7 @@ def test_node_info_cache(get_cluster_info): eq_(backend._cache.get.call_count, 2) eq_(backend._cache.set.call_count, 2) - get_cluster_info.assert_called_once_with('h', '0') + get_cluster_info.assert_called_once_with('h', '0', False) @patch('django.conf.settings', global_settings) diff --git a/tests/test_protocol.py b/tests/test_protocol.py index bf8d500..7232a1a 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -8,26 +8,44 @@ from unittest.mock import patch, call, MagicMock -TEST_PROTOCOL_1 = [ +TEST_PROTOCOL_1_READ_UNTIL = [ b'VERSION 1.4.14', - b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n', ] -TEST_PROTOCOL_2 = [ +TEST_PROTOCOL_1_EXPECT = [ + (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'), # NOQA +] + +TEST_PROTOCOL_2_READ_UNTIL = [ b'VERSION 1.4.13', - b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n', ] -TEST_PROTOCOL_3 = [ +TEST_PROTOCOL_2_EXPECT = [ + (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'), # NOQA +] + +TEST_PROTOCOL_3_READ_UNTIL = [ b'VERSION 1.4.14 (Ubuntu)', - b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n', +] + +TEST_PROTOCOL_3_EXPECT = [ + (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'), # NOQA +] + +TEST_PROTOCOL_4_READ_UNTIL = [ + b'VERSION 1.4.34', +] + +TEST_PROTOCOL_4_EXPECT = [ + (0, None, b'ERROR\r\n'), ] @patch('django_elasticache.cluster_utils.Telnet') def test_happy_path(Telnet): client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_1 + client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL + client.expect.side_effect = TEST_PROTOCOL_1_EXPECT info = get_cluster_info('', 0) eq_(info['version'], 1) eq_(info['nodes'], ['ip:port', 'host:port']) @@ -42,7 +60,8 @@ def test_bad_protocol(): @patch('django_elasticache.cluster_utils.Telnet') def test_last_versions(Telnet): client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_1 + client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL + client.expect.side_effect = TEST_PROTOCOL_1_EXPECT get_cluster_info('', 0) client.write.assert_has_calls([ call(b'version\n'), @@ -53,7 +72,8 @@ def test_last_versions(Telnet): @patch('django_elasticache.cluster_utils.Telnet') def test_prev_versions(Telnet): client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_2 + client.read_until.side_effect = TEST_PROTOCOL_2_READ_UNTIL + client.expect.side_effect = TEST_PROTOCOL_2_EXPECT get_cluster_info('', 0) client.write.assert_has_calls([ call(b'version\n'), @@ -64,7 +84,8 @@ def test_prev_versions(Telnet): @patch('django_elasticache.cluster_utils.Telnet') def test_ubuntu_protocol(Telnet): client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_3 + client.read_until.side_effect = TEST_PROTOCOL_3_READ_UNTIL + client.expect.side_effect = TEST_PROTOCOL_3_EXPECT try: get_cluster_info('', 0) @@ -75,3 +96,30 @@ def test_ubuntu_protocol(Telnet): call(b'version\n'), call(b'config get cluster\n'), ]) + + +@patch('django_elasticache.cluster_utils.Telnet') +def test_no_configuration_protocol_support_with_errors_ignored(Telnet): + client = Telnet.return_value + client.read_until.side_effect = TEST_PROTOCOL_4_READ_UNTIL + client.expect.side_effect = TEST_PROTOCOL_4_EXPECT + info = get_cluster_info('test', 0, ignore_cluster_errors=True) + client.write.assert_has_calls([ + call(b'version\n'), + call(b'config get cluster\n'), + ]) + eq_(info['version'], '1.4.34') + eq_(info['nodes'], ['test:0']) + + +@raises(WrongProtocolData) +@patch('django_elasticache.cluster_utils.Telnet') +def test_no_configuration_protocol_support_with_errors(Telnet): + client = Telnet.return_value + client.read_until.side_effect = TEST_PROTOCOL_4_READ_UNTIL + client.expect.side_effect = TEST_PROTOCOL_4_EXPECT + get_cluster_info('test', 0, ignore_cluster_errors=False) + client.write.assert_has_calls([ + call(b'version\n'), + call(b'config get cluster\n'), + ])