on ConnectionError, update the master and slave nodes.

bucket_monitor/redis.py

@@ -134,8 +134,6 @@ def __init__(self, host, port, backoff=1):
 
     def _update_masters_and_slaves(self):
         try:
-            self._sentinel.sentinel_masters()
-
             sentinel_masters = self._sentinel.sentinel_masters()
 
             for master_set in sentinel_masters:
@@ -162,20 +160,21 @@ def _get_redis_client(self, host, port):  # pylint: disable=R0201
                                  charset='utf-8')
 
     def __getattr__(self, name):
-        if name in REDIS_READONLY_COMMANDS:
-            redis_client = random.choice(self._redis_slaves)
-        else:
-            redis_client = self._redis_master
-
-        redis_function = getattr(redis_client, name)
 
         def wrapper(*args, **kwargs):
             values = list(args) + list(kwargs.values())
             values = [str(v) for v in values]
             while True:
                 try:
+                    if name in REDIS_READONLY_COMMANDS:
+                        redis_client = random.choice(self._redis_slaves)
+                    else:
+                        redis_client = self._redis_master
+
+                    redis_function = getattr(redis_client, name)
                     return redis_function(*args, **kwargs)
                 except redis.exceptions.ConnectionError as err:
+                    self._update_masters_and_slaves()
                     self.logger.warning('Encountered %s: %s when calling '
                                         '`%s %s`. Retrying in %s seconds.',
                                         type(err).__name__, err,

bucket_monitor/redis_test.py

@@ -36,22 +36,25 @@
 import bucket_monitor
 
 
+FAIL_COUNT = 0
+
+
 class DummyRedis(object):
     def __init__(self, fail_tolerance=0, hard_fail=False, err=None):
-        self.fail_count = 0
         self.fail_tolerance = fail_tolerance
         self.hard_fail = hard_fail
         if err is None:
             err = redis.exceptions.ConnectionError('thrown on purpose')
         self.err = err
 
     def get_fail_count(self):
+        global FAIL_COUNT
         if self.hard_fail:
             raise AssertionError('thrown on purpose')
-        if self.fail_count < self.fail_tolerance:
-            self.fail_count += 1
+        if FAIL_COUNT < self.fail_tolerance:
+            FAIL_COUNT += 1
             raise self.err
-        return self.fail_count
+        return FAIL_COUNT
 
     def sentinel_masters(self):
         return {'mymaster': {'ip': 'master', 'port': 6379}}
@@ -64,6 +67,8 @@ def sentinel_slaves(self, _):
 class TestRedis(object):
 
     def test_redis_client(self):  # pylint: disable=R0201
+        global FAIL_COUNT
+
         fails = random.randint(1, 3)
         RedisClient = bucket_monitor.redis.RedisClient
 
@@ -75,6 +80,7 @@ def _get_redis_client(*args, **kwargs):  # pylint: disable=W0613
 
         client = RedisClient(host='host', port='port', backoff=0)
         assert client.get_fail_count() == fails
+        FAIL_COUNT = 0  # reset for the next test
 
         with pytest.raises(AttributeError):
             client.unknown_function()
@@ -88,6 +94,7 @@ def _get_redis_client_retry(*args, **kwargs):  # pylint: disable=W0613
 
         client = RedisClient(host='host', port='port', backoff=0)
         assert client.get_fail_count() == fails
+        FAIL_COUNT = 0  # reset for the next test
 
         # test ResponseError other - should fail
         def _get_redis_client_err(*args, **kwargs):  # pylint: disable=W0613