Catch Redis server errors

gnocchi/incoming/redis.py

@@ -17,10 +17,13 @@
 import uuid
 
 import daiquiri
+from redis.exceptions import ConnectionError
 import six
+import tenacity
 
 from gnocchi.common import redis
 from gnocchi import incoming
+from gnocchi import utils
 
 
 LOG = daiquiri.getLogger(__name__)
@@ -93,6 +96,11 @@ def add_measures_batch(self, metrics_and_measures):
                 notified_sacks.add(sack_name)
         pipe.execute()
 
+    # if ConnectionError exception occurs, try again, max 5 times.
+    @tenacity.retry(
+        wait=utils.wait_exponential,
+        stop=tenacity.stop_after_attempt(5),
+        retry=tenacity.retry_if_exception_type(ConnectionError))
     def _build_report(self, details):
         report_vars = {'measures': 0, 'metric_details': {}}
 
@@ -176,6 +184,11 @@ def process_measures_for_sack(self, sack):
             pipe.ltrim(key, item_len + 1, -1)
         pipe.execute()
 
+    # if ConnectionError exception occurs, try again, max 5 times.
+    @tenacity.retry(
+        wait=utils.wait_exponential,
+        stop=tenacity.stop_after_attempt(5),
+        retry=tenacity.retry_if_exception_type(ConnectionError))
     def iter_on_sacks_to_process(self):
         self._client.config_set("notify-keyspace-events", "K$")
         p = self._client.pubsub()

setup.cfg

@@ -70,7 +70,7 @@ s3 =
     boto3
     botocore>=1.5
 redis =
-    redis>=2.10.0 # MIT
+    redis >= 3.2.0 # MIT
     hiredis
 swift =
     python-swiftclient>=3.1.0