rotating logfile handler (#30)

* explicitly catch redis ConnectionErrors

* RotatingFileHandler

consume-redis-events.py

@@ -34,6 +34,7 @@
 import sys
 import time
 import logging
+import logging.handlers
 
 from redis import StrictRedis
 
@@ -50,7 +51,10 @@ def initialize_logger(debug_mode=False):
     console = logging.StreamHandler(stream=sys.stdout)
     console.setFormatter(formatter)
 
-    fh = logging.FileHandler('redis-consumer.log')
+    fh = logging.handlers.RotatingFileHandler(
+        filename='redis-consumer.log',
+        maxBytes=10000000,
+        backupCount=10)
     fh.setFormatter(formatter)
 
     if debug_mode:

redis_consumer/consumers.py

@@ -38,8 +38,8 @@
 import zipfile
 
 import grpc
+import redis
 import numpy as np
-from redis.exceptions import ConnectionError
 
 from redis_consumer.grpc_clients import PredictClient
 from redis_consumer.grpc_clients import ProcessClient
@@ -112,9 +112,12 @@ def _consume(self, redis_hash):
     def _redis_type(self, redis_key):
         while True:
             try:
+                # start = timeit.default_timer()
                 response = self.redis.type(redis_key)
+                # self.logger.debug('Finished `TYPE %s` in %s seconds.',
+                #                   redis_key, timeit.default_timer() - start)
                 break
-            except ConnectionError as err:
+            except redis.exceptions.ConnectionError as err:
                 self.logger.warning('Encountered %s: %s when calling '
                                     'redis.type(). Retrying in %s seconds.',
                                     type(err).__name__, err,
@@ -127,10 +130,10 @@ def scan_iter(self, match=None):
             try:
                 start = timeit.default_timer()
                 response = self.redis.scan_iter(match=match)
-                self.logger.debug('Finished SCAN in %s seconds.',
-                                  timeit.default_timer() - start)
+                self.logger.debug('Finished `SCAN %s` in %s seconds.',
+                                  match, timeit.default_timer() - start)
                 break
-            except ConnectionError as err:
+            except redis.exceptions.ConnectionError as err:
                 self.logger.warning('Encountered %s: %s when calling '
                                     'SCAN. Retrying in %s seconds.',
                                     type(err).__name__, err,
@@ -146,7 +149,7 @@ def keys(self):
                 self.logger.debug('KEYS got %s results in %s seconds.',
                                   len(response), timeit.default_timer() - start)
                 break
-            except ConnectionError as err:
+            except redis.exceptions.ConnectionError as err:
                 self.logger.warning('Encountered %s: %s when calling '
                                     'KEYS. Retrying in %s seconds.',
                                     type(err).__name__, err,
@@ -157,9 +160,13 @@ def keys(self):
     def hset(self, rhash, key, value):
         while True:
             try:
+                start = timeit.default_timer()
                 response = self.redis.hset(rhash, key, value)
+                self.logger.debug('Finished `HSET %s %s %s` in %s seconds.',
+                                  rhash, key, value,
+                                  timeit.default_timer() - start)
                 break
-            except ConnectionError as err:
+            except redis.exceptions.ConnectionError as err:
                 self.logger.warning('Encountered %s: %s when calling '
                                     'HSET. Retrying in %s seconds.',
                                     type(err).__name__, err,
@@ -170,9 +177,12 @@ def hset(self, rhash, key, value):
     def hget(self, rhash, key):
         while True:
             try:
+                # start = timeit.default_timer()
                 response = self.redis.hget(rhash, key)
+                # self.logger.debug('Finished `HGET %s %s` in %s seconds.',
+                #                   rhash, key, timeit.default_timer() - start)
                 break
-            except ConnectionError as err:
+            except redis.exceptions.ConnectionError as err:
                 self.logger.warning('Encountered %s: %s when calling '
                                     'redis.hget(). Retrying in %s seconds.',
                                     type(err).__name__, err,
@@ -183,11 +193,12 @@ def hget(self, rhash, key):
     def hmset(self, rhash, data):
         while True:
             try:
+                start = timeit.default_timer()
                 response = self.redis.hmset(rhash, data)
-                self.logger.debug('Updated hash %s with values: %s.',
-                                  rhash, data)
+                self.logger.debug('`HMSET %s %s` finished in %s seconds.',
+                                  rhash, data, timeit.default_timer() - start)
                 break
-            except ConnectionError as err:
+            except redis.exceptions.ConnectionError as err:
                 self.logger.warning('Encountered %s: %s when calling '
                                     'redis.hmset(). Retrying in %s seconds.',
                                     type(err).__name__, err,
@@ -198,9 +209,12 @@ def hmset(self, rhash, data):
     def hgetall(self, rhash):
         while True:
             try:
+                start = timeit.default_timer()
                 response = self.redis.hgetall(rhash)
+                self.logger.debug('Finished `HGETALL %s` in %s seconds.',
+                                  rhash, timeit.default_timer() - start)
                 break
-            except ConnectionError as err:
+            except redis.exceptions.ConnectionError as err:
                 self.logger.warning('Encountered %s: %s when calling '
                                     'redis.hgetall(). Retrying in %s seconds.',
                                     type(err).__name__, err,