Merge 403ed52 into 7e1e497

istresearch · Nov 21, 2018 · 94867e0 · 94867e0
2 parents 7e1e497 + 403ed52
commit 94867e0
Show file tree

Hide file tree

Showing 20 changed files with 77 additions and 19 deletions.
diff --git a/crawler/crawling/distributed_scheduler.py b/crawler/crawling/distributed_scheduler.py
@@ -324,7 +324,9 @@ def from_settings(cls, settings):
                              port=settings.get('REDIS_PORT'),
                              db=settings.get('REDIS_DB'),
                              password=settings.get('REDIS_PASSWORD'),
-                             decode_responses=True)
+                             decode_responses=True,
+                             socket_timeout=settings.get('REDIS_SOCKET_TIMEOUT'),
+                             socket_connect_timeout=settings.get('REDIS_SOCKET_TIMEOUT'))
         persist = settings.get('SCHEDULER_PERSIST', True)
         up_int = settings.get('SCHEDULER_QUEUE_REFRESH', 10)
         hits = settings.get('QUEUE_HITS', 10)

diff --git a/crawler/crawling/log_retry_middleware.py b/crawler/crawling/log_retry_middleware.py
@@ -63,7 +63,9 @@ def setup(self, settings):
                                           port=self.settings.get('REDIS_PORT'),
                                           db=settings.get('REDIS_DB'),
                                           password=self.settings.get('REDIS_PASSWORD'),
-                                          decode_responses=True)
+                                          decode_responses=True,
+                                          socket_timeout=self.settings.get('REDIS_SOCKET_TIMEOUT'),
+                                          socket_connect_timeout=self.settings.get('REDIS_SOCKET_TIMEOUT'))
 
             try:
                 self.redis_conn.info()

diff --git a/crawler/crawling/redis_stats_middleware.py b/crawler/crawling/redis_stats_middleware.py
@@ -40,10 +40,12 @@ def setup(self, settings):
 
         # set up redis
         self.redis_conn = redis.Redis(host=settings.get('REDIS_HOST'),
-            port=settings.get('REDIS_PORT'),
-            db=settings.get('REDIS_DB'),
-            password=settings.get('REDIS_PASSWORD'),
-            decode_responses=True)
+                                      port=settings.get('REDIS_PORT'),
+                                      db=settings.get('REDIS_DB'),
+                                      password=settings.get('REDIS_PASSWORD'),
+                                      decode_responses=True,
+                                      socket_timeout=settings.get('REDIS_SOCKET_TIMEOUT'),
+                                      socket_connect_timeout=settings.get('REDIS_SOCKET_TIMEOUT'))
 
         try:
             self.redis_conn.info()

diff --git a/crawler/crawling/settings.py b/crawler/crawling/settings.py
@@ -10,6 +10,7 @@
 REDIS_PORT = '6379'
 REDIS_DB = 0
 REDIS_PASSWORD = None
+REDIS_SOCKET_TIMEOUT = 10
 
 # Kafka server information
 KAFKA_HOSTS = ['localhost:9092']

diff --git a/docker/crawler/settings.py b/docker/crawler/settings.py
@@ -15,6 +15,7 @@ def str2bool(v):
 REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
 REDIS_DB = int(os.getenv('REDIS_DB', 0))
 REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', None)
+REDIS_SOCKET_TIMEOUT = int(os.getenv('REDIS_SOCKET_TIMEOUT', 10))
 
 # Kafka server information
 KAFKA_HOSTS = [x.strip() for x in os.getenv('KAFKA_HOSTS', 'kafka:9092').split(',')]

diff --git a/docker/kafka-monitor/settings.py b/docker/kafka-monitor/settings.py
@@ -9,6 +9,7 @@ def str2bool(v):
 REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
 REDIS_DB = int(os.getenv('REDIS_DB', 0))
 REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', None)
+REDIS_SOCKET_TIMEOUT = int(os.getenv('REDIS_SOCKET_TIMEOUT', 10))
 
 # Kafka server information
 KAFKA_HOSTS = [x.strip() for x in os.getenv('KAFKA_HOSTS', 'kafka:9092').split(',')]

diff --git a/docker/redis-monitor/settings.py b/docker/redis-monitor/settings.py
@@ -11,6 +11,7 @@ def str2bool(v):
 REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
 REDIS_DB = int(os.getenv('REDIS_DB', 0))
 REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', None)
+REDIS_SOCKET_TIMEOUT = int(os.getenv('REDIS_SOCKET_TIMEOUT', 10))
 
 KAFKA_HOSTS = [x.strip() for x in os.getenv('KAFKA_HOSTS', 'kafka:9092').split(',')]
 KAFKA_TOPIC_PREFIX = os.getenv('KAFKA_TOPIC_PREFIX', 'demo')

diff --git a/docs/topics/changelog.rst b/docs/topics/changelog.rst
@@ -20,8 +20,19 @@ Date: ??/??/????
 
 - Minor documentation changes/updates
 
+- Added REDIS_SOCKET_TIMEOUT setting to control socket_timeout and socket_connect_timeout
+
+Kafka Monitor Changes
+^^^^^^^^^^^^^^^^^^^^^
+- Added REDIS_SOCKET_TIMEOUT setting
+
+Redis Monitor Changes
+^^^^^^^^^^^^^^^^^^^^^
+- Added REDIS_SOCKET_TIMEOUT setting
+
 Crawler
 ^^^^^^^
+- Added REDIS_SOCKET_TIMEOUT setting
 
 - Improved request to dictionary serialization support
 
@@ -36,6 +47,7 @@ Rest
 
 Utils
 ^^^^^
+- Added REDIS_SOCKET_TIMEOUT setting
 
 - Fixed LogFactory callback with correct extras dictionary
 

diff --git a/docs/topics/crawler/settings.rst b/docs/topics/crawler/settings.rst
@@ -30,6 +30,12 @@ Default: ``None``
 
 The password to use when connecting to the ``REDIS_HOST``.
 
+**REDIS_SOCKET_TIMEOUT**
+
+Default: ``10``
+
+The number of seconds to wait while establishing a TCP connection, or to wait for a response from an existing TCP connection before timing out.
+
 Kafka
 -----
 

diff --git a/docs/topics/kafka-monitor/settings.rst b/docs/topics/kafka-monitor/settings.rst
@@ -45,6 +45,12 @@ Default: ``None``
 
 The password to use when connecting to the ``REDIS_HOST``.
 
+**REDIS_SOCKET_TIMEOUT**
+
+Default: ``10``
+
+The number of seconds to wait while establishing a TCP connection, or to wait for a response from an existing TCP connection before timing out.
+
 Kafka
 -----
 

diff --git a/docs/topics/redis-monitor/settings.rst b/docs/topics/redis-monitor/settings.rst
@@ -65,6 +65,12 @@ Default: ``6``
 
 The number of seconds a vacant worker lock will stay within Redis before becoming available to a new worker
 
+**REDIS_SOCKET_TIMEOUT**
+
+Default: ``10``
+
+The number of seconds to wait while establishing a TCP connection, or to wait for a response from an existing TCP connection before timing out.
+
 Kafka
 -----
 

diff --git a/kafka-monitor/kafka_monitor.py b/kafka-monitor/kafka_monitor.py
@@ -125,7 +125,9 @@ def _setup_stats(self):
                                  port=self.settings['REDIS_PORT'],
                                  db=self.settings.get('REDIS_DB'),
                                  password=self.settings['REDIS_PASSWORD'],
-                                 decode_responses=True)
+                                 decode_responses=True,
+                                 socket_timeout=self.settings.get('REDIS_SOCKET_TIMEOUT'),
+                                 socket_connect_timeout=self.settings.get('REDIS_SOCKET_TIMEOUT'))
 
         try:
             redis_conn.info()

diff --git a/kafka-monitor/plugins/action_handler.py b/kafka-monitor/plugins/action_handler.py
@@ -19,7 +19,9 @@ def setup(self, settings):
                                       port=settings['REDIS_PORT'],
                                       db=settings.get('REDIS_DB'),
                                       password=settings['REDIS_PASSWORD'],
-                                      decode_responses=True)
+                                      decode_responses=True,
+                                      socket_timeout=settings.get('REDIS_SOCKET_TIMEOUT'),
+                                      socket_connect_timeout=settings.get('REDIS_SOCKET_TIMEOUT'))
 
         try:
             self.redis_conn.info()

diff --git a/kafka-monitor/plugins/scraper_handler.py b/kafka-monitor/plugins/scraper_handler.py
@@ -20,7 +20,9 @@ def setup(self, settings):
                                       port=settings['REDIS_PORT'],
                                       db=settings.get('REDIS_DB'),
                                       password=settings['REDIS_PASSWORD'],
-                                      decode_responses=True)
+                                      decode_responses=True,
+                                      socket_timeout=settings.get('REDIS_SOCKET_TIMEOUT'),
+                                      socket_connect_timeout=settings.get('REDIS_SOCKET_TIMEOUT'))
 
         try:
             self.redis_conn.info()

diff --git a/kafka-monitor/plugins/stats_handler.py b/kafka-monitor/plugins/stats_handler.py
@@ -17,7 +17,9 @@ def setup(self, settings):
                                       port=settings['REDIS_PORT'],
                                       db=settings.get('REDIS_DB'),
                                       password=settings['REDIS_PASSWORD'],
-                                      decode_responses=True)
+                                      decode_responses=True,
+                                      socket_timeout=settings.get('REDIS_SOCKET_TIMEOUT'),
+                                      socket_connect_timeout=settings.get('REDIS_SOCKET_TIMEOUT'))
 
         try:
             self.redis_conn.info()

diff --git a/kafka-monitor/plugins/zookeeper_handler.py b/kafka-monitor/plugins/zookeeper_handler.py
@@ -20,7 +20,9 @@ def setup(self, settings):
                                       port=settings['REDIS_PORT'],
                                       db=settings.get('REDIS_DB'),
                                       password=settings['REDIS_PASSWORD'],
-                                      decode_responses=True)
+                                      decode_responses=True,
+                                      socket_timeout=settings.get('REDIS_SOCKET_TIMEOUT'),
+                                      socket_connect_timeout=settings.get('REDIS_SOCKET_TIMEOUT'))
 
         try:
             self.redis_conn.info()

diff --git a/kafka-monitor/settings.py b/kafka-monitor/settings.py
@@ -6,6 +6,7 @@
 REDIS_PORT = 6379
 REDIS_DB = 0
 REDIS_PASSWORD = None
+REDIS_SOCKET_TIMEOUT = 10
 
 # Kafka server information
 KAFKA_HOSTS = ['localhost:9092']

diff --git a/redis-monitor/redis_monitor.py b/redis-monitor/redis_monitor.py
@@ -59,16 +59,20 @@ def setup(self, level=None, log_file=None, json=None):
                                               backups=self.settings['LOG_BACKUPS'])
 
         self.redis_conn = redis.StrictRedis(host=self.settings['REDIS_HOST'],
-                                      port=self.settings['REDIS_PORT'],
-                                      db=self.settings['REDIS_DB'],
-                                      password=self.settings['REDIS_PASSWORD'],
-                                      decode_responses=True)
+                                            port=self.settings['REDIS_PORT'],
+                                            db=self.settings['REDIS_DB'],
+                                            password=self.settings['REDIS_PASSWORD'],
+                                            decode_responses=True,
+                                            socket_timeout = self.settings.get('REDIS_SOCKET_TIMEOUT'),
+                                            socket_connect_timeout = self.settings.get('REDIS_SOCKET_TIMEOUT'))
         # redis_lock needs a redis connection without setting decode_responses
         # to True
         self.lock_redis_conn = redis.StrictRedis(host=self.settings['REDIS_HOST'],
                                                  port=self.settings['REDIS_PORT'],
                                                  db=self.settings['REDIS_DB'],
-                                                 password=self.settings['REDIS_PASSWORD'])
+                                                 password=self.settings['REDIS_PASSWORD'],
+                                                 socket_timeout=self.settings.get('REDIS_SOCKET_TIMEOUT'),
+                                                 socket_connect_timeout=self.settings.get('REDIS_SOCKET_TIMEOUT'))
 
         try:
             self.redis_conn.info()

diff --git a/redis-monitor/settings.py b/redis-monitor/settings.py
@@ -6,6 +6,7 @@
 REDIS_PORT = 6379
 REDIS_DB = 0
 REDIS_PASSWORD = None
+REDIS_SOCKET_TIMEOUT = 10
 
 KAFKA_HOSTS = 'localhost:9092'
 KAFKA_TOPIC_PREFIX = 'demo'

diff --git a/utils/scutils/stats_collector.py b/utils/scutils/stats_collector.py
@@ -39,6 +39,7 @@ class StatsCollector(object):
     SECONDS_7_DAY = SECONDS_1_DAY * 7
     SECONDS_1_WEEK = SECONDS_7_DAY
     SECONDS_30_DAY = SECONDS_1_DAY * 30
+    REDIS_SOCKET_TIMEOUT = 10
 
     @classmethod
     def get_time_window(self, redis_conn=None, host='localhost', port=6379,
@@ -220,10 +221,11 @@ def setup(self, redis_conn=None, host='localhost', port=6379, password=None):
         if redis_conn is None:
             if host is not None and port is not None:
                 self.redis_conn = redis.Redis(host=host, port=port, password=password,
-                                              decode_responses=True)
+                                              decode_responses=True,
+                                              socket_timeout=self.REDIS_SOCKET_TIMEOUT,
+                                              socket_connect_timeout=self.REDIS_SOCKET_TIMEOUT)
             else:
-                raise Exception("Please specify some form of connection "
-                                    "to Redis")
+                raise Exception("Please specify some form of connection to Redis")
         else:
             self.redis_conn = redis_conn