Merge pull request #2196 from DanCech/timeouts

Rename timeout settings

docs/config-local-settings.rst

@@ -180,6 +180,16 @@ STORAGE_FINDERS
   It is possible to use an alternate storage layer than the default, Whisper, in order to accommodate specific needs.
   See: http://graphite.readthedocs.io/en/latest/storage-backends.html
 
+FETCH_TIMEOUT
+  `Default: 6`
+
+  Timeout for data fetches in seconds.
+
+FIND_TIMEOUT
+  `Default: 3`
+
+  Timeout for find requests (metric browsing) in seconds.
+
 TAGDB
   `Default: 'graphite.tags.localdatabase.LocalDatabaseTagDB'`
   Tag database driver to use, other options include `graphite.tags.redis.RedisTagDB`
@@ -409,16 +419,6 @@ POOL_MAX_WORKERS
 
    The maximum number of worker threads that should be created.
 
-REMOTE_FETCH_TIMEOUT
-  `Default: 6`
-
-  Timeout for remote data fetches in seconds.
-
-REMOTE_FIND_TIMEOUT
-  `Default: 2.5`
-
-  Timeout for remote find requests (metric browsing) in seconds.
-
 REMOTE_RETRY_DELAY
   `Default: 60`
 

webapp/graphite/finders/remote.py

@@ -104,7 +104,7 @@ def find_nodes(self, query, timer=None):
                 url,
                 fields=query_params,
                 headers=query.headers,
-                timeout=settings.REMOTE_FIND_TIMEOUT)
+                timeout=settings.FIND_TIMEOUT)
 
             try:
                 if result.getheader('content-type') == 'application/x-msgpack':
@@ -163,7 +163,7 @@ def get_index(self, requestContext):
               ('local', self.params.get('local', '1')),
             ],
             headers=headers,
-            timeout=settings.REMOTE_FIND_TIMEOUT)
+            timeout=settings.FIND_TIMEOUT)
 
         try:
             reader = codecs.getreader('utf-8')
@@ -197,7 +197,7 @@ def auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestContext=N
             '/tags/autoComplete/tags',
             fields,
             headers=requestContext.get('forwardHeaders') if requestContext else None,
-            timeout=settings.REMOTE_FIND_TIMEOUT)
+            timeout=settings.FIND_TIMEOUT)
         try:
             reader = codecs.getreader('utf-8')
             results = json.load(reader(result))
@@ -231,7 +231,7 @@ def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None, request
             '/tags/autoComplete/values',
             fields,
             headers=requestContext.get('forwardHeaders') if requestContext else None,
-            timeout=settings.REMOTE_FIND_TIMEOUT)
+            timeout=settings.FIND_TIMEOUT)
         try:
             reader = codecs.getreader('utf-8')
             results = json.load(reader(result))

webapp/graphite/local_settings.py.example

@@ -90,6 +90,10 @@ DEFAULT_XFILES_FACTOR = 0
 # Interval for the Auto-Refresh feature in the Composer, measured in seconds.
 #AUTO_REFRESH_INTERVAL = 60
 
+# Timeouts for find and render requests
+#FIND_TIMEOUT = 3.0  # Timeout for metric find requests
+#FETCH_TIMEOUT = 3.0  # Timeout to fetch series data
+
 #####################################
 # Filesystem Paths #
 #####################################
@@ -274,10 +278,8 @@ DEFAULT_XFILES_FACTOR = 0
 # This setting controls whether https is used to communicate between cluster members
 #INTRACLUSTER_HTTPS = False
 
-# These are timeout values (in seconds) for requests to remote webapps
-#REMOTE_FIND_TIMEOUT = 3.0           # Timeout for metric find requests
-#REMOTE_FETCH_TIMEOUT = 3.0          # Timeout to fetch series data
-#REMOTE_RETRY_DELAY = 60.0           # Time before retrying a failed remote webapp
+# Time before retrying a failed remote webapp
+#REMOTE_RETRY_DELAY = 60.0
 
 # Try to detect when a cluster server is localhost and don't forward queries
 #REMOTE_EXCLUDE_LOCAL = False

webapp/graphite/readers/remote.py

@@ -61,7 +61,7 @@ def fetch_multi(self, startTime, endTime, now=None, requestContext=None):
                 '/render/',
                 fields=query_params,
                 headers=headers,
-                timeout=settings.REMOTE_FETCH_TIMEOUT,
+                timeout=settings.FETCH_TIMEOUT,
             )
             break
           except Exception:

webapp/graphite/settings.py

@@ -54,6 +54,10 @@
 RRD_DIR = ''
 STANDARD_DIRS = []
 
+# Timeout settings
+FIND_TIMEOUT = None  # default 3.0 see below
+FETCH_TIMEOUT = None  # default 6.0 see below
+
 # Cluster settings
 CLUSTER_SERVERS = []
 
@@ -63,8 +67,8 @@
 
 # This settings control whether https is used to communicate between cluster members
 INTRACLUSTER_HTTPS = False
-REMOTE_FIND_TIMEOUT = 3.0
-REMOTE_FETCH_TIMEOUT = 3.0
+REMOTE_FIND_TIMEOUT = None  # Replaced by FIND_TIMEOUT
+REMOTE_FETCH_TIMEOUT = None  # Replaced by FETCH_TIMEOUT
 REMOTE_RETRY_DELAY = 60.0
 REMOTE_EXCLUDE_LOCAL = False
 REMOTE_STORE_MERGE_RESULTS = True
@@ -210,6 +214,10 @@
     join(WEBAPP_DIR, 'content'),
 )
 
+# Handle renamed timeout settings
+FIND_TIMEOUT = FIND_TIMEOUT or REMOTE_FIND_TIMEOUT or 3.0
+FETCH_TIMEOUT = FETCH_TIMEOUT or REMOTE_FETCH_TIMEOUT or 6.0
+
 ## Set config dependent on flags set in local_settings
 # Path configuration
 if not STATIC_ROOT:

webapp/graphite/storage.py

@@ -122,7 +122,7 @@ def fetch(self, patterns, startTime, endTime, now, requestContext):
         # Start fetches
         start = time.time()
         try:
-          for job in self.pool_exec(jobs, settings.REMOTE_FETCH_TIMEOUT):
+          for job in self.pool_exec(jobs, settings.FETCH_TIMEOUT):
             done += 1
 
             if job.exception:
@@ -199,7 +199,7 @@ def get_index(self, requestContext=None):
         # Start index lookups
         start = time.time()
         try:
-          for job in self.pool_exec(jobs, settings.REMOTE_FETCH_TIMEOUT):
+          for job in self.pool_exec(jobs, settings.FETCH_TIMEOUT):
             done += 1
 
             if job.exception:
@@ -264,7 +264,7 @@ def _find(self, query):
         # Start finds
         start = time.time()
         try:
-          for job in self.pool_exec(jobs, settings.REMOTE_FIND_TIMEOUT):
+          for job in self.pool_exec(jobs, settings.FIND_TIMEOUT):
             done += 1
 
             if job.exception:
@@ -420,7 +420,7 @@ def tagdb_auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestCon
           return self.tagdb.auto_complete_tags(exprs, tagPrefix=tagPrefix, limit=limit, requestContext=requestContext)
 
         # start finder jobs
-        jobs = self.pool_exec(jobs, settings.REMOTE_FIND_TIMEOUT)
+        jobs = self.pool_exec(jobs, settings.FIND_TIMEOUT)
 
         results = set()
 
@@ -482,7 +482,7 @@ def tagdb_auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None, r
           return self.tagdb.auto_complete_values(exprs, tag, valuePrefix=valuePrefix, limit=limit, requestContext=requestContext)
 
         # start finder jobs
-        jobs = self.pool_exec(jobs, settings.REMOTE_FIND_TIMEOUT)
+        jobs = self.pool_exec(jobs, settings.FIND_TIMEOUT)
 
         results = set()
 

webapp/graphite/tags/http.py

@@ -45,7 +45,7 @@ def request(self, method, url, fields, requestContext=None):
       self.base_url + url,
       fields=req_fields,
       headers=headers,
-      timeout=self.settings.REMOTE_FIND_TIMEOUT,
+      timeout=self.settings.FIND_TIMEOUT,
     )
 
     if result.status == 400:

webapp/tests/test_finders_remote.py

@@ -61,7 +61,7 @@ def test_fail(self):
     @patch('urllib3.PoolManager.request')
     @override_settings(INTRACLUSTER_HTTPS=False)
     @override_settings(REMOTE_STORE_USE_POST=True)
-    @override_settings(REMOTE_FIND_TIMEOUT=10)
+    @override_settings(FIND_TIMEOUT=10)
     def test_find_nodes(self, http_request):
       finder = RemoteFinder('127.0.0.1')
 
@@ -220,7 +220,7 @@ def test_find_nodes_cached(self, http_request, cache_get):
     @patch('urllib3.PoolManager.request')
     @override_settings(INTRACLUSTER_HTTPS=True)
     @override_settings(REMOTE_STORE_USE_POST=True)
-    @override_settings(REMOTE_FETCH_TIMEOUT=10)
+    @override_settings(FETCH_TIMEOUT=10)
     def test_RemoteFinder_fetch(self, http_request):
       test_finders = RemoteFinder.factory()
       finder = test_finders[0]
@@ -270,7 +270,7 @@ def test_RemoteFinder_fetch(self, http_request):
     @patch('urllib3.PoolManager.request')
     @override_settings(INTRACLUSTER_HTTPS=False)
     @override_settings(REMOTE_STORE_USE_POST=True)
-    @override_settings(REMOTE_FIND_TIMEOUT=10)
+    @override_settings(FIND_TIMEOUT=10)
     def test_get_index(self, http_request):
       finder = RemoteFinder('127.0.0.1')
 
@@ -314,7 +314,7 @@ def test_get_index(self, http_request):
     @override_settings(
       INTRACLUSTER_HTTPS=False,
       REMOTE_STORE_USE_POST=True,
-      REMOTE_FIND_TIMEOUT=10,
+      FIND_TIMEOUT=10,
       TAGDB_AUTOCOMPLETE_LIMIT=100)
     def test_auto_complete_tags(self, http_request):
       finder = RemoteFinder('127.0.0.1')
@@ -391,7 +391,7 @@ def test_auto_complete_tags(self, http_request):
     @override_settings(
       INTRACLUSTER_HTTPS=False,
       REMOTE_STORE_USE_POST=True,
-      REMOTE_FIND_TIMEOUT=10,
+      FIND_TIMEOUT=10,
       TAGDB_AUTOCOMPLETE_LIMIT=100)
     def test_auto_complete_values(self, http_request):
       finder = RemoteFinder('127.0.0.1')

webapp/tests/test_readers_remote.py

@@ -38,7 +38,7 @@ def test_RemoteReader_init_repr_get_intervals(self):
     @mock.patch('django.conf.settings.CLUSTER_SERVERS', ['127.0.0.1', 'http://8.8.8.8/graphite?format=msgpack&local=0'])
     @mock.patch('django.conf.settings.INTRACLUSTER_HTTPS', False)
     @mock.patch('django.conf.settings.REMOTE_STORE_USE_POST', False)
-    @mock.patch('django.conf.settings.REMOTE_FETCH_TIMEOUT', 10)
+    @mock.patch('django.conf.settings.FETCH_TIMEOUT', 10)
     def test_RemoteReader_fetch_multi(self, http_request):
         test_finders = RemoteFinder.factory()
         finder = test_finders[0]
@@ -186,7 +186,7 @@ def test_RemoteReader_fetch_multi(self, http_request):
     @mock.patch('django.conf.settings.CLUSTER_SERVERS', ['127.0.0.1', '8.8.8.8'])
     @mock.patch('django.conf.settings.INTRACLUSTER_HTTPS', False)
     @mock.patch('django.conf.settings.REMOTE_STORE_USE_POST', False)
-    @mock.patch('django.conf.settings.REMOTE_FETCH_TIMEOUT', 10)
+    @mock.patch('django.conf.settings.FETCH_TIMEOUT', 10)
     def test_RemoteReader_fetch(self, http_request):
         test_finders = RemoteFinder.factory()
         finder = test_finders[0]

webapp/tests/test_storage.py

@@ -515,7 +515,7 @@ def mockAutoCompleteValues(exprs, tag, valuePrefix=None, limit=None, requestCont
     # test pool timeout handling
     store = mockStore([TestFinderTagsTimeout()])
 
-    with self.settings(USE_WORKER_POOL=True, REMOTE_FIND_TIMEOUT=0):
+    with self.settings(USE_WORKER_POOL=True, FIND_TIMEOUT=0):
       with self.assertRaisesRegexp(Exception, 'Timed out in autocomplete tags for \[\'tag1=value1\'\] test after [-.e0-9]+s'):
         store.tagdb_auto_complete_tags(['tag1=value1'], 'test', 100, request_context)