Merge pull request #47 from ben-jones/add_overall_timeouts

Add overall timeouts

centinel/backend.py

@@ -1,9 +1,10 @@
-import os
 import glob
-import uuid
 import json
-import requests
 import logging
+import os
+import requests
+import time
+import uuid
 
 
 class User:
@@ -50,9 +51,11 @@ def submit_result(self, file_name):
             files = {'result': result_file}
             url   = "%s/%s" % (self.config['server']['server_url'], "results")
             req   = requests.post(url, proxies=self.config['proxy']['proxy'],
-                                  files=files, auth=self.auth)
+                                  files=files, auth=self.auth,
+                                  timeout=self.config['server']['req_timeout'])
 
         req.raise_for_status()
+        os.remove(file_name)
 
     def download_experiment(self, name):
         logging.info("Downloading experiment - %s", name)
@@ -99,6 +102,7 @@ def create_user(self):
 def sync(config):
     logging.info("Starting sync with %s", config['server']['server_url'])
 
+    start = time.time()
     try:
         user = User(config)
     except Exception, e:
@@ -113,6 +117,9 @@ def sync(config):
             user.submit_result(path)
         except Exception, e:
             logging.error("Unable to send result file: %s" % str(e))
+        if time.time() - start > config['server']['total_timeout']:
+            logging.error("Interaction with server took too long. Preempting")
+            return
 
     # get all experiment names
     available_experiments = []
@@ -121,13 +128,19 @@ def sync(config):
         file_name, _ = os.path.splitext(os.path.basename(path))
         available_experiments.append(file_name)
     available_experiments = set(available_experiments)
+    if time.time() - start > config['server']['total_timeout']:
+        logging.error("Interaction with server took too long. Preempting")
+        return
 
     # download new experiments from server
-    try:
-        map(user.download_experiment,
-            set(user.experiments) - available_experiments)
-    except Exception, e:
-        logging.error("Unable to download experiment files %s", str(e))
+    for experiment in (set(user.experiments) - available_experiments):
+        try:
+            user.download_experiment(experiment)
+        except Exception, e:
+            logging.error("Unable to download experiment files %s", str(e))
+        if time.time() - start > config['server']['total_timeout']:
+            logging.error("Interaction with server took too long. Preempting")
+            return
 
     logging.info("Finished sync with %s", config['server']['server_url'])
 

centinel/config.py

@@ -37,9 +37,14 @@ def __init__(self,):
 
         # server
         servers = {}
-        servers['server_url'] = "http://130.245.145.7:8082"
+        servers['server_url'] = "http://server.iclab.org:8082"
         servers['login_file'] = os.path.join(self.params['user']['centinel_home'],
                                              'login')
+        # the entire transaction should take less than 5 min
+        servers['total_timeout'] = 60*5
+        # set a socket timeout of 15 seconds (no way to do per request
+        # platform independently)
+        servers['req_timeout'] = 15
         self.params['server'] = servers
 
         # proxy