diff --git a/centinel/backend.py b/centinel/backend.py index 0b9311b..e5392f9 100644 --- a/centinel/backend.py +++ b/centinel/backend.py @@ -1,9 +1,10 @@ -import os import glob -import uuid import json -import requests import logging +import os +import requests +import time +import uuid class User: @@ -50,9 +51,11 @@ def submit_result(self, file_name): files = {'result': result_file} url = "%s/%s" % (self.config['server']['server_url'], "results") req = requests.post(url, proxies=self.config['proxy']['proxy'], - files=files, auth=self.auth) + files=files, auth=self.auth, + timeout=self.config['server']['req_timeout']) req.raise_for_status() + os.remove(file_name) def download_experiment(self, name): logging.info("Downloading experiment - %s", name) @@ -99,6 +102,7 @@ def create_user(self): def sync(config): logging.info("Starting sync with %s", config['server']['server_url']) + start = time.time() try: user = User(config) except Exception, e: @@ -113,6 +117,9 @@ def sync(config): user.submit_result(path) except Exception, e: logging.error("Unable to send result file: %s" % str(e)) + if time.time() - start > config['server']['total_timeout']: + logging.error("Interaction with server took too long. Preempting") + return # get all experiment names available_experiments = [] @@ -121,13 +128,19 @@ def sync(config): file_name, _ = os.path.splitext(os.path.basename(path)) available_experiments.append(file_name) available_experiments = set(available_experiments) + if time.time() - start > config['server']['total_timeout']: + logging.error("Interaction with server took too long. Preempting") + return # download new experiments from server - try: - map(user.download_experiment, - set(user.experiments) - available_experiments) - except Exception, e: - logging.error("Unable to download experiment files %s", str(e)) + for experiment in (set(user.experiments) - available_experiments): + try: + user.download_experiment(experiment) + except Exception, e: + logging.error("Unable to download experiment files %s", str(e)) + if time.time() - start > config['server']['total_timeout']: + logging.error("Interaction with server took too long. Preempting") + return logging.info("Finished sync with %s", config['server']['server_url']) diff --git a/centinel/config.py b/centinel/config.py index e0f9e0a..9bbbab4 100644 --- a/centinel/config.py +++ b/centinel/config.py @@ -37,9 +37,14 @@ def __init__(self,): # server servers = {} - servers['server_url'] = "http://130.245.145.7:8082" + servers['server_url'] = "http://server.iclab.org:8082" servers['login_file'] = os.path.join(self.params['user']['centinel_home'], 'login') + # the entire transaction should take less than 5 min + servers['total_timeout'] = 60*5 + # set a socket timeout of 15 seconds (no way to do per request + # platform independently) + servers['req_timeout'] = 15 self.params['server'] = servers # proxy