Skip to content

Commit

Permalink
Merge pull request #47 from ben-jones/add_overall_timeouts
Browse files Browse the repository at this point in the history
Add overall timeouts
  • Loading branch information
gsathya committed Sep 22, 2014
2 parents 7dd4485 + 9634cec commit 614105b
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 10 deletions.
31 changes: 22 additions & 9 deletions centinel/backend.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import os
import glob
import uuid
import json
import requests
import logging
import os
import requests
import time
import uuid


class User:
Expand Down Expand Up @@ -50,9 +51,11 @@ def submit_result(self, file_name):
files = {'result': result_file}
url = "%s/%s" % (self.config['server']['server_url'], "results")
req = requests.post(url, proxies=self.config['proxy']['proxy'],
files=files, auth=self.auth)
files=files, auth=self.auth,
timeout=self.config['server']['req_timeout'])

req.raise_for_status()
os.remove(file_name)

def download_experiment(self, name):
logging.info("Downloading experiment - %s", name)
Expand Down Expand Up @@ -99,6 +102,7 @@ def create_user(self):
def sync(config):
logging.info("Starting sync with %s", config['server']['server_url'])

start = time.time()
try:
user = User(config)
except Exception, e:
Expand All @@ -113,6 +117,9 @@ def sync(config):
user.submit_result(path)
except Exception, e:
logging.error("Unable to send result file: %s" % str(e))
if time.time() - start > config['server']['total_timeout']:
logging.error("Interaction with server took too long. Preempting")
return

# get all experiment names
available_experiments = []
Expand All @@ -121,13 +128,19 @@ def sync(config):
file_name, _ = os.path.splitext(os.path.basename(path))
available_experiments.append(file_name)
available_experiments = set(available_experiments)
if time.time() - start > config['server']['total_timeout']:
logging.error("Interaction with server took too long. Preempting")
return

# download new experiments from server
try:
map(user.download_experiment,
set(user.experiments) - available_experiments)
except Exception, e:
logging.error("Unable to download experiment files %s", str(e))
for experiment in (set(user.experiments) - available_experiments):
try:
user.download_experiment(experiment)
except Exception, e:
logging.error("Unable to download experiment files %s", str(e))
if time.time() - start > config['server']['total_timeout']:
logging.error("Interaction with server took too long. Preempting")
return

logging.info("Finished sync with %s", config['server']['server_url'])

Expand Down
7 changes: 6 additions & 1 deletion centinel/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,14 @@ def __init__(self,):

# server
servers = {}
servers['server_url'] = "http://130.245.145.7:8082"
servers['server_url'] = "http://server.iclab.org:8082"
servers['login_file'] = os.path.join(self.params['user']['centinel_home'],
'login')
# the entire transaction should take less than 5 min
servers['total_timeout'] = 60*5
# set a socket timeout of 15 seconds (no way to do per request
# platform independently)
servers['req_timeout'] = 15
self.params['server'] = servers

# proxy
Expand Down

0 comments on commit 614105b

Please sign in to comment.