Skip to content
Browse files

_hub.launch: implemented pending timeout

  • Loading branch information...
1 parent f87609b commit e3f5ff30509cd1777142cb08527a65b3eb188b20 @lirazsiri lirazsiri committed Aug 11, 2011
Showing with 17 additions and 3 deletions.
  1. +17 −3 cloudtask/_hub.py
View
20 cloudtask/_hub.py
@@ -17,13 +17,21 @@
from StringIO import StringIO
class Hub:
+ WAIT_FIRST = 30
+ WAIT_STATUS = 10
+ WAIT_RETRY = 5
+
+ RETRIES = 2
+
+ PENDING_TIMEOUT = 300
+
class Error(Exception):
pass
class Stopped(Error):
pass
- def __init__(self, apikey, wait_first=30, wait_status=10, wait_retry=5, retries=2):
+ def __init__(self, apikey, wait_first=WAIT_FIRST, wait_status=WAIT_STATUS, wait_retry=WAIT_RETRY, retries=RETRIES):
self.apikey = apikey
self.wait_first = wait_first
@@ -70,13 +78,15 @@ def get_pending_servers():
for server in retry(hub.servers.get, refresh_cache=True)
if server.instanceid in (pending_ids - yielded_ids) ]
- stopped = False
+ stopped = None
while True:
if callback and not stopped:
- stopped = callback() is False
+ if callback() is False:
+ stopped = time.time()
if stopped:
+
servers = [ server for server in get_pending_servers() ]
if not servers:
@@ -87,6 +97,10 @@ def get_pending_servers():
retry(server.destroy, auto_unregister=True)
pending_ids.remove(server.instanceid)
+ if server.status == 'pending' and \
+ (time.time() - stopped > self.PENDING_TIMEOUT):
+ raise self.Error("stuck pending instance")
+
time.sleep(self.wait_status)
continue

0 comments on commit e3f5ff3

Please sign in to comment.
Something went wrong with that request. Please try again.