Merge pull request pgq#46 from kristjanf/master

BaseScript: allow more advance excpetion sleep patterns

python/skytools/scripting.py

@@ -266,6 +266,7 @@ def __init__(self, service_name, args):
         self.service_name = service_name
         self.go_daemon = 0
         self.need_reload = 0
+        self.exception_count = 0
         self.stat_dict = {}
         self.log_level = logging.INFO
 
@@ -578,6 +579,8 @@ def run_func_safely(self, func, prefer_looping = False):
             r = func()
             if self.last_func_fail and time.time() > self.last_func_fail + self.exception_reset:
                 self.last_func_fail = None
+            # set exception count to 0 after success
+            self.exception_count = 0
             return r
         except UsageError, d:
             self.log.error(str(d))
@@ -613,7 +616,9 @@ def run_func_safely(self, func, prefer_looping = False):
         # reset and sleep
         self.reset()
         if prefer_looping and self.looping and self.loop_delay > 0:
-            self.sleep(self.exception_sleep)
+            # increase exception count & sleep
+            self.exception_count += 1
+            self.sleep_on_exception()
             return -1
         sys.exit(1)
 
@@ -625,6 +630,15 @@ def sleep(self, secs):
             if ex.errno != errno.EINTR:
                 raise
 
+    def sleep_on_exception(self):
+        """Make script sleep for some amount of time when an exception occurs.
+
+        To implement more advance exception sleeping like exponential backoff you
+        can override this method. Also note that you can use self.exception_count
+        to track the number of consecutive exceptions.
+        """
+        self.sleep(self.exception_sleep)
+
     def _is_quiet_exception(self, ex):
         return ((self.exception_quiet == ["ALL"] or ex.__class__.__name__ in self.exception_quiet)
                 and self.last_func_fail and time.time() < self.last_func_fail + self.exception_grace)