Bug 687976 - retry function should use exponential backoff. r=jhford,…

… a=catlee

buildfarm/utils/retry.py

@@ -88,8 +88,11 @@ def run_with_timeout(cmd, timeout, stdout_regexp=None, stderr_regexp=None,
                       help="""timeout each request after this many seconds.  Set
                       to 0 to have no timeout.  Defaults to 300""")
     parser.add_option("-s", "--sleeptime", type="int", dest="sleeptime",
-                      help="""sleep this many seconds between tries.
-                      Defaults to 30""")
+                      help="""sleep this many seconds between tries, doubling
+                      each retry iteration.  Defaults to 30""")
+    parser.add_option("-m", "--maxsleeptime", type="int", dest="maxsleeptime",
+                      help="""when doubling sleeptime, do not exceed this value.
+                      Defaults to 300""")
     parser.add_option("--stdout-regexp", dest="stdout_regexp",
                       help="""Fail if the expected regexp is not found in
                       stdout""")
@@ -108,6 +111,7 @@ def run_with_timeout(cmd, timeout, stdout_regexp=None, stderr_regexp=None,
             retries=10,
             timeout=300,
             sleeptime=30,
+            maxsleeptime=300,
             )
 
     options, args = parser.parse_args()
@@ -123,7 +127,7 @@ def run_with_timeout(cmd, timeout, stdout_regexp=None, stderr_regexp=None,
 
     try:
         rc = retry(run_with_timeout, attempts=options.retries,
-                   sleeptime=options.sleeptime,
+                   sleeptime=options.sleeptime, max_sleeptime=options.maxsleeptime,
                    args=(args, options.timeout, options.stdout_regexp,
                          options.stderr_regexp, options.fail_if_match,
                          options.print_output))

lib/python/util/retry.py

@@ -5,12 +5,13 @@
 import logging
 log = logging.getLogger(__name__)
 
-def retry(action, attempts=5, sleeptime=60, retry_exceptions=(Exception,),
+def retry(action, attempts=5, sleeptime=60, max_sleeptime=5*60, retry_exceptions=(Exception,),
           cleanup=None, args=(), kwargs={}):
     """Call `action' a maximum of `attempts' times until it succeeds,
         defaulting to 5. `sleeptime' is the number of seconds to wait
-        between attempts, defaulting to 0. `retry_exceptions' is a tuple
-        of Exceptions that should be caught. If exceptions other than those
+	between attempts, defaulting to 60 and doubling each retry attempt, to
+        a maximum of `max_sleeptime'.  `retry_exceptions' is a tuple of
+        Exceptions that should be caught. If exceptions other than those
         listed in `retry_exceptions' are raised from `action', they will be
         raised immediately. If `cleanup' is provided and callable it will
         be called immediately after an Exception is caught. No arguments
@@ -20,6 +21,8 @@ def retry(action, attempts=5, sleeptime=60, retry_exceptions=(Exception,),
         to `callable'"""
     assert callable(action)
     assert not cleanup or callable(cleanup)
+    if max_sleeptime < sleeptime:
+        log.debug("max_sleeptime %d less than sleeptime %d" % (max_sleeptime, sleeptime))
     n = 1
     while n <= attempts:
         try:
@@ -36,6 +39,9 @@ def retry(action, attempts=5, sleeptime=60, retry_exceptions=(Exception,),
             if sleeptime > 0:
                 log.info("retry: Failed, sleeping %d seconds before retrying" % sleeptime)
                 time.sleep(sleeptime)
+                sleeptime = sleeptime * 2
+                if sleeptime > max_sleeptime:
+                    sleeptime = max_sleeptime
             continue
         finally:
             n += 1