Skip to content
This repository has been archived by the owner on Sep 15, 2021. It is now read-only.

Commit

Permalink
Bug 791924 - Refactor + fork retrier from retry.py + fix retrying for…
Browse files Browse the repository at this point in the history 
… second file fetch. r=rail
  • Loading branch information
Armen Zambrano Gasparnian committed Jan 19, 2015
1 parent 33a6582 commit 970d5eb
Showing 1 changed file with 192 additions and 139 deletions.
331 changes: 192 additions & 139 deletions buildfarm/utils/repository_manifest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"""
import json
import logging
import os
import random
import time
import urllib2

Expand All @@ -29,179 +29,232 @@
# let sheriffs determine if re-triggering is needed
INFRA_CODE = 3

# Logic based on lib/python/util/retry.py
# The day that we don't wget repository_manifest.py
# we can import directly the functionality
def retry(action, attempts=5, sleeptime=60, max_sleeptime=5 * 60,
retry_exceptions=(Exception,), cleanup=None, args=(), kwargs={}):
"""Call `action' a maximum of `attempts' times until it succeeds,
defaulting to 5. `sleeptime' is the number of seconds to wait
between attempts, defaulting to 60 and doubling each retry attempt, to
a maximum of `max_sleeptime'. `retry_exceptions' is a tuple of
Exceptions that should be caught. If exceptions other than those
listed in `retry_exceptions' are raised from `action', they will be
raised immediately. If `cleanup' is provided and callable it will
be called immediately after an Exception is caught. No arguments
will be passed to it. If your cleanup function requires arguments
it is recommended that you wrap it in an argumentless function.
`args' and `kwargs' are a tuple and dict of arguments to pass onto
to `callable'"""
assert callable(action)
assert not cleanup or callable(cleanup)
if max_sleeptime < sleeptime:
logging.debug("max_sleeptime %d less than sleeptime %d" % (
max_sleeptime, sleeptime))
n = 1
while n <= attempts:
try:
logging.info("retry: Calling %s with args: %s, kwargs: %s, "
"attempt #%d" % (action, str(args), str(kwargs), n))
return action(*args, **kwargs)
except retry_exceptions:
logging.debug("retry: Caught exception: ", exc_info=True)
if cleanup:
cleanup()
if n == attempts:
logging.info("retry: Giving up on %s" % action)
raise
if sleeptime > 0:
logging.info(
"retry: Failed, sleeping %d seconds before retrying" %
sleeptime)
time.sleep(sleeptime)
sleeptime = sleeptime * 2
if sleeptime > max_sleeptime:
sleeptime = max_sleeptime
continue
finally:
n += 1
EXIT_CODE = FAILURE_CODE

def main():
logging.basicConfig(format='%(asctime)s %(message)s')
log = logging.getLogger(__name__)

# This has been copied from lib.python.util.retry
def retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=1.5,
jitter=1):
''' It helps us retry '''
for _ in range(attempts):
log.debug("attempt %i/%i", _ + 1, attempts)
yield
if jitter:
sleeptime += random.randint(-jitter, jitter)
if _ == attempts - 1:
# Don't need to sleep the last time
break
log.debug("sleeping for %.2fs (attempt %i/%i)",
sleeptime, _ + 1, attempts)
time.sleep(sleeptime)
sleeptime *= sleepscale
if sleeptime > max_sleeptime:
sleeptime = max_sleeptime

def options_args():
'''
Determine which repository and revision mozharness.json indicates.
If none is found we fall back to the default repository
Validate options and args and return them.
'''
parser = OptionParser(__doc__)
parser.add_option("--manifest-url", dest="manifest_url",
help="URL to json file which specifies 'repo' and "
help="URL to json file which specifies 'repo' and "
"'revision' values.")
parser.add_option("--default-repo", dest="default_repo",
help="If manifest_url is a 404 this is the repository we "
help="If manifest_url is a 404 this is the repository we "
"default to.")
parser.add_option("--default-revision", dest="default_revision",
help="If manifest_url is a 404 this is the revision we "
help="If manifest_url is a 404 this is the revision we "
"default to.")
parser.add_option("--default-checkout", dest="default_checkout",
help="If manifest_url's repo is the same as default_repo "
help="If manifest_url's repo is the same as default_repo "
"then use the default_checkout, otherwise, use checkout."
"This is an optimization for when we have the ability "
"of using an hg-shared checkout.")
parser.add_option("--checkout", dest="checkout",
help="If default_repo and the repo in manifest_url are "
"distinct we use a different checkout. In other words, "
"we don't want to use an hg-shared checkout since we're "
"trying to checkout a complete different repository")
help="If default_repo and the repo in manifest_url are "
"distinct we use a different checkout. In other words, "
"we don't want to use an hg-shared checkout since we're "
"trying to checkout a complete different repository")
parser.add_option("--timeout", dest="timeout", type="float", default=30,
help="Used to specify how long to wait until timing out "
help="Used to specify how long to wait until timing out "
"for network requests.")
parser.add_option("--max-retries", dest="max_retries", type="int",
default=10,
help="A maximum number of retries for network requests.")
help="A maximum number of retries for network requests.")
parser.add_option("--sleeptime", dest="sleeptime", type="int", default=10,
help="How long to sleep in between network requests.")
help="How long to sleep in between network requests.")
parser.add_option("--debug", dest="debug", action="store_true",
default=False, help="Enable debug logging.")

options, args = parser.parse_args()

if not options.manifest_url or \
not options.default_repo or \
not options.default_revision:
parser.error("You have to call the script with all options")
parser.error("You have to call the script with all required options.")

if options.default_checkout and options.checkout \
and options.default_checkout == options.checkout:
parser.error("If you use --default-checkout and --checkout,"
"you can't have them be the same value.")

exit_code = FAILURE_CODE
if options.debug:
log.setLevel(logging.DEBUG)
log.info("Setting DEBUG logging.")

return options, args

def fetch_manifest(url, options):
'''
This function simply fetches a manifest from a URL.
We return the manifest (None if unsuccessful) and the EXIT_CODE.
'''
global EXIT_CODE

manifest = None

log.debug("Fetching %s", url)

try:
url = options.manifest_url
url_opener = retry(
urllib2.urlopen,
attempts=options.max_retries,
sleeptime=options.sleeptime,
max_sleeptime=options.max_retries * options.sleeptime,
retry_exceptions=(
SSLError,
urllib2.URLError,
),
args=(url, ),
kwargs={
"timeout": options.timeout,
},
)
http_code = url_opener.getcode()
if http_code == 200:
num = 0
for _ in retrier(attempts=options.max_retries,
sleeptime=options.sleeptime,
max_sleeptime=options.max_retries * options.sleeptime):
log.debug("Attempt number %d out of %d", num+1, options.max_retries)
try:
manifest = json.load(url_opener)
repo = manifest["repo"]
revision = manifest["revision"]
# Let's determine if the repo and revision exist
url = '%s/rev/%s' % (repo, revision)
urllib2.urlopen(url, timeout=options.timeout)
print "script_repo_url: %s" % repo
print "script_repo_revision: %s" % revision
exit_code = SUCCESS_CODE
# This block allow us to take advantage of default shared
# checkouts
if options.default_checkout and options.checkout:
if options.default_repo == repo:
# In the manifest we've requested to checkout the defult
# repo, hence, we will use the default_checkout
# location in order to use the hg-shared checkout
checkout = options.default_checkout
else:
# We are not requesting the default repository, hence,
# we can't use the hg-shared checkout
checkout = options.checkout

print "script_repo_checkout: %s" % checkout

except urllib2.HTTPError:
url_opener = urllib2.urlopen(url, timeout=options.timeout)
http_code = url_opener.getcode()
if http_code == 200:
log.debug("Attempting to load manifest...")
try:
manifest = json.load(url_opener)
log.debug("Contents of manifest: %s", manifest)
except ValueError:
log.exception("We have a non-valid json manifest.")
EXIT_CODE = FAILURE_CODE
break
else:
log.error("We have failed to retrieve the manifest "
"(http code: %s)", http_code)
EXIT_CODE = INFRA_CODE
except urllib2.HTTPError, e:
if e.getcode() == 404:
# Fallback to default values for branches where the manifest
# is not defined
print_values(options.default_repo,
options.default_revision,
options.default_checkout,
options)

break
else:
EXIT_CODE = FAILURE_CODE
except SSLError:
if num == options.max_retries - 1:
log.exception("SSLError for %s", url)
EXIT_CODE = INFRA_CODE
break
except urllib2.URLError:
if num == options.max_retries - 1:
log.exception("URLError for %s", url)
EXIT_CODE = INFRA_CODE
break

num += 1
except:
log.exception("Unknown case")
EXIT_CODE = FAILURE_CODE

return manifest

def valid_repository_revision(url, options):
'''
We validate that the URL passes is existent.
If not, we dump the exception's infromation and change the global EXIT_CODE.
'''
global EXIT_CODE

log.debug("Making sure that %s is a valid URL.", url)
num = 0
for _ in retrier(attempts=options.max_retries,
sleeptime=options.sleeptime,
max_sleeptime=options.max_retries * options.sleeptime):
log.debug("Attempt number %d out of %d", num+1, options.max_retries)
try:
# This is just to check that the repository referenced exists
urllib2.urlopen(url, timeout=options.timeout)
log.debug("We have been able to reach %s", url)
return True
except urllib2.HTTPError:
if num == options.max_retries - 1:
# This is a 404 case because either the repo or the
# revision have wrong values
logging.exception(url)
exit_code = FAILURE_CODE
except ValueError:
logging.exception("We have a non-valid json manifest.")
exit_code = FAILURE_CODE
else:
logging.error("We have failed to retrieve the manifest "
"(http code: %s)" % http_code)
exit_code = INFRA_CODE

except urllib2.HTTPError, e:
if e.getcode() == 404:
# Fallback to default values for branches where the manifest
# is not defined
print "script_repo_url: %s" % options.default_repo
print "script_repo_revision: %s" % options.default_revision
if options.default_checkout:
print "script_repo_checkout: %s" % options.default_checkout

exit_code = SUCCESS_CODE
# revision in the manifest have invalid values
log.exception("We are trying to reach a non-existant "
"url: %s", url)
EXIT_CODE = FAILURE_CODE
return False
except urllib2.URLError:
if num == options.max_retries - 1:
# The manifest has invalid data
log.exception("URLError for %s", url)
EXIT_CODE = FAILURE_CODE
return False
num += 1

def print_values(repo, revision, checkout, options):
'''
This function simply prints the correct values for:
script_repo_url
script_repo_revision
script_repo_checkout - if applicable
'''
global EXIT_CODE
log.debug("About to print the values...")

print "script_repo_url: %s" % repo
print "script_repo_revision: %s" % revision

# This block allow us to take advantage of default shared
# checkouts
if options.default_checkout and options.checkout:
if options.default_repo == repo:
# In the manifest we've requested to checkout the defult
# repo, hence, we will use the default_checkout
# location in order to use the hg-shared checkout
checkout = options.default_checkout
else:
logging.exception("We got HTTPError code: %s." % e.getcode())
exit_code = FAILURE_CODE
except SSLError:
logging.exception("SSLError for %s" % url)
exit_code = INFRA_CODE
except urllib2.URLError:
logging.exception("URLError for %s" % url)
exit_code = INFRA_CODE
except Exception:
logging.exception("Unknown case")
exit_code = FAILURE_CODE

exit(exit_code)
# We are not requesting the default repository, hence,
# we can't use the hg-shared checkout
checkout = options.checkout

print "script_repo_checkout: %s" % checkout

EXIT_CODE = SUCCESS_CODE

def main():
'''
Determine which repository and revision mozharness.json indicates.
If none is found we fall back to the default repository
'''
options, args = options_args()

global EXIT_CODE
EXIT_CODE = FAILURE_CODE

url = options.manifest_url
manifest = fetch_manifest(options.manifest_url, options)

if manifest is not None:
log.debug("We have a manifest %s", manifest)
repo = manifest["repo"]
revision = manifest["revision"]
url = '%s/rev/%s' % (repo, revision)
if valid_repository_revision(url, options):
print_values(repo, revision, options.checkout, options)

exit(EXIT_CODE)

if __name__ == '__main__':
main()

0 comments on commit 970d5eb

Please sign in to comment.