Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

bug 808814 - mozharness download-and-extract should detect, retry, an…

…d report download errors. r=rail
  • Loading branch information...
commit 4ab319b425c525f7a45f94bb2d43e7bcd129cbd8 1 parent 1f4a276
@escapewindow escapewindow authored
View
7 mozharness/base/config.py
@@ -227,13 +227,6 @@ def _create_config_parser(self, config_options, usage):
self.config_parser.add_option_group(log_option_group)
- # TODO deal with noop properly.
- #self.config_parser.add_option(
- # "--noop", "--dry-run", action="store_true", default=False,
- # dest="noop",
- # help="Echo commands without executing them."
- #)
-
# Actions
action_option_group = OptionGroup(self.config_parser, "Actions",
"Use these options to list or enable/disable actions.")
View
131 mozharness/base/script.py
@@ -29,7 +29,7 @@
from mozharness.base.config import BaseConfig
from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \
- LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL
+ LogMixin, OutputParser, DEBUG, INFO, ERROR, WARNING, FATAL
# OSMixin {{{1
class OSMixin(object):
@@ -43,13 +43,12 @@ def mkdir_p(self, path, error_level=ERROR):
"""
if not os.path.exists(path):
self.info("mkdir: %s" % path)
- if not self.config.get('noop'):
- try:
- os.makedirs(path)
- except OSError:
- self.log("Can't create directory %s!" % path,
- level=error_level)
- return -1
+ try:
+ os.makedirs(path)
+ except OSError:
+ self.log("Can't create directory %s!" % path,
+ level=error_level)
+ return -1
else:
self.debug("mkdir_p: %s Already exists." % path)
@@ -145,12 +144,9 @@ def get_filename_from_url(self, url):
# TODO thinking about creating a transfer object.
def download_file(self, url, file_name=None, parent_dir=None,
create_parent_dir=True, error_level=ERROR,
- exit_code=-1):
+ num_retries=None, exit_code=-1):
"""Python wget.
- TODO: should noop touch the filename? seems counter-noop.
- TODO: the initial log line should say "Downloading url to file_name"
"""
- message = ""
if not file_name:
try:
file_name = self.get_filename_from_url(url)
@@ -160,60 +156,74 @@ def download_file(self, url, file_name=None, parent_dir=None,
return
if parent_dir:
file_name = os.path.join(parent_dir, file_name)
- parent_dir = os.path.dirname(file_name)
- if self.config.get('noop'):
- self.info("Downloading %s%s" % (url, message))
- return file_name
- req = urllib2.Request(url)
- try:
- self.info("Downloading %s%s" % (url, message))
- f = urllib2.urlopen(req)
- if create_parent_dir and parent_dir:
+ if create_parent_dir:
self.mkdir_p(parent_dir, error_level=error_level)
- local_file = open(file_name, 'wb')
- while True:
- block = f.read(1024**2)
- if not block:
- break
- local_file.write(block)
- local_file.close()
- except urllib2.HTTPError, e:
- self.log("HTTP Error: %s %s" % (e.code, url), level=error_level,
- exit_code=exit_code)
- return
- except urllib2.URLError, e:
- self.log("URL Error: %s" % (url), level=error_level,
- exit_code=exit_code)
- return
- return file_name
+ if num_retries is None:
+ num_retries = self.config.get("global_retries", 5)
+ self.info("Downloading %s to %s" % (url, file_name))
+ try_num = 0
+ while try_num <= num_retries:
+ try_num += 1
+ level = WARNING
+ if try_num > num_retries:
+ level = error_level
+ try:
+ f = urllib2.urlopen(url)
+ local_file = open(file_name, 'wb')
+ while True:
+ block = f.read(1024**2)
+ if not block:
+ break
+ local_file.write(block)
+ local_file.close()
+ return file_name
+ except urllib2.HTTPError, e:
+ self.log("Try %d: Server returned status %s %s for %s" % (try_num, str(e.code), str(e), url),
+ level=level, exit_code=exit_code)
+ except urllib2.URLError, e:
+ if try_num > num_retries:
+ remote_host = urlparse.urlsplit(url)[1]
+ if not remote_host:
+ return
+ nslookup = self.query_exe('nslookup')
+ error_list = [{
+ 'substr': "server can't find %s" % remote_host,
+ 'level': ERROR,
+ 'explanation': "Either %s is an invalid hostname, or DNS is busted." % remote_host,
+ }]
+ self.run_command([nslookup, remote_host],
+ error_list=error_list)
+ self.log("Try %d: URL Error: %s" % (try_num, url), level=level,
+ exit_code=exit_code)
+ if try_num <= num_retries:
+ sleep_time = try_num * 20
+ self.info("Sleeping %d seconds..." % sleep_time)
+ time.sleep(sleep_time)
def move(self, src, dest, log_level=INFO, error_level=ERROR,
exit_code=-1):
self.log("Moving %s to %s" % (src, dest), level=log_level)
- if not self.config.get('noop'):
- try:
- shutil.move(src, dest)
- # http://docs.python.org/tutorial/errors.html
- except IOError, e:
- self.log("IO error: %s" % str(e),
- level=error_level, exit_code=exit_code)
- return -1
+ try:
+ shutil.move(src, dest)
+ # http://docs.python.org/tutorial/errors.html
+ except IOError, e:
+ self.log("IO error: %s" % str(e),
+ level=error_level, exit_code=exit_code)
+ return -1
return 0
def chmod(self, path, mode):
self.info("Chmoding %s to %s" % (path, str(oct(mode))))
- if not self.config.get('noop'):
- os.chmod(path, mode)
+ os.chmod(path, mode)
def copyfile(self, src, dest, log_level=INFO, error_level=ERROR):
self.log("Copying %s to %s" % (src, dest), level=log_level)
- if not self.config.get('noop'):
- try:
- shutil.copyfile(src, dest)
- except (IOError, shutil.Error), e:
- self.log("Can't copy %s to %s: %s!" % (src, dest, str(e)),
- level=error_level)
- return -1
+ try:
+ shutil.copyfile(src, dest)
+ except (IOError, shutil.Error), e:
+ self.log("Can't copy %s to %s: %s!" % (src, dest, str(e)),
+ level=error_level)
+ return -1
def copytree(self, src, dest, overwrite='no_overwrite', log_level=INFO,
error_level=ERROR):
@@ -321,12 +331,9 @@ def read_from_file(self, file_path, verbose=True, open_mode='r',
self.log("%s can't be opened for reading!" % file_path,
level=error_level)
- def chdir(self, dir_name, ignore_if_noop=False):
+ def chdir(self, dir_name):
self.log("Changing directory to %s." % dir_name)
- if self.config.get('noop') and ignore_if_noop:
- self.info("noop: not changing dir")
- else:
- os.chdir(dir_name)
+ os.chdir(dir_name)
def which(self, program):
"""
@@ -469,9 +476,6 @@ def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
self.info("Running command: %s" % command)
if isinstance(command, list):
self.info("Copy/paste: %s" % subprocess.list2cmdline(command))
- if self.config.get('noop'):
- self.info("(Dry run; skipping)")
- return
shell = True
if isinstance(command, list):
shell = False
@@ -547,9 +551,6 @@ def get_output_from_command(self, command, cwd=None,
if isinstance(command, list):
self.info("Copy/paste: %s" % subprocess.list2cmdline(command))
# This could potentially return something?
- if self.config.get('noop'):
- self.info("(Dry run; skipping)")
- return ''
tmp_stdout = None
tmp_stderr = None
tmp_stdout_filename = '%s_stdout' % tmpfile_base_path
View
5 mozharness/mozilla/testing/testbase.py
@@ -178,7 +178,7 @@ def _extract_test_zip(self, target_unzip_dirs=None):
if target_unzip_dirs:
unzip_cmd.extend(target_unzip_dirs)
# TODO error_list
- self.run_command(unzip_cmd, cwd=test_install_dir)
+ self.run_command(unzip_cmd, cwd=test_install_dir, halt_on_failure=True)
def _download_installer(self):
file_name = None
@@ -205,7 +205,8 @@ def _download_and_extract_symbols(self):
error_level=FATAL)
self.set_buildbot_property("symbols_url", self.symbols_url,
write_to_file=True)
- self.run_command(['unzip', '-q', source], cwd=self.symbols_path)
+ self.run_command(['unzip', '-q', source], cwd=self.symbols_path,
+ halt_on_failure=True)
def download_and_extract(self):
"""
View
20 mozharness/mozilla/tooltool.py
@@ -1,7 +1,12 @@
import os
+import time
from mozharness.base.errors import PythonErrorList
-from mozharness.base.log import FATAL
+from mozharness.base.log import ERROR, FATAL
+
+TooltoolErrorList = PythonErrorList + [{
+ 'substr': 'ERROR - ', 'level': ERROR
+}]
class TooltoolMixin(object):
"""Mixin class for handling tooltool manifests.
@@ -14,7 +19,18 @@ def tooltool_fetch(self, manifest, output_dir=None):
for s in self.config['tooltool_servers']:
cmd.extend(['--url', s])
cmd.extend(['fetch', '-m', manifest, '-o'])
- self.run_command(cmd, cwd=output_dir, error_list=PythonErrorList)
+ num_retries = self.config.get("global_retries", 5)
+ try_num = 0
+ while try_num <= num_retries:
+ try_num += 1
+ if not self.run_command(cmd, cwd=output_dir, error_list=TooltoolErrorList):
+ return
+ if try_num <= num_retries:
+ sleep_time = 2 * try_num
+ self.warning("Try %d failed; sleeping %d..." % (try_num, sleep_time))
+ time.sleep(sleep_time)
+ else:
+ self.fatal("Tooltool %s fetch failed after %d tries!" % (manifest, try_num))
def create_tooltool_manifest(self, contents, path=None):
""" Currently just creates a manifest, given the contents.
View
1  scripts/sourcetool.py
@@ -197,7 +197,6 @@ def source(self):
'vcs_share_base': self.config.get('vcs_shared_dir'),
'allow_unshared_local_clones': self.config.get('vcs_allow_unshared_local_clones'),
'halt_on_failure': self.config.get('halt_on_failure', True),
- 'noop': self.config.get('noop'),
}
)
else:
View
38 test/test_base_script.py
@@ -40,11 +40,6 @@ def get_debug_script_obj():
initial_config_file='test/test.json')
return s
-def get_noop_script_obj():
- s = script.BaseScript(config={'noop': True},
- initial_config_file='test/test.json')
- return s
-
class TestScript(unittest.TestCase):
def setUp(self):
cleanup()
@@ -68,41 +63,12 @@ def test_existing_mkdir_p(self):
self.assertTrue(os.path.isdir('test_dir/foo/bar/baz'),
msg="mkdir_p error when dir exists")
- def test_noop_mkdir_p(self):
- self.s = get_noop_script_obj()
- self.s.mkdir_p('test_dir/foo/bar/baz')
- self.assertFalse(os.path.exists('test_dir'),
- msg="mkdir_p noop error")
-
- def test_noop_mkdir_p(self):
- self.s = get_noop_script_obj()
- self.s.download_file("http://www.mozilla.com", file_name="test_logs/mozilla.com",
- error_level=IGNORE)
- self.assertFalse(os.path.exists('test_logs/mozilla.com'),
- msg="download_file noop error")
-
- def test_noop_get_output_from_command(self):
- self.s = get_noop_script_obj()
- contents1 = self.s.get_output_from_command("cat test/test.json", cwd="configs")
- self.assertEqual(contents1, '',
- msg="get_output_from_command noop error")
-
- def test_noop_run_command(self):
- self.s = get_noop_script_obj()
- self.s.run_command("touch test_logs/foo")
- self.assertFalse(os.path.exists('test_logs/foo'),
- msg="run_command noop error")
-
def test_chdir(self):
- self.s = get_noop_script_obj()
+ self.s = script.BaseScript(initial_config_file='test/test.json')
cwd = os.getcwd()
- self.s.chdir('test_logs', ignore_if_noop=True)
- self.assertEqual(cwd, os.getcwd(),
- msg="chdir noop error")
- os.chdir(cwd)
self.s.chdir('test_logs')
self.assertEqual(os.path.join(cwd, "test_logs"), os.getcwd(),
- msg="chdir noop noignore error")
+ msg="chdir error")
self.s.chdir(cwd)
def _test_log_helper(self, obj):
Please sign in to comment.
Something went wrong with that request. Please try again.