Permalink
Browse files

bug 700415 - run peptest on try. r=jhammel r=ahal

  • Loading branch information...
1 parent 5cbc0fe commit d0acf5dfc35dbd947d0637630a12228823ec40c0 @escapewindow escapewindow committed Jan 17, 2012
View
32 configs/peptest/prod_config.py
@@ -2,28 +2,20 @@
# The peptest mozharness script is set up so that specifying None
# is the same as not specifying the option at all
-import os
config = {
# mozharness script options
- "base_work_dir": os.getcwd(),
- "work_dir": "build",
"log_name": "pep",
- "log_level": "info",
- "test_url": "url_to_packaged_tests",
- # path or url to a zip or folder containing the mozbase packages
- "mozbase_path": "url_to_mozbase_zip",
- # path or url to a zip or folder containing peptest
- "peptest_path": "url_to_peptest_zip",
-
+ "buildbot_json_path": "buildprops.json",
+ "virtualenv_modules": ["simplejson"],
+ "simplejson_url": "http://build.mozilla.org/talos/zips/simplejson-2.2.1.tar.gz",
# peptest options
- "appname": "url_to_application", # i.e the firefox build on ftp.m.o
# defaults to firefox, can also be thunderbird, fennec, etc.
"app": "firefox",
# if test_url is specified, this should be the relative
# path to the manifest inside the extracted test directory
# otherwise, should be path to a test manifest on the local file system
- "test_manifest": "path_to_test_manifest",
+ "test_manifest": "tests/firefox/firefox_all.ini",
# optional, use an existing profile (temp profile created by default)
"profile_path": None,
# global timeout in seconds (without output)
@@ -38,15 +30,9 @@
"tracer_interval": 10,
# URL or path to the symbols directory for debugging crashes
"symbols_path": None,
-}
-# these config options depend on the abs_work_dir option
-abs_work_dir = os.path.abspath(os.path.join(config['base_work_dir'],
- config['work_dir']))
-
-config['virtualenv_path'] = os.path.join(abs_work_dir, "venv")
-# directory to extract tests to
-config['test_install_dir'] = os.path.join(abs_work_dir, "tests")
-# directory to install application to
-config['application_install_dir'] = os.path.join(abs_work_dir,
- "application")
+ "exes": {
+ 'python': '/tools/buildbot/bin/python',
+ 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+ },
+}
View
19 configs/peptest/test_config.py
@@ -2,18 +2,13 @@
config = {
# mozharness script options
- "base_work_dir": os.getcwd(),
- "work_dir": "build",
"log_name": "pep",
- "log_level": "info",
- "test_url": "https://github.com/downloads/ahal/peptest/tests.zip",
- "mozbase_path": "https://github.com/mozilla/mozbase/zipball/master",
- "peptest_path": "https://github.com/mozilla/peptest/zipball/master",
+ "test_url": "ftp://ftp.mozilla.org/pub/firefox/tinderbox-builds/mozilla-central-linux/1323881237/firefox-11.0a1.en-US.linux-i686.tests.zip",
# peptest options
- "appname": "ftp://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-10.0a1.en-US.linux-i686.tar.bz2",
+ "appname": "ftp://ftp.mozilla.org/pub/firefox/tinderbox-builds/mozilla-central-linux/1323881237/firefox-11.0a1.en-US.linux-i686.tar.bz2",
"app": "firefox",
- "test_manifest": "firefox/all_tests.ini", # this is relative to the test folder specified by test_url
+ "test_manifest": "tests/firefox/firefox_all.ini", # this is relative to the test folder specified by test_url
"profile_path": None,
"timeout": 60,
"server_path": None,
@@ -27,11 +22,3 @@
"get_latest_tinderbox_platform": None,
"get_latest_tinderbox_debug_build": False,
}
-
-abs_work_dir = os.path.abspath(os.path.join(config['base_work_dir'],
- config['work_dir']))
-
-config['virtualenv_path'] = os.path.join(abs_work_dir, "venv")
-config['test_install_dir'] = os.path.join(abs_work_dir, "tests")
-config['application_install_dir'] = os.path.join(abs_work_dir,
- "application")
View
4 configs/peptest/user_config.py
@@ -12,9 +12,9 @@
"log_level": "info",
"test_url": "url_to_packaged_tests",
# path or url to a zip or folder containing the mozbase packages
- "mozbase_path": "url_to_mozbase_zip",
+ "mozbase_url": "url_to_mozbase_zip",
# path or url to a zip or folder containing peptest
- "peptest_path": "url_to_peptest_zip",
+ "peptest_url": "url_to_peptest_zip",
# peptest options
View
44 configs/peptest/win7_config.py
@@ -0,0 +1,44 @@
+# This is a template config file for peptest production
+
+# The peptest mozharness script is set up so that specifying None
+# is the same as not specifying the option at all
+
+config = {
+ # mozharness script options
+ "log_name": "pep",
+ "buildbot_json_path": "buildprops.json",
+ "virtualenv_path": 'c:/talos-slave/test/build/venv',
+ "virtualenv_modules": ["simplejson"],
+ "virtualenv_python_dll": 'c:/mozilla-build/python25/python25.dll',
+# "virtualenv_options": ['--no-site-packages', '--distribute', '--never-download'],
+ "distribute_url": "http://build.mozilla.org/talos/zips/distribute-0.6.24.tar.gz",
+ "pip_url": "http://build.mozilla.org/talos/zips/pip-1.0.2.tar.gz",
+ "simplejson_url": "http://build.mozilla.org/talos/zips/simplejson-2.2.1.tar.gz",
+ # peptest options
+ # defaults to firefox, can also be thunderbird, fennec, etc.
+ "app": "firefox",
+ # if test_url is specified, this should be the relative
+ # path to the manifest inside the extracted test directory
+ # otherwise, should be path to a test manifest on the local file system
+ "test_manifest": "tests/firefox/firefox_all.ini",
+ # optional, use an existing profile (temp profile created by default)
+ "profile_path": None,
+ # global timeout in seconds (without output)
+ "timeout": 60,
+ # if specified, creates a webserver for hosting test
+ # related files at this document root
+ "server_path": None,
+ "server_port": None,
+ # EventTracer setting, the threshold to count a failure (ms)
+ "tracer_threshold": 50,
+ # EventTracer setting, interval at which to send tracer events (ms)
+ "tracer_interval": 10,
+ # URL or path to the symbols directory for debugging crashes
+ "symbols_path": None,
+
+ "exes": {
+ 'python': 'c:/mozilla-build/python25/python',
+ 'virtualenv': ['c:/mozilla-build/python25/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
+ 'hg': 'c:/mozilla-build/hg/hg',
+ },
+}
View
43 configs/peptest/xp_config.py
@@ -0,0 +1,43 @@
+# This is a template config file for peptest production
+
+# The peptest mozharness script is set up so that specifying None
+# is the same as not specifying the option at all
+
+config = {
+ # mozharness script options
+ "log_name": "pep",
+ "buildbot_json_path": "buildprops.json",
+ "virtualenv_path": 'c:/talos-slave/test/build/venv',
+ "virtualenv_modules": ["simplejson"],
+ "virtualenv_python_dll": 'c:/mozilla-build/python25/python25.dll',
+ "distribute_url": "http://build.mozilla.org/talos/zips/distribute-0.6.24.tar.gz",
+ "pip_url": "http://build.mozilla.org/talos/zips/pip-1.0.2.tar.gz",
+ "simplejson_url": "http://build.mozilla.org/talos/zips/simplejson-2.2.1.tar.gz",
+ # peptest options
+ # defaults to firefox, can also be thunderbird, fennec, etc.
+ "app": "firefox",
+ # if test_url is specified, this should be the relative
+ # path to the manifest inside the extracted test directory
+ # otherwise, should be path to a test manifest on the local file system
+ "test_manifest": "tests/firefox/firefox_all.ini",
+ # optional, use an existing profile (temp profile created by default)
+ "profile_path": None,
+ # global timeout in seconds (without output)
+ "timeout": 60,
+ # if specified, creates a webserver for hosting test
+ # related files at this document root
+ "server_path": None,
+ "server_port": None,
+ # EventTracer setting, the threshold to count a failure (ms)
+ "tracer_threshold": 50,
+ # EventTracer setting, interval at which to send tracer events (ms)
+ "tracer_interval": 10,
+ # URL or path to the symbols directory for debugging crashes
+ "symbols_path": None,
+
+ "exes": {
+ 'python': 'c:/mozilla-build/python25/python',
+ 'virtualenv': ['c:/mozilla-build/python25/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
+ 'hg': 'c:/mozilla-build/hg/hg',
+ },
+}
View
70 mozharness/base/python.py
@@ -48,14 +48,13 @@
["--venv-path", "--virtualenv-path"],
{"action": "store",
"dest": "virtualenv_path",
- "default": os.path.join(os.getcwd(), "venv"),
- "help": "Specify the virtualenv path"
- }
+ "default": "venv",
+ "help": "Specify the path to the virtualenv top level directory"
+ }
],
[["--virtualenv"],
{"action": "store",
"dest": "virtualenv",
- "default": "virtualenv",
"help": "Specify the virtualenv executable to use"
}
]]
@@ -72,6 +71,15 @@ class VirtualenvMixin(object):
'''
python_paths = {}
+ def query_virtualenv_path(self):
+ c = self.config
+ dirs = self.query_abs_dirs()
+ if 'abs_virtualenv_dir' in dirs:
+ return dirs['abs_virtualenv_dir']
+ if os.path.isabs(c['virtualenv_path']):
+ return c['virtualenv_path']
+ return os.path.join(dirs['abs_base_dir'], c['virtualenv_path'])
+
def query_python_path(self, binary="python"):
"""Return the path of a binary inside the virtualenv, if
c['virtualenv_path'] is set; otherwise return the binary name.
@@ -81,11 +89,12 @@ def query_python_path(self, binary="python"):
bin_dir = 'bin'
if self._is_windows():
bin_dir = 'Scripts'
- if self.config.get('virtualenv_path'):
- self.python_paths[binary] = os.path.abspath(os.path.join(self.config['virtualenv_path'], bin_dir, binary))
+ virtualenv_path = self.query_virtualenv_path()
+ if virtualenv_path:
+ self.python_paths[binary] = os.path.abspath(os.path.join(virtualenv_path, bin_dir, binary))
else:
- self.python_paths[binary] = binary
- return self.which(self.python_paths[binary])
+ self.python_paths[binary] = self.query_exe(binary)
+ return self.python_paths[binary]
def query_package(self, package_name, error_level=WARNING):
"""
@@ -106,18 +115,41 @@ def query_package(self, package_name, error_level=WARNING):
def create_virtualenv(self):
c = self.config
- if not c.get('virtualenv_path'):
- self.add_summary("No virtualenv specified; not creating virtualenv!", level=FATAL)
- return -1
- venv_path = os.path.abspath(c['virtualenv_path'])
+ dirs = self.query_abs_dirs()
+ venv_path = self.query_virtualenv_path()
self.info("Creating virtualenv %s" % venv_path)
- virtualenv = c['virtualenv']
- if not os.path.exists(virtualenv) and not self.which(virtualenv):
- self.add_summary("The executable '%s' is not found; not creating virtualenv!" % virtualenv, level=FATAL)
- return -1
-
- self.run_command([virtualenv, "--no-site-packages",
- venv_path],
+ virtualenv = c.get('virtualenv', self.query_exe('virtualenv'))
+ if isinstance(virtualenv, str):
+ if not os.path.exists(virtualenv) and not self.which(virtualenv):
+ self.add_summary("The executable '%s' is not found; not creating virtualenv!" % virtualenv, level=FATAL)
+ return -1
+ # allow for [python, virtualenv] in config
+ virtualenv = [virtualenv]
+
+ # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3
+ # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50
+ if c.get('virtualenv_python_dll'):
+ # We may someday want to copy a differently-named dll, but
+ # let's not think about that right now =\
+ dll_name = os.path.basename(c['virtualenv_python_dll'])
+ target = self.query_python_path(dll_name)
+ scripts_dir = os.path.dirname(target)
+ self.mkdir_p(scripts_dir)
+ self.copyfile(c['virtualenv_python_dll'], target)
+ else:
+ self.mkdir_p(dirs['abs_work_dir'])
+
+ # make this list configurable?
+ for module in ('distribute', 'pip'):
+ if c.get('%s_url' % module):
+ self.download_file(c['%s_url' % module],
+ parent_dir=dirs['abs_work_dir'])
+
+ virtualenv_options = c.get('virtualenv_options',
+ ['--no-site-packages', '--distribute'])
+
+ self.run_command(virtualenv + virtualenv_options + [venv_path],
+ cwd=dirs['abs_work_dir'],
error_list=PythonErrorList,
halt_on_failure=True)
pip = self.query_python_path("pip")
View
72 mozharness/base/script.py
@@ -103,9 +103,10 @@ def rmtree(self, path, error_level=ERROR, exit_code=-1):
return 0
def _is_windows(self):
- if platform.system() in ("Windows",):
+ system = platform.system()
+ if system in ("Windows", "Microsoft"):
return True
- if platform.system().startswith("CYGWIN"):
+ if system.startswith("CYGWIN"):
return True
def _rmdir_recursive(self, path):
@@ -146,21 +147,31 @@ def get_filename_from_url(self, url):
# http://www.techniqal.com/blog/2008/07/31/python-file-read-write-with-urllib2/
# TODO thinking about creating a transfer object.
- def download_file(self, url, file_name=None,
- error_level=ERROR, exit_code=-1):
+ def download_file(self, url, file_name=None, parent_dir=None,
+ create_parent_dir=True, error_level=ERROR,
+ exit_code=-1):
"""Python wget.
- TODO: option to mkdir_p dirname(file_name) if it doesn't exist.
TODO: should noop touch the filename? seems counter-noop.
"""
if not file_name:
- file_name = self.get_filename_from_url(url)
+ try:
+ file_name = self.get_filename_from_url(url)
+ except AttributeError:
+ self.log("Unable to get filename from %s; bad url?" % url,
+ level=error_level, exit_code=exit_code)
+ return
+ if parent_dir:
+ file_name = os.path.join(parent_dir, file_name)
+ parent_dir = os.path.dirname(file_name)
if self.config.get('noop'):
self.info("Downloading %s" % url)
return file_name
req = urllib2.Request(url)
try:
self.info("Downloading %s" % url)
f = urllib2.urlopen(req)
+ if create_parent_dir and parent_dir:
+ self.mkdir_p(parent_dir)
local_file = open(file_name, 'wb')
local_file.write(f.read())
local_file.close()
@@ -187,21 +198,38 @@ def extract(self, path, extdir=None, delete=False,
# determine directory to extract to
if extdir is None:
extdir = os.path.dirname(path)
- elif not os.path.exists(extdir):
+ elif not os.path.isdir(extdir):
+ if os.path.isfile(extdir):
+ self.log("%s is a file!" % extdir, level=error_level,
+ exit_code=exit_code)
self.mkdir_p(extdir)
self.info("Extracting %s to %s" % (os.path.abspath(path),
os.path.abspath(extdir)))
try:
if zipfile.is_zipfile(path):
bundle = zipfile.ZipFile(path)
namelist = bundle.namelist()
+ if hasattr(bundle, 'extractall'):
+ bundle.extractall(path=extdir)
+ # zipfile.extractall doesn't exist in Python 2.5
+ else:
+ for name in namelist:
+ filename = os.path.realpath(os.path.join(extdir, name))
+ if name.endswith("/"):
+ os.makedirs(filename)
+ else:
+ path = os.path.dirname(filename)
+ if not os.path.isdir(path):
+ os.makedirs(path)
+ dest = open(filename, "wb")
+ dest.write(bundle.read(name))
elif tarfile.is_tarfile(path):
bundle = tarfile.open(path)
namelist = bundle.getnames()
+ bundle.extractall(path=extdir)
else:
- # unkown filetype
+ # unknown filetype
self.warning("Unsupported file type: %s" % path)
- bundle.extractall(path=extdir)
bundle.close()
except (zipfile.BadZipfile, zipfile.LargeZipFile,
tarfile.ReadError, tarfile.CompressionError), e:
@@ -216,13 +244,11 @@ def extract(self, path, extdir=None, delete=False,
# namelist returns paths with forward slashes even in windows
top_level_files = [os.path.join(extdir, name) for name in namelist
if len(name.rstrip('/').split('/')) == 1]
- # namelist doesn't include folders in windows, append these to the list
- if platform.system() == "Windows":
- for name in namelist:
- root = name[:name.find('/')]
- if root not in top_level_files:
- top_level_files.append(root)
- # return list of paths of top level extracted files
+ # namelist doesn't include folders, append these to the list
+ for name in namelist:
+ root = os.path.join(extdir, name[:name.find('/')])
+ if root not in top_level_files:
+ top_level_files.append(root)
return top_level_files
def move(self, src, dest, error_level="error", exit_code=-1):
@@ -322,6 +348,18 @@ def query_env(self, partial_env=None, replace_dict=None):
self.env = env
return env
+ def query_exe(self, exe_name, exe_dict='exes'):
+ """One way to work around PATH rewrites.
+
+ By default, return exe_name, and we'll fall through to searching
+ os.environ["PATH"].
+ However, if self.config[exe_dict][exe_name] exists, return that.
+ This lets us override exe paths via config file.
+
+ If we need runtime setting, we can build in self.exes support later.
+ """
+ return self.config.get(exe_dict, {}).get(exe_name, exe_name)
+
def run_command(self, command, cwd=None, error_list=[], parse_at_end=False,
halt_on_failure=False, success_codes=[0],
env=None, return_type='status', throw_exception=False):
@@ -610,7 +648,7 @@ def clobber(self):
"""
dirs = self.query_abs_dirs()
self.rmtree(dirs['abs_work_dir'])
-
+
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
View
40 mozharness/base/vcs/mercurial.py
@@ -55,7 +55,7 @@
from mozharness.base.log import LogMixin
from mozharness.base.script import BaseScript, ShellMixin, OSMixin
-HG = ['hg', '--config', 'ui.merge=internal:merge']
+HG_OPTIONS = ['--config', 'ui.merge=internal:merge']
# MercurialVCS {{{1
# TODO Make the remaining functions more mozharness-friendly.
@@ -92,7 +92,10 @@ def __init__(self, log_obj=None, config=None, vcs_config=None):
super(MercurialVCS, self).__init__()
self.can_share = None
self.log_obj = log_obj
- self.config = config
+ if config:
+ self.config = config
+ else:
+ self.config = {}
# vcs_config = {
# hg_host: hg_host,
# repo: repository,
@@ -102,6 +105,7 @@ def __init__(self, log_obj=None, config=None, vcs_config=None):
# ssh_key: ssh_key,
# }
self.vcs_config = vcs_config
+ self.hg = [self.query_exe('hg')] + HG_OPTIONS
def _make_absolute(self, repo):
if repo.startswith("file://"):
@@ -125,24 +129,24 @@ def get_repo_path(self, repo):
def get_revision_from_path(self, path):
"""Returns which revision directory `path` currently has checked out."""
return self.get_output_from_command(
- HG + ['parent', '--template', '{node|short}'], cwd=path
+ self.hg + ['parent', '--template', '{node|short}'], cwd=path
)
def get_branch_from_path(self, path):
- branch = self.get_output_from_command(HG + ['branch'], cwd=path)
+ branch = self.get_output_from_command(self.hg + ['branch'], cwd=path)
return str(branch).strip()
def get_branches_from_path(self, path):
branches = []
- for line in self.get_output_from_command(HG + ['branches', '-c'],
+ for line in self.get_output_from_command(self.hg + ['branches', '-c'],
cwd=path).splitlines():
branches.append(line.split()[0])
return branches
def hg_ver(self):
"""Returns the current version of hg, as a tuple of
(major, minor, build)"""
- ver_string = self.get_output_from_command(HG + ['-q', 'version'])
+ ver_string = self.get_output_from_command(self.hg + ['-q', 'version'])
match = re.search("\(version ([0-9.]+)\)", ver_string)
if match:
bits = match.group(1).split(".")
@@ -169,13 +173,13 @@ def update(self, dest, branch=None, revision=None):
msg += " revision %s" % revision
self.info("%s." % msg)
if revision is not None:
- cmd = HG + ['update', '-C', '-r', revision]
+ cmd = self.hg + ['update', '-C', '-r', revision]
self.run_command(cmd, cwd=dest, error_list=HgErrorList)
else:
# Check & switch branch
local_branch = self.get_branch_from_path(dest)
- cmd = HG + ['update', '-C']
+ cmd = self.hg + ['update', '-C']
# If this is different, checkout the other branch
if branch and branch != local_branch:
@@ -207,7 +211,7 @@ def clone(self, repo, dest, branch=None, revision=None, update_dest=True):
self.info("Removing %s before clone." % dest)
self.rmtree(dest)
- cmd = HG + ['clone']
+ cmd = self.hg + ['clone']
if not update_dest:
cmd.append('-U')
@@ -265,7 +269,7 @@ def pull(self, repo, dest, update_dest=True, **kwargs):
return -1
# Convert repo to an absolute path if it's a local repository
repo = self._make_absolute(repo)
- cmd = HG + ['pull']
+ cmd = self.hg + ['pull']
cmd.extend(self.common_args(**kwargs))
cmd.append(repo)
self.run_command(cmd, cwd=dest, error_list=HgErrorList,
@@ -281,7 +285,7 @@ def pull(self, repo, dest, update_dest=True, **kwargs):
def out(self, src, remote, **kwargs):
"""Check for outgoing changesets present in a repo"""
self.info("Checking for outgoing changesets from %s to %s." % (src, remote))
- cmd = HG + ['-q', 'out', '--template', '{node} {branches}\n']
+ cmd = self.hg + ['-q', 'out', '--template', '{node} {branches}\n']
cmd.extend(self.common_args(**kwargs))
cmd.append(remote)
if os.path.exists(src):
@@ -309,7 +313,7 @@ def push(self, src, remote, push_new_branches=True, **kwargs):
# This doesn't appear to work with hg_ver < (1, 6, 0).
# Error out, or let you try?
self.info("Pushing new changes from %s to %s." % (src, remote))
- cmd = HG + ['push']
+ cmd = self.hg + ['push']
cmd.extend(self.common_args(**kwargs))
if push_new_branches and self.hg_ver() >= (1, 6, 0):
cmd.append('--new-branch')
@@ -325,7 +329,7 @@ def query_can_share(self):
self.can_share = True
try:
self.info("Checking if share extension works.")
- output = self.get_output_from_command(HG + ['help', 'share'],
+ output = self.get_output_from_command(self.hg + ['help', 'share'],
silent=True,
throw_exception=True)
if 'no commands defined' in output:
@@ -432,7 +436,7 @@ def share(self, source, dest, branch=None, revision=None):
with "source" using Mercurial's share extension
"""
self.info("Sharing %s to %s." % (source, dest))
- self.run_command(HG + ['share', '-U', source, dest],
+ self.run_command(self.hg + ['share', '-U', source, dest],
error_list=HgErrorList,
throw_exception=True)
return self.update(dest, branch=branch, revision=revision)
@@ -513,22 +517,22 @@ def apply_and_push(self, localrepo, remote, changer, max_attempts=10,
if n == max_attempts:
self.debug("Tried %d times, giving up" % max_attempts)
for r in reversed(new_revs):
- self.run_command(HG + ['strip', '-n', r[REVISION]],
+ self.run_command(self.hg + ['strip', '-n', r[REVISION]],
cwd=localrepo, error_list=HgErrorList)
raise VCSException("Failed to push")
self.pull(remote, localrepo, update_dest=False,
ssh_username=ssh_username, ssh_key=ssh_key)
# After we successfully rebase or strip away heads the push
# is is attempted again at the start of the loop
try:
- self.run_command(HG + ['rebase'], cwd=localrepo,
+ self.run_command(self.hg + ['rebase'], cwd=localrepo,
error_list=HgErrorList,
throw_exception=True)
except subprocess.CalledProcessError, e:
self.debug("Failed to rebase: %s" % str(e))
self.update(localrepo, branch=branch)
for r in reversed(new_revs):
- self.run_command(HG + ['strip', '-n', r[REVISION]],
+ self.run_command(self.hg + ['strip', '-n', r[REVISION]],
cwd=localrepo, error_list=HgErrorList)
changer(localrepo, n+1)
@@ -539,7 +543,7 @@ def cleanOutgoingRevs(self, reponame, remote, username, sshKey):
outgoingRevs = self.out(src=reponame, remote=remote,
ssh_username=username, ssh_key=sshKey)
for r in reversed(outgoingRevs):
- self.run_command(HG + ['strip', '-n', r[REVISION]],
+ self.run_command(self.hg + ['strip', '-n', r[REVISION]],
cwd=reponame, error_list=HgErrorList)
View
88 mozharness/buildbot.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla.
+#
+# The Initial Developer of the Original Code is
+# the Mozilla Foundation <http://www.mozilla.org/>.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Aki Sasaki <aki@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+"""Code to tie into buildbot.
+Ideally this will go away if and when we retire buildbot.
+"""
+
+import os
+import pprint
+import sys
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.config import parse_config_file
+from mozharness.base.log import DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL, IGNORE
+
+# BuildbotMixin {{{1
+
+TBPL_SUCCESS = 'SUCCESS'
+TBPL_WARNING = 'WARNING'
+TBPL_FAILURE = 'FAILURE'
+TBPL_EXCEPTION = 'EXCEPTION'
+TBPL_RETRY = 'RETRY'
+TBPL_STATUS_DICT = {
+ TBPL_SUCCESS: INFO,
+ TBPL_WARNING: WARNING,
+ TBPL_FAILURE: ERROR,
+ TBPL_EXCEPTION: ERROR,
+ TBPL_RETRY: WARNING,
+}
+
+class BuildbotMixin(object):
+ buildbot_config = None
+
+ def read_buildbot_config(self):
+ c = self.config
+ if not c.get("buildbot_json_path"):
+ # If we need to fail out, add postflight_read_buildbot_config()
+ self.info("buildbot_json_path is not set. Skipping...")
+ else:
+ # TODO try/except?
+ self.buildbot_config = parse_config_file(c['buildbot_json_path'])
+ self.info(pprint.pformat(self.buildbot_config))
+
+ def tryserver_email(self):
+ pass
+
+ def buildbot_status(self, tbpl_status, level=None):
+ if tbpl_status not in TBPL_STATUS_DICT:
+ self.error("buildbot_status() doesn't grok the status %s!" % tbpl_status)
+ else:
+ if not level:
+ level = TBPL_STATUS_DICT[tbpl_status]
+ self.add_summary("# TBPL %s #" % tbpl_status, level=level)
View
283 scripts/peptest.py
@@ -36,18 +36,23 @@
#
# ***** END LICENSE BLOCK *****
+import os
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
from mozharness.base.errors import PythonErrorList
from mozharness.base.log import DEBUG, INFO, WARNING, ERROR, FATAL
from mozharness.base.python import virtualenv_config_options, VirtualenvMixin
from mozharness.base.script import BaseScript
+from mozharness.buildbot import BuildbotMixin, TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE, TBPL_RETRY
import urlparse
import tarfile
import zipfile
import platform
-import os
-import sys
-class PepTest(VirtualenvMixin, BaseScript):
+class PepTest(VirtualenvMixin, BuildbotMixin, BaseScript):
config_options = [
[["--appname"],
{"action": "store",
@@ -61,12 +66,6 @@ class PepTest(VirtualenvMixin, BaseScript):
"default":None,
"help": "Path to test manifest to run",
}],
- [["--mozbase-url"],
- {"action":"store",
- "dest": "mozbase_url",
- "default": "https://github.com/mozilla/mozbase/zipball/master",
- "help": "URL to mozbase zip file",
- }],
[["--peptest-url"],
{"action": "store",
"dest": "peptest_url",
@@ -91,98 +90,180 @@ def __init__(self, require_config_file=False):
super(PepTest, self).__init__(
config_options=self.config_options,
all_actions=['clobber',
- 'create-deps',
+ 'create-virtualenv',
+ 'read-buildbot-config',
'get-latest-tinderbox',
+ 'create-deps',
'run-peptest'],
- default_actions=['run-peptest'],
+ default_actions=['clobber',
+ 'create-virtualenv',
+ 'read-buildbot-config',
+ 'create-deps',
+ 'run-peptest'],
require_config_file=require_config_file,
config={'dependencies': ['mozlog',
'mozinfo',
'mozhttpd',
+ 'mozinstall',
'manifestdestiny',
'mozprofile',
'mozprocess',
- 'mozrunner']})
+ 'mozrunner'],})
# these are necessary since self.config is read only
- self.appname = self.config.get('appname')
- self.symbols = self.config.get('symbols_path')
- self.test_path = self.config.get('test_manifest')
+ c = self.config
+ dirs = self.query_abs_dirs()
+ self.appname = c.get('appname')
+ self.symbols = c.get('symbols_path')
+ self.test_path = os.path.join(dirs['abs_peptest_dir'],
+ c['test_manifest'])
+ self.test_url = self.config.get('test_url')
+
+
+ # Helper methods {{{1
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = super(PepTest, self).query_abs_dirs()
+ c = self.config
+ dirs = {}
+ dirs['abs_test_install_dir'] = os.path.join(
+ abs_dirs['abs_work_dir'],
+ c.get('test_install_dir', 'tests'))
+ dirs['abs_app_install_dir'] = os.path.join(
+ abs_dirs['abs_work_dir'],
+ c.get('app_install_dir', 'application'))
+ dirs['abs_mozbase_dir'] = os.path.join(
+ dirs['abs_test_install_dir'], "mozbase")
+ dirs['abs_peptest_dir'] = os.path.join(
+ dirs['abs_test_install_dir'], "peptest")
+ if os.path.isabs(c['virtualenv_path']):
+ dirs['abs_virtualenv_dir'] = c['virtualenv_path']
+ else:
+ dirs['abs_virtualenv_dir'] = os.path.join(
+ abs_dirs['abs_work_dir'],
+ c['virtualenv_path'])
+ for key in dirs.keys():
+ if key not in abs_dirs:
+ abs_dirs[key] = dirs[key]
+ self.abs_dirs = abs_dirs
+ return self.abs_dirs
- def create_deps(self):
+ def _is_url(self, path):
"""
- Create virtualenv and install dependencies
+ Return True if path looks like a URL.
"""
- self.create_virtualenv()
- self._install_deps()
- self._install_peptest()
+ if path is not None:
+ parsed = urlparse.urlparse(path)
+ return parsed.scheme != '' or parsed.netloc != ''
+ return False
- def _install_deps(self):
+ def _build_arg(self, option, value):
"""
- Download and install dependencies
+ Build a command line argument
"""
- # download and extract mozbase
- work_dir = self.query_abs_dirs()['abs_work_dir']
+ if not value:
+ return []
+ return [str(option), str(value)]
- mozbase = self.config.get('mozbase_path');
- if not mozbase:
- self.fatal("No path to mozbase specified. Aborting")
+ def _install_from_url(self, url, error_level=FATAL):
+ """
+ Accepts a URL to the application (usually on ftp.m.o)
+ Downloads and installs the application
+ Returns the binary path
+ """
+ c = self.config
+ dirs = self.query_abs_dirs()
+
+ # download the application
+ source = self.download_file(url,
+ error_level=error_level,
+ parent_dir=dirs['abs_work_dir'])
+ if not source:
+ return
+ source = os.path.realpath(source)
- if self._is_url(mozbase):
- mozbase = self.download_file(mozbase,
- file_name=os.path.join(work_dir, 'mozbase'),
- error_level=FATAL)
+ # install the application
+ mozinstall = self.query_python_path("mozinstall")
+ cmd = [mozinstall, '--source', source]
+ cmd.extend(self._build_arg('--destination',
+ dirs['abs_app_install_dir']))
+ binary = self.get_output_from_command(cmd)
- if os.path.isfile(mozbase):
- mozbase = self.extract(mozbase, delete=True, error_level=FATAL)[0]
+ # cleanup
+ return binary
+ def _install_deps(self):
+ """
+ Download and install dependencies
+ """
+ dirs = self.query_abs_dirs()
python = self.query_python_path()
# install dependencies
for module in self.config['dependencies']:
- self.run_command(python + " setup.py install",
- cwd=os.path.join(mozbase, module),
+ self.run_command([python, "setup.py", "install"],
+ cwd=os.path.join(dirs['abs_mozbase_dir'], module),
error_list=PythonErrorList)
- self.rmtree(mozbase)
def _install_peptest(self):
"""
Download and install peptest
"""
- # download and extract peptest
- work_dir = self.query_abs_dirs()['abs_work_dir']
+ dirs = self.query_abs_dirs()
+ python = self.query_python_path()
+ self.run_command([python, "setup.py", "install"],
+ cwd=dirs['abs_peptest_dir'],
+ error_list=PythonErrorList)
- peptest = self.config.get('peptest_path')
- if not peptest:
- self.fatal("No path to peptest specified. Aborting")
- if self._is_url(peptest):
- peptest = self.download_file(peptest,
- file_name=os.path.join(work_dir, 'peptest'),
- error_level=FATAL)
- if os.path.isfile(peptest):
- peptest = self.extract(peptest, delete=True, error_level=FATAL)[0]
+ # Actions {{{1
+ # create_virtualenv is in VirtualenvMixin.
+ # read_buildbot_config is in BuildbotMixin.
+
+ def postflight_read_buildbot_config(self):
+ if self.buildbot_config:
+ try:
+ files = self.buildbot_config['sourcestamp']['changes'][0]['files']
+ for file_num in (0, 1):
+ if files[file_num]['name'].endswith('tests.zip'): # yuk
+ # str() because of unicode issues on mac
+ self.test_url = str(files[file_num]['name'])
+ else:
+ self.appname = str(files[file_num]['name'])
+ except IndexError, e:
+ self.fatal("Unable to set appname+test_url from the the buildbot config: %s!" % str(e))
+
+
+ def create_deps(self):
+ """
+ Create virtualenv and install dependencies
+ """
+ c = self.config
+ dirs = self.query_abs_dirs()
+ if self.test_url:
+ bundle = self.download_file(self.test_url,
+ parent_dir=dirs['abs_work_dir'],
+ error_level=FATAL)
+ unzip = self.query_exe("unzip")
+ self.mkdir_p(dirs['abs_test_install_dir'])
+ # TODO error_list
+ self.run_command([unzip, bundle],
+ cwd=dirs['abs_test_install_dir'])
+ self._install_deps()
+ self._install_peptest()
- python = self.query_python_path()
- self.run_command(python + " setup.py install",
- cwd=peptest,
- error_list=PythonErrorList)
- self.rmtree(peptest)
def get_latest_tinderbox(self):
"""
Find the url to the latest-tinderbox build and
point the appname at it
"""
- if not self.query_python_path():
- self.create_virtualenv()
dirs = self.query_abs_dirs()
if len(self.query_package('getlatesttinderbox')) == 0:
# install getlatest-tinderbox
self.info("Installing getlatest-tinderbox")
pip = self.query_python_path("pip")
- if not pip:
- self.error("No application named 'pip' installed")
self.run_command(pip + " install GetLatestTinderbox",
cwd=dirs['abs_work_dir'],
error_list=PythonErrorList)
@@ -211,36 +292,18 @@ def preflight_run_peptest(self):
if not self.config.get('test_manifest'):
self.fatal("No test manifest specified. Aborting")
- if self.config.get('test_url'):
- bundle = self.download_file(self.config['test_url'])
- files = self.extract(bundle,
- extdir=self.config.get('test_install_dir'),
- delete=True)
- self.test_path = os.path.join(files[0],
- self.config['test_manifest'])
-
if not os.path.isfile(self.test_path):
- self.fatal("Test manifest does not exist. Aborting")
-
- if "create-deps" not in self.actions:
- # ensure all the dependencies are installed
- for module in self.config['dependencies'] + ['peptest']:
- if len(self.query_package(module)) == 0:
- self.action_message("Dependencies missing, " +
- "running create-deps step")
- self.create_deps()
- break
+ self.fatal("Test manifest %s does not exist. Aborting" % self.test_path)
if not self.appname:
- self.action_message("No appname specified, " +
- "running get-latest-tinderbox step")
- self.get_latest_tinderbox()
+ self.fatal("No appname specified! Rerun with appname set, or --get-latest-tinderbox")
def run_peptest(self):
"""
Run the peptests
"""
+ dirs = self.query_abs_dirs()
if self._is_url(self.appname):
self.appname = self._install_from_url(self.appname,
error_level=FATAL)
@@ -249,8 +312,9 @@ def run_peptest(self):
error_list.extend(PythonErrorList)
# build the peptest command arguments
- peptest = self.query_python_path('peptest')
- cmd = [peptest]
+ python = self.query_python_path('python')
+ cmd = [python, '-u', os.path.join(dirs['abs_peptest_dir'], 'peptest',
+ 'runpeptests.py')]
cmd.extend(self._build_arg('--app', self.config.get('app')))
cmd.extend(self._build_arg('--binary', self.appname))
cmd.extend(self._build_arg('--test-path', self.test_path))
@@ -270,74 +334,27 @@ def run_peptest(self):
['debug', 'info', 'warning', 'error']):
cmd.extend(['--log-level', self.config['log_level'].upper()])
- code = self.run_command(cmd,
- error_list=error_list)
+ code = self.run_command(cmd, error_list=error_list)
# get status and set summary
level = ERROR
if code == 0:
status = "success"
+ tbpl_status = TBPL_SUCCESS
level = INFO
elif code == 1:
status = "test failures"
+ tbpl_status = TBPL_WARNING
else:
status = "harness failure"
+ tbpl_status = TBPL_FAILURE
# TODO create a better summary for peptest
# for now just display return code
self.add_summary("%s exited with return code %s: %s" % (cmd[0],
code,
- status))
-
- def _build_arg(self, option, value):
- """
- Build a command line argument
- """
- if not value:
- return []
- return [str(option), str(value)]
-
- def _install_from_url(self, url, error_level=ERROR):
- """
- Accepts a URL to the application (usually on ftp.m.o)
- Downloads and installs the application
- Returns the binary path
- """
- dirs = self.query_abs_dirs()
- # ensure mozinstall is available
- if len(self.query_package("mozinstall")) == 0:
- # install mozinstall
- self.info("Installing mozinstall")
- pip = self.query_python_path("pip")
- if not pip:
- self.log("No application named 'pip' installed",
- level=error_level)
- self.run_command(pip + " install mozInstall",
- cwd=dirs['abs_work_dir'],
- error_list=PythonErrorList)
-
- # download the application
- source = os.path.realpath(self.download_file(url))
-
- # install the application
- mozinstall = self.query_python_path("mozinstall")
- cmd = [mozinstall, '--source', source]
- cmd.extend(self._build_arg('--destination',
- self.config.get('application_install_dir')))
- binary = self.get_output_from_command(cmd)
-
- # cleanup
- self.rmtree(source)
- return binary
-
- def _is_url(self, path):
- """
- Return True if path looks like a URL.
- """
- if path is not None:
- parsed = urlparse.urlparse(path)
- return parsed.scheme != '' or parsed.netloc != ''
- return False
-
+ status),
+ level=level)
+ self.buildbot_status(tbpl_status)
View
2 test/networked/test_base_script.py
@@ -33,7 +33,7 @@ def test_download_bad_domain(self):
s = script.BaseScript(initial_config_file='test/test.json')
os.mkdir('test_dir')
self.assertRaises(SystemExit, s.download_file,
- "http://www.mozilla.sdfsdsdf",
+ "http://www.mozilla.sdfsdsdf", file_name="foo",
error_level=FATAL)
def test_download_bad_file(self):
View
2 test/test_base_vcs_mercurial.py
@@ -18,7 +18,7 @@
bar
baz'''
-HG = mercurial.HG
+HG = ['hg'] + mercurial.HG_OPTIONS
def cleanup():
if os.path.exists('test_logs'):

0 comments on commit d0acf5d

Please sign in to comment.