Browse files

bug 692091 - mozharness code for peptest. p=ahal, r=me

  • Loading branch information...
1 parent e9eead4 commit 4e7318c67f6c283db7344a1163b652195a3a3838 @escapewindow escapewindow committed Nov 4, 2011
View
52 configs/peptest/prod_config.py
@@ -0,0 +1,52 @@
+# This is a template config file for peptest production
+
+# The peptest mozharness script is set up so that specifying None
+# is the same as not specifying the option at all
+import os
+
+config = {
+ # mozharness script options
+ "base_work_dir": os.getcwd(),
+ "work_dir": "build",
+ "log_name": "pep",
+ "log_level": "info",
+ "test_url": "url_to_packaged_tests",
+ # path or url to a zip or folder containing the mozbase packages
+ "mozbase_path": "url_to_mozbase_zip",
+ # path or url to a zip or folder containing peptest
+ "peptest_path": "url_to_peptest_zip",
+
+ # peptest options
+ "appname": "url_to_application", # i.e the firefox build on ftp.m.o
+ # defaults to firefox, can also be thunderbird, fennec, etc.
+ "app": "firefox",
+ # if test_url is specified, this should be the relative
+ # path to the manifest inside the extracted test directory
+ # otherwise, should be path to a test manifest on the local file system
+ "test_manifest": "path_to_test_manifest",
+ # optional, use an existing profile (temp profile created by default)
+ "profile_path": None,
+ # global timeout in seconds (without output)
+ "timeout": 60,
+ # if specified, creates a webserver for hosting test
+ # related files at this document root
+ "server_path": None,
+ "server_port": None,
+ # EventTracer setting, the threshold to count a failure (ms)
+ "tracer_threshold": 50,
+ # EventTracer setting, interval at which to send tracer events (ms)
+ "tracer_interval": 10,
+ # URL or path to the symbols directory for debugging crashes
+ "symbols_path": None,
+}
+
+# these config options depend on the abs_work_dir option
+abs_work_dir = os.path.abspath(os.path.join(config['base_work_dir'],
+ config['work_dir']))
+
+config['virtualenv_path'] = os.path.join(abs_work_dir, "venv")
+# directory to extract tests to
+config['test_install_dir'] = os.path.join(abs_work_dir, "tests")
+# directory to install application to
+config['application_install_dir'] = os.path.join(abs_work_dir,
+ "application")
View
37 configs/peptest/test_config.py
@@ -0,0 +1,37 @@
+import os
+
+config = {
+ # mozharness script options
+ "base_work_dir": os.getcwd(),
+ "work_dir": "build",
+ "log_name": "pep",
+ "log_level": "info",
+ "test_url": "https://github.com/downloads/ahal/peptest/tests.zip",
+ "mozbase_path": "https://github.com/mozilla/mozbase/zipball/master",
+ "peptest_path": "https://github.com/mozilla/peptest/zipball/master",
+
+ # peptest options
+ "appname": "ftp://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-10.0a1.en-US.linux-i686.tar.bz2",
+ "app": "firefox",
+ "test_manifest": "firefox/all_tests.ini", # this is relative to the test folder specified by test_url
+ "profile_path": None,
+ "timeout": 60,
+ "server_path": None,
+ "server_port": None,
+ "tracer_threshold": 50,
+ "tracer_interval": 10,
+ "symbols_path": None,
+
+ # get latest tinderbox options
+ "get_latest_tinderbox_product": "mozilla-central",
+ "get_latest_tinderbox_platform": None,
+ "get_latest_tinderbox_debug_build": False,
+}
+
+abs_work_dir = os.path.abspath(os.path.join(config['base_work_dir'],
+ config['work_dir']))
+
+config['virtualenv_path'] = os.path.join(abs_work_dir, "venv")
+config['test_install_dir'] = os.path.join(abs_work_dir, "tests")
+config['application_install_dir'] = os.path.join(abs_work_dir,
+ "application")
View
56 configs/peptest/user_config.py
@@ -0,0 +1,56 @@
+# This is a template config file for peptest user
+
+# The peptest mozharness script is set up so that specifying None
+# is the same as not specifying the option at all
+import os
+
+config = {
+ # mozharness script options
+ "base_work_dir": os.getcwd(),
+ "work_dir": "build",
+ "log_name": "pep",
+ "log_level": "info",
+ "test_url": "url_to_packaged_tests",
+ # path or url to a zip or folder containing the mozbase packages
+ "mozbase_path": "url_to_mozbase_zip",
+ # path or url to a zip or folder containing peptest
+ "peptest_path": "url_to_peptest_zip",
+
+ # peptest options
+
+ "appname": "path_to_application_binary",
+ # defaults to firefox, can also be thunderbird, fennec, etc.
+ "app": "firefox",
+ "test_manifest": "path_to_test_manifest",
+ # optional, use an existing profile (temp profile created by default)
+ "profile_path": None,
+ # global timeout in seconds (without output)
+ "timeout": 60,
+ # if specified, creates a webserver for hosting test
+ # related files at this document root
+ "server_path": None,
+ "server_port": None,
+ # EventTracer setting, the threshold to count a failure (ms)
+ "tracer_threshold": 50,
+ # EventTracer setting, interval at which to send tracer events (ms)
+ "tracer_interval": 10,
+ # URL or path to the symbols directory for debugging crashes
+ "symbols_path": None,
+
+ # get latest tinderbox options
+ # (these are only used by the get-latest-tinderbox action)
+ "get_latest_tinderbox_product": "mozilla-central",
+ "get_latest_tinderbox_platform": None, # defaults to current platform
+ "get_latest_tinderbox_debug_build": False,
+}
+
+# these config options depend on the abs_work_dir option
+abs_work_dir = os.path.abspath(os.path.join(config['base_work_dir'],
+ config['work_dir']))
+
+config['virtualenv_path'] = os.path.join(abs_work_dir, "venv")
+# directory to extract tests to
+config['test_install_dir'] = os.path.join(abs_work_dir, "tests")
+# directory to install application to
+config['application_install_dir'] = os.path.join(abs_work_dir,
+ "application")
View
21 mozharness/base/python.py
@@ -61,12 +61,14 @@ class VirtualenvMixin(object):
* virtualenv_modules lists the module names.
* MODULE_url list points to the module URLs (optional)
Requires virtualenv to be in PATH.
+ Depends on OSMixin
'''
python_paths = {}
def query_python_path(self, binary="python"):
"""Return the path of a binary inside the virtualenv, if
c['virtualenv_path'] is set; otherwise return the binary name.
+ Otherwise return None
"""
if binary not in self.python_paths:
bin_dir = 'bin'
@@ -76,7 +78,24 @@ def query_python_path(self, binary="python"):
self.python_paths[binary] = os.path.abspath(os.path.join(self.config['virtualenv_path'], bin_dir, binary))
else:
self.python_paths[binary] = binary
- return self.python_paths[binary]
+ return self.which(self.python_paths[binary])
+
+ def query_package(self, package_name, error_level=WARNING):
+ """
+ Returns a list of all installed packages
+ that contain package_name in their name
+ """
+ pip = self.query_python_path("pip")
+ if not pip:
+ self.log("query_package: Program pip not in path", level=error_level)
+ return []
+ output = self.get_output_from_command(pip + " freeze",
+ silent=True)
+ if not output:
+ return []
+ packages = output.split()
+ return [package for package in packages
+ if package.lower().find(package_name.lower()) != -1]
def create_virtualenv(self):
c = self.config
View
27 mozharness/base/script.py
@@ -73,7 +73,7 @@
class OSMixin(object):
"""Filesystem commands and the like.
- Currently dependent on LogMixin, and a self.config of some sort.
+ Depends on LogMixin, ShellMixin, and a self.config of some sort.
"""
def mkdir_p(self, path):
if not os.path.exists(path):
@@ -161,7 +161,7 @@ def download_file(self, url, file_name=None,
try:
self.info("Downloading %s" % url)
f = urllib2.urlopen(req)
- local_file = open(file_name, 'w')
+ local_file = open(file_name, 'wb')
local_file.write(f.read())
local_file.close()
except urllib2.HTTPError, e:
@@ -258,6 +258,29 @@ def chdir(self, dir_name, ignore_if_noop=False):
else:
os.chdir(dir_name)
+ def which(self, program):
+ """
+ OS independent implementation of Unix's which command
+ Takes in a program name
+ Returns path to executable or None
+ """
+ def is_exe(fpath):
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+ if self._is_windows() and not program.endswith(".exe"):
+ program += ".exe"
+ fpath, fname = os.path.split(program)
+ if fpath:
+ if is_exe(program):
+ return program
+ else:
+ env = self.query_env()
+ for path in env["PATH"].split(os.pathsep):
+ exe_file = os.path.join(path, program)
+ if is_exe(exe_file):
+ return exe_file
+ return None
+
# ShellMixin {{{1
class ShellMixin(object):
"""These are very special but very complex methods that, together with
View
353 scripts/peptest.py
@@ -0,0 +1,353 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Peptest Mozharness script.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+from mozharness.base.errors import PythonErrorList
+from mozharness.base.log import DEBUG, INFO, WARNING, ERROR, FATAL
+from mozharness.base.python import virtualenv_config_options, VirtualenvMixin
+from mozharness.base.script import BaseScript
+import urlparse
+import tarfile
+import zipfile
+import platform
+import os
+import sys
+
+class PepTest(VirtualenvMixin, BaseScript):
+ config_options = [
+ [["--appname"],
+ {"action": "store",
+ "dest": "appname",
+ "default": None,
+ "help": "Path to the binary (file path or URL) to run the tests on",
+ }],
+ [["--test-manifest"],
+ {"action":"store",
+ "dest": "test_manifest",
+ "default":None,
+ "help": "Path to test manifest to run",
+ }],
+ [["--mozbase-url"],
+ {"action":"store",
+ "dest": "mozbase_url",
+ "default": "https://github.com/mozilla/mozbase/zipball/master",
+ "help": "URL to mozbase zip file",
+ }],
+ [["--peptest-url"],
+ {"action": "store",
+ "dest": "peptest_url",
+ "default": "https://github.com/mozilla/peptest/zipball/master",
+ "help": "URL to peptest zip file",
+ }],
+ [["--test-url"],
+ {"action":"store",
+ "dest": "test_url",
+ "default": None,
+ "help": "URL to the zip file containing the actual tests",
+ }]] + virtualenv_config_options
+
+ error_list = [
+ {'substr': r'''PEP TEST-UNEXPECTED-FAIL''', 'level': ERROR},
+ {'substr': r'''PEP ERROR''', 'level': ERROR},
+ {'substr': r'''PEP WARNING''', 'level': WARNING},
+ {'substr': r'''PEP DEBUG''', 'level': DEBUG},
+ ]
+
+ def __init__(self, require_config_file=False):
+ super(PepTest, self).__init__(
+ config_options=self.config_options,
+ all_actions=['clobber',
+ 'create-deps',
+ 'get-latest-tinderbox',
+ 'run-peptest'],
+ default_actions=['run-peptest'],
+ require_config_file=require_config_file,
+ config={'dependencies': ['mozlog',
+ 'mozinfo',
+ 'mozhttpd',
+ 'manifestdestiny',
+ 'mozprofile',
+ 'mozprocess',
+ 'mozrunner']})
+ # these are necessary since self.config is read only
+ self.appname = self.config.get('appname')
+ self.symbols = self.config.get('symbols_path')
+ self.test_path = self.config.get('test_manifest')
+
+ def create_deps(self):
+ """
+ Create virtualenv and install dependencies
+ """
+ self.create_virtualenv()
+ self._install_deps()
+ self._install_peptest()
+
+ def _install_deps(self):
+ """
+ Download and install dependencies
+ """
+ # download and extract mozbase
+ work_dir = self.query_abs_dirs()['abs_work_dir']
+
+ mozbase = self.config.get('mozbase_path');
+ if not mozbase:
+ self.fatal("No path to mozbase specified. Aborting")
+
+ if self._is_url(mozbase):
+ mozbase = self.download_file(mozbase,
+ file_name=os.path.join(work_dir, 'mozbase'),
+ error_level=FATAL)
+
+ if os.path.isfile(mozbase):
+ mozbase = self.extract(mozbase, delete=True, error_level=FATAL)[0]
+
+ python = self.query_python_path()
+ # install dependencies
+ for module in self.config['dependencies']:
+ self.run_command(python + " setup.py install",
+ cwd=os.path.join(mozbase, module),
+ error_list=PythonErrorList)
+ self.rmtree(mozbase)
+
+ def _install_peptest(self):
+ """
+ Download and install peptest
+ """
+ # download and extract peptest
+ work_dir = self.query_abs_dirs()['abs_work_dir']
+
+ peptest = self.config.get('peptest_path')
+ if not peptest:
+ self.fatal("No path to peptest specified. Aborting")
+
+ if self._is_url(peptest):
+ peptest = self.download_file(peptest,
+ file_name=os.path.join(work_dir, 'peptest'),
+ error_level=FATAL)
+
+ if os.path.isfile(peptest):
+ peptest = self.extract(peptest, delete=True, error_level=FATAL)[0]
+
+ python = self.query_python_path()
+ self.run_command(python + " setup.py install",
+ cwd=peptest,
+ error_list=PythonErrorList)
+ self.rmtree(peptest)
+
+ def clobber(self):
+ """
+ Delete the working directory
+ """
+ dirs = self.query_abs_dirs()
+ self.rmtree(dirs['abs_work_dir'])
+
+ def get_latest_tinderbox(self):
+ """
+ Find the url to the latest-tinderbox build and
+ point the appname at it
+ """
+ if not self.query_python_path():
+ self.create_virtualenv()
+ dirs = self.query_abs_dirs()
+
+ if len(self.query_package('getlatesttinderbox')) == 0:
+ # install getlatest-tinderbox
+ self.info("Installing getlatest-tinderbox")
+ pip = self.query_python_path("pip")
+ if not pip:
+ self.error("No application named 'pip' installed")
+ self.run_command(pip + " install GetLatestTinderbox",
+ cwd=dirs['abs_work_dir'],
+ error_list=PythonErrorList)
+
+ # get latest tinderbox build url
+ getlatest = self.query_python_path("get-latest-tinderbox")
+ cmd = [getlatest, '--latest']
+ cmd.extend(self._build_arg('--product',
+ self.config.get('get_latest_tinderbox_product')))
+ cmd.extend(self._build_arg('--platform',
+ self.config.get('get_latest_tinderbox_platform')))
+ if self.config.get('get_latest_tinderbox_debug_build'):
+ cmd.append('--debug')
+ url = self.get_output_from_command(cmd)
+
+ # get the symbols url to use for debugging crashes
+ cmd = [getlatest, '--url', url, '--symbols']
+ self.symbols = self.get_output_from_command(cmd)
+
+ # get the application url to download and install
+ cmd = [getlatest, '--url', url]
+ self.appname = self.get_output_from_command(cmd)
+
+
+ def preflight_run_peptest(self):
+ if not self.config.get('test_manifest'):
+ self.fatal("No test manifest specified. Aborting")
+
+ if self.config.get('test_url'):
+ bundle = self.download_file(self.config['test_url'])
+ files = self.extract(bundle,
+ extdir=self.config.get('test_install_dir'),
+ delete=True)
+ self.test_path = os.path.join(files[0],
+ self.config['test_manifest'])
+
+ if not os.path.isfile(self.test_path):
+ self.fatal("Test manifest does not exist. Aborting")
+
+ if "create-deps" not in self.actions:
+ # ensure all the dependencies are installed
+ for module in self.config['dependencies'] + ['peptest']:
+ if len(self.query_package(module)) == 0:
+ self.action_message("Dependencies missing, " +
+ "running create-deps step")
+ self.create_deps()
+ break
+
+ if not self.appname:
+ self.action_message("No appname specified, " +
+ "running get-latest-tinderbox step")
+ self.get_latest_tinderbox()
+
+
+ def run_peptest(self):
+ """
+ Run the peptests
+ """
+ if self._is_url(self.appname):
+ self.appname = self._install_from_url(self.appname,
+ error_level=FATAL)
+
+ error_list = self.error_list
+ error_list.extend(PythonErrorList)
+
+ # build the peptest command arguments
+ peptest = self.query_python_path('peptest')
+ cmd = [peptest]
+ cmd.extend(self._build_arg('--app', self.config.get('app')))
+ cmd.extend(self._build_arg('--binary', self.appname))
+ cmd.extend(self._build_arg('--test-path', self.test_path))
+ cmd.extend(self._build_arg('--profile-path',
+ self.config.get('profile_path')))
+ cmd.extend(self._build_arg('--timeout', self.config.get('timeout')))
+ cmd.extend(self._build_arg('--server-path',
+ self.config.get('server_path')))
+ cmd.extend(self._build_arg('--server-port',
+ self.config.get('server_port')))
+ cmd.extend(self._build_arg('--tracer-threshold',
+ self.config.get('tracer_threshold')))
+ cmd.extend(self._build_arg('--tracer-interval',
+ self.config.get('tracer_interval')))
+ cmd.extend(self._build_arg('--symbols-path', self.symbols))
+ if (self.config.get('log_level') in
+ ['debug', 'info', 'warning', 'error']):
+ cmd.extend(['--log-level', self.config['log_level'].upper()])
+
+ code = self.run_command(cmd,
+ error_list=error_list)
+ # get status and set summary
+ level = ERROR
+ if code == 0:
+ status = "success"
+ level = INFO
+ elif code == 1:
+ status = "test failures"
+ else:
+ status = "harness failure"
+
+ # TODO create a better summary for peptest
+ # for now just display return code
+ self.add_summary("%s exited with return code %s: %s" % (cmd[0],
+ code,
+ status))
+
+ def _build_arg(self, option, value):
+ """
+ Build a command line argument
+ """
+ if not value:
+ return []
+ return [str(option), str(value)]
+
+ def _install_from_url(self, url, error_level=ERROR):
+ """
+ Accepts a URL to the application (usually on ftp.m.o)
+ Downloads and installs the application
+ Returns the binary path
+ """
+ dirs = self.query_abs_dirs()
+ # ensure mozinstall is available
+ if len(self.query_package("mozinstall")) == 0:
+ # install mozinstall
+ self.info("Installing mozinstall")
+ pip = self.query_python_path("pip")
+ if not pip:
+ self.log("No application named 'pip' installed",
+ level=error_level)
+ self.run_command(pip + " install mozInstall",
+ cwd=dirs['abs_work_dir'],
+ error_list=PythonErrorList)
+
+ # download the application
+ source = os.path.realpath(self.download_file(url))
+
+ # install the application
+ mozinstall = self.query_python_path("mozinstall")
+ cmd = [mozinstall, '--source', source]
+ cmd.extend(self._build_arg('--destination',
+ self.config.get('application_install_dir')))
+ binary = self.get_output_from_command(cmd)
+
+ # cleanup
+ self.rmtree(source)
+ return binary
+
+ def _is_url(self, path):
+ """
+ Return True if path looks like a URL.
+ """
+ if path is not None:
+ parsed = urlparse.urlparse(path)
+ return parsed.scheme != '' or parsed.netloc != ''
+ return False
+
+
+
+
+if __name__ == '__main__':
+ peptest = PepTest()
+ peptest.run()

0 comments on commit 4e7318c

Please sign in to comment.