Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

bug 650880 - port desktop unit tests to mozharness. r=aki

--HG--
extra : rebase_source : 123e25803cdfbcccab2e39891760b581d2fade1e
  • Loading branch information...
commit 90811981ba274894e95b83ce44b6d905f065d139 1 parent dea0f9b
@lundjordan lundjordan authored
View
4 .hgignore
@@ -10,4 +10,6 @@ test.py
.coverage
coverage
htmlcov
-venv
+venv
+build/
+logs/
View
98 configs/unittests/linux_unittest.py
@@ -0,0 +1,98 @@
+#### OS Specifics ####
+APP_NAME_DIR = "firefox"
+BINARY_PATH = "firefox-bin"
+INSTALLER_PATH = "installer.tar.bz2"
+XPCSHELL_NAME = "xpcshell"
+DISABLE_SCREEN_SAVER = True
+ADJUST_MOUSE_AND_SCREEN = False
+#####
+config = {
+ ### BUILDBOT
+ "buildbot_json_path": "buildprops.json",
+ "exes": {
+ 'python': '/tools/buildbot/bin/python',
+ 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+ },
+ ###
+ "app_name_dir": APP_NAME_DIR,
+ "installer_path": INSTALLER_PATH,
+ "binary_path": APP_NAME_DIR + "/" + BINARY_PATH,
+ "xpcshell_name": XPCSHELL_NAME,
+ "simplejson_url": "http://build.mozilla.org/talos/zips/simplejson-2.2.1.tar.gz",
+ "repos": [{
+ "repo": "http://hg.mozilla.org/build/tools",
+ "revision": "default",
+ "dest": "tools"
+ }],
+ "run_file_names": {
+ "mochitest": "runtests.py",
+ "reftest": "runreftest.py",
+ "xpcshell": "runxpcshelltests.py"
+ },
+ "minimum_tests_zip_dirs": ["bin/*", "certs/*", "modules/*", "mozbase/*"],
+ "specific_tests_zip_dirs": {
+ "mochitest": ["mochitest/*"],
+ "reftest": ["reftest/*", "jsreftest/*"],
+ "xpcshell": ["xpcshell/*"]
+ },
+ "reftest_options": [
+ "--appname=%(binary_path)s", "--utility-path=tests/bin",
+ "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
+ ],
+ "mochitest_options": [
+ "--appname=%(binary_path)s", "--utility-path=tests/bin",
+ "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
+ "--certificate-path=tests/certs", "--autorun", "--close-when-done",
+ "--console-level=INFO"
+ ],
+ "xpcshell_options": [
+ "--symbols-path=%(symbols_path)s"
+ ],
+ #local mochi suites
+ "all_mochitest_suites": {
+ "plain1": ["--total-chunks=5", "--this-chunk=1", "--chunk-by-dir=4"],
+ "plain2": ["--total-chunks=5", "--this-chunk=2", "--chunk-by-dir=4"],
+ "plain3": ["--total-chunks=5", "--this-chunk=3", "--chunk-by-dir=4"],
+ "plain4": ["--total-chunks=5", "--this-chunk=4", "--chunk-by-dir=4"],
+ "plain5": ["--total-chunks=5", "--this-chunk=5", "--chunk-by-dir=4"],
+ "chrome": ["--chrome"],
+ "browser-chrome": ["--browser-chrome"],
+ "a11y": ["--a11y"],
+ "plugins": ['--setpref=dom.ipc.plugins.enabled=false',
+ '--setpref=dom.ipc.plugins.enabled.x86_64=false',
+ '--ipcplugins']
+ },
+ #local reftests suites
+ "all_reftest_suites": {
+ "reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
+ "crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
+ "jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
+ },
+ "all_xpcshell_suites": {
+ "xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
+ "application/" + APP_NAME_DIR + "/" + XPCSHELL_NAME]
+ },
+ "preflight_run_cmd_suites": [
+ # NOTE 'enabled' is only here while we have unconsolidated configs
+ {
+ "name": "disable_screen_saver",
+ "cmd": ["xset", "s", "reset"],
+ "halt_on_failure": False,
+ "architectures": ["32bit", "64bit"],
+ "enabled": DISABLE_SCREEN_SAVER
+ },
+ {
+ "name": "run mouse & screen adjustment script",
+ "cmd": [
+ # when configs are consolidated this python path will only show
+ # for windows.
+ "python", "tools/scripts/support/mouse_and_screen_resolution.py",
+ "--configuration-url",
+ "http://hg.mozilla.org/%(branch)s/raw-file/%(revision)s/" +
+ "testing/machine-configuration.json"],
+ "architectures": ["32bit"],
+ "halt_on_failure": True,
+ "enabled": ADJUST_MOUSE_AND_SCREEN
+ },
+ ],
+}
View
97 configs/unittests/mac_unittest.py
@@ -0,0 +1,97 @@
+#### OS Specifics ####
+APP_NAME_DIR = "FirefoxNightly.app/Contents/MacOS"
+BINARY_PATH = "firefox-bin"
+INSTALLER_PATH = "installer.dmg"
+XPCSHELL_NAME = 'xpcshell'
+DISABLE_SCREEN_SAVER = False
+ADJUST_MOUSE_AND_SCREEN = False
+#####
+config = {
+ ### BUILDBOT
+ "buildbot_json_path": "buildprops.json",
+ "exes": {
+ 'python': '/tools/buildbot/bin/python',
+ 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+ },
+ ###
+ "app_name_dir": APP_NAME_DIR,
+ "installer_path": INSTALLER_PATH,
+ "binary_path": APP_NAME_DIR + "/" + BINARY_PATH,
+ "xpcshell_name": XPCSHELL_NAME,
+ "simplejson_url": "http://build.mozilla.org/talos/zips/simplejson-2.2.1.tar.gz",
+ "repos": [{
+ "repo": "http://hg.mozilla.org/build/tools",
+ "revision": "default",
+ "dest": "tools"
+ }],
+ "run_file_names": {
+ "mochitest": "runtests.py",
+ "reftest": "runreftest.py",
+ "xpcshell": "runxpcshelltests.py"
+ },
+ "minimum_tests_zip_dirs": ["bin/*", "certs/*", "modules/*", "mozbase/*"],
+ "specific_tests_zip_dirs": {
+ "mochitest": ["mochitest/*"],
+ "reftest": ["reftest/*", "jsreftest/*"],
+ "xpcshell": ["xpcshell/*"]
+ },
+ "reftest_options": [
+ "--appname=%(binary_path)s", "--utility-path=tests/bin",
+ "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
+ ],
+ "mochitest_options": [
+ "--appname=%(binary_path)s", "--utility-path=tests/bin",
+ "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
+ "--certificate-path=tests/certs", "--autorun", "--close-when-done",
+ "--console-level=INFO"
+ ],
+ "xpcshell_options": [
+ "--symbols-path=%(symbols_path)s"
+ ],
+ #local mochi suites
+ "all_mochitest_suites": {
+ "plain1": ["--total-chunks=5", "--this-chunk=1", "--chunk-by-dir=4"],
+ "plain2": ["--total-chunks=5", "--this-chunk=2", "--chunk-by-dir=4"],
+ "plain3": ["--total-chunks=5", "--this-chunk=3", "--chunk-by-dir=4"],
+ "plain4": ["--total-chunks=5", "--this-chunk=4", "--chunk-by-dir=4"],
+ "plain5": ["--total-chunks=5", "--this-chunk=5", "--chunk-by-dir=4"],
+ "chrome": ["--chrome"],
+ "browser-chrome": ["--browser-chrome"],
+ "plugins": ['--setpref=dom.ipc.plugins.enabled=false',
+ '--setpref=dom.ipc.plugins.enabled.x86_64=false',
+ '--ipcplugins']
+ },
+ #local reftests suites
+ "all_reftest_suites": {
+ "reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
+ "crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
+ "jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
+ },
+ "all_xpcshell_suites": {
+ "xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
+ "application/" + APP_NAME_DIR + "/" + XPCSHELL_NAME]
+ },
+ "preflight_run_cmd_suites": [
+ # NOTE 'enabled' is only here while we have unconsolidated configs
+ {
+ "name": "disable_screen_saver",
+ "cmd": ["xset", "s", "reset"],
+ "architectures": ["32bit", "64bit"],
+ "halt_on_failure": False,
+ "enabled": DISABLE_SCREEN_SAVER
+ },
+ {
+ "name": "run mouse & screen adjustment script",
+ "cmd": [
+ # when configs are consolidated this python path will only show
+ # for windows.
+ "python", "tools/scripts/support/mouse_and_screen_resolution.py",
+ "--configuration-url",
+ "http://hg.mozilla.org/%(branch)s/raw-file/%(revision)s/" +
+ "testing/machine-configuration.json"],
+ "architectures": ["32bit"],
+ "halt_on_failure": True,
+ "enabled": ADJUST_MOUSE_AND_SCREEN
+ },
+ ],
+}
View
102 configs/unittests/win_unittest.py
@@ -0,0 +1,102 @@
+#### OS Specifics ####
+APP_NAME_DIR = "firefox"
+BINARY_PATH = "firefox.exe"
+INSTALLER_PATH = "installer.zip"
+XPCSHELL_NAME = 'xpcshell.exe'
+DISABLE_SCREEN_SAVER = False
+ADJUST_MOUSE_AND_SCREEN = True
+#####
+config = {
+ ### BUILDBOT
+ "buildbot_json_path": "buildprops.json",
+ "exes": {
+ 'python': 'c:/mozilla-build/python27/python',
+ 'virtualenv': ['c:/mozilla-build/python27/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
+ 'hg': 'c:/mozilla-build/hg/hg',
+ },
+ ###
+ "app_name_dir": APP_NAME_DIR,
+ "installer_path": INSTALLER_PATH,
+ "binary_path": APP_NAME_DIR + "/" + BINARY_PATH,
+ "xpcshell_name": XPCSHELL_NAME,
+ "virtualenv_path": 'c:/talos-slave/test/build/venv',
+ "virtualenv_python_dll": 'c:/mozilla-build/python27/python27.dll',
+ "simplejson_url": "http://build.mozilla.org/talos/zips/simplejson-2.2.1.tar.gz",
+ "repos": [{
+ "repo": "http://hg.mozilla.org/build/tools",
+ "revision": "default",
+ "dest": "tools"
+ }],
+ "run_file_names": {
+ "mochitest": "runtests.py",
+ "reftest": "runreftest.py",
+ "xpcshell": "runxpcshelltests.py"
+ },
+ "minimum_tests_zip_dirs": ["bin/*", "certs/*", "modules/*", "mozbase/*"],
+ "specific_tests_zip_dirs": {
+ "mochitest": ["mochitest/*"],
+ "reftest": ["reftest/*", "jsreftest/*"],
+ "xpcshell": ["xpcshell/*"]
+ },
+ "reftest_options": [
+ "--appname=%(binary_path)s", "--utility-path=tests/bin",
+ "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
+ ],
+ "mochitest_options": [
+ "--appname=%(binary_path)s", "--utility-path=tests/bin",
+ "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
+ "--certificate-path=tests/certs", "--autorun", "--close-when-done",
+ "--console-level=INFO"
+ ],
+ "xpcshell_options": [
+ "--symbols-path=%(symbols_path)s"
+ ],
+ #local mochi suites
+ "all_mochitest_suites":
+ {
+ "plain1": ["--total-chunks=5", "--this-chunk=1", "--chunk-by-dir=4"],
+ "plain2": ["--total-chunks=5", "--this-chunk=2", "--chunk-by-dir=4"],
+ "plain3": ["--total-chunks=5", "--this-chunk=3", "--chunk-by-dir=4"],
+ "plain4": ["--total-chunks=5", "--this-chunk=4", "--chunk-by-dir=4"],
+ "plain5": ["--total-chunks=5", "--this-chunk=5", "--chunk-by-dir=4"],
+ "chrome": ["--chrome"],
+ "browser-chrome": ["--browser-chrome"],
+ "a11y": ["--a11y"],
+ "plugins": ['--setpref=dom.ipc.plugins.enabled=false',
+ '--setpref=dom.ipc.plugins.enabled.x86_64=false',
+ '--ipcplugins']
+ },
+ #local reftests suites
+ "all_reftest_suites": {
+ "reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
+ "crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
+ "jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
+ },
+ "all_xpcshell_suites": {
+ "xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
+ "application/" + APP_NAME_DIR + "/" + XPCSHELL_NAME]
+ },
+ "preflight_run_cmd_suites": [
+ # NOTE 'enabled' is only here while we have unconsolidated configs
+ {
+ "name": "disable_screen_saver",
+ "cmd": ["xset", "s", "reset"],
+ "architectures": ["32bit", "64bit"],
+ "halt_on_failure": False,
+ "enabled": DISABLE_SCREEN_SAVER
+ },
+ {
+ "name": "run mouse & screen adjustment script",
+ "cmd": [
+ # when configs are consolidated this python path will only show
+ # for windows.
+ "C:\\mozilla-build\\python25\\python.exe", "tools/scripts/support/mouse_and_screen_resolution.py",
+ "--configuration-url",
+ "http://hg.mozilla.org/%(branch)s/raw-file/%(revision)s/" +
+ "testing/machine-configuration.json"],
+ "architectures": ["32bit"],
+ "halt_on_failure": True,
+ "enabled": ADJUST_MOUSE_AND_SCREEN
+ },
+ ],
+}
View
50 mozharness/base/log.py
@@ -96,8 +96,6 @@ def critical(self, message):
def fatal(self, message, exit_code=-1):
self.log(message, level=FATAL, exit_code=exit_code)
-
-
# OutputParser {{{1
class OutputParser(LogMixin):
""" Helper object to parse command output.
@@ -118,26 +116,21 @@ class OutputParser(LogMixin):
buffered up to self.num_pre_context_lines (set to the largest
pre-context-line setting in error_list.)
"""
- def __init__(self, config=None, log_obj=None, error_list=None,
- log_output=True):
+ def __init__(self, config=None, log_obj=None, error_list=None, log_output=True):
self.config = config
self.log_obj = log_obj
- self.error_list = error_list
+ self.error_list = error_list or []
self.log_output = log_output
self.num_errors = 0
+ self.num_warnings = 0
# TODO context_lines.
# Not in use yet, but will be based off error_list.
self.context_buffer = []
self.num_pre_context_lines = 0
self.num_post_context_lines = 0
- # TODO set self.error_level to the worst error level hit
- # (WARNING, ERROR, CRITICAL, FATAL)
- # self.error_level = INFO
+ self.worst_log_level = INFO
def parse_single_line(self, line):
- if not line or line.isspace():
- return
- line = line.decode("utf-8").rstrip()
for error_check in self.error_list:
# TODO buffer for context_lines.
match = False
@@ -148,33 +141,50 @@ def parse_single_line(self, line):
if error_check['regex'].search(line):
match = True
else:
- self.warn("error_list: 'substr' and 'regex' not in %s" % \
- error_check)
+ self.warning("error_list: 'substr' and 'regex' not in %s" %
+ error_check)
if match:
- level = error_check.get('level', INFO)
+ log_level = error_check.get('level', INFO)
if self.log_output:
message = ' %s' % line
if error_check.get('explanation'):
message += '\n %s' % error_check['explanation']
if error_check.get('summary'):
- self.add_summary(message, level=level)
+ self.add_summary(message, level=log_level)
else:
- self.log(message, level=level)
- if level in (ERROR, CRITICAL, FATAL):
+ self.log(message, level=log_level)
+ if log_level in (ERROR, CRITICAL, FATAL):
self.num_errors += 1
- # TODO set self.error_status (or something)
- # that sets the worst error level hit.
+ if log_level == WARNING:
+ self.num_warnings += 1
+ self.worst_log_level = self.worst_level(log_level,
+ self.worst_log_level)
break
else:
if self.log_output:
self.info(' %s' % line)
def add_lines(self, output):
- if str(output) == output:
+ if isinstance(output, basestring):
output = [output]
for line in output:
+ if not line or line.isspace():
+ continue
+ line = line.decode("utf-8").rstrip()
self.parse_single_line(line)
+ def worst_level(self, target_level, existing_level, levels=None):
+ """returns either existing_level or target level.
+ This depends on which is closest to levels[0]
+ By default, levels is the list of log levels"""
+ if not levels:
+ levels = [FATAL, CRITICAL, ERROR, WARNING, INFO, DEBUG, IGNORE]
+ if target_level not in levels:
+ self.fatal("'%s' not in %s'." % (target_level, levels))
+ for l in levels:
+ if l in (target_level, existing_level):
+ return l
+
# BaseLogger {{{1
class BaseLogger(object):
View
69 mozharness/base/script.py
@@ -154,7 +154,7 @@ def download_file(self, url, file_name=None, parent_dir=None,
return
except urllib2.URLError, e:
self.log("URL Error: %s" % (url), level=error_level,
- exit_code=exit_code)
+ exit_code=exit_code)
return
return file_name
@@ -186,6 +186,59 @@ def copyfile(self, src, dest, log_level=INFO, error_level=ERROR):
level=error_level)
return -1
+ def copytree(self, src, dest, overwrite='no_overwrite', log_level=INFO,
+ error_level=ERROR):
+ """an implementation of shutil.copytree however it allows for
+ dest to exist and implements differen't overwrite levels.
+ overwrite uses:
+ 'no_overwrite' will keep all(any) existing files in destination tree
+ 'overwrite_if_exists' will only overwrite destination paths that have
+ the same path names relative to the root of the src and
+ destination tree
+ 'clobber' will replace the whole destination tree(clobber) if it exists"""
+
+ self.info('copying tree: %s to %s' % (src, dest))
+ try:
+ if overwrite == 'clobber':
+ self.rmtree(dest)
+ shutil.copytree(src, dest)
+ elif overwrite == 'no_overwrite' or overwrite == 'overwrite_if_exists':
+ files = os.listdir(src)
+ for f in files:
+ abs_src_f = os.path.join(src, f)
+ abs_dest_f = os.path.join(dest, f)
+ if not os.path.exists(abs_dest_f):
+ if os.path.isdir(abs_src_f):
+ self.mkdir_p(abs_dest_f)
+ self.copytree(abs_src_f, abs_dest_f,
+ overwrite='clobber')
+ else:
+ shutil.copy2(abs_src_f, abs_dest_f)
+ elif overwrite == 'no_overwrite': # destination path exists
+ if os.path.isdir(abs_src_f) and os.path.isdir(abs_dest_f):
+ self.copytree(abs_src_f, abs_dest_f,
+ overwrite='no_overwrite')
+ else:
+ self.debug('ignoring path: %s as destination: \
+ %s exists' % (abs_src_f, abs_dest_f))
+ else: # overwrite == 'overwrite_if_exists' and destination exists
+ self.debug('overwriting: %s with: %s' %
+ (abs_dest_f, abs_src_f))
+ self.rmtree(abs_dest_f)
+
+ if os.path.isdir(abs_src_f):
+ self.mkdir_p(abs_dest_f)
+ self.copytree(abs_src_f, abs_dest_f,
+ overwrite='overwrite_if_exists')
+ else:
+ shutil.copy2(abs_src_f, abs_dest_f)
+ else:
+ self.fatal("%s is not a valid argument for param overwrite" % (overwrite))
+ except (IOError, shutil.Error):
+ self.dump_exception("There was an error while copying %s to %s!" % (src, dest),
+ level=error_level)
+ return -1
+
def write_to_file(self, file_path, contents, verbose=True,
open_mode='w', create_parent_dir=False,
error_level=ERROR):
@@ -372,8 +425,6 @@ def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
]
(context_lines isn't written yet)
"""
- if error_list is None:
- error_list = []
if success_codes is None:
success_codes = [0]
if cwd:
@@ -395,8 +446,16 @@ def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
shell = True
if isinstance(command, list):
shell = False
- p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE,
- cwd=cwd, stderr=subprocess.STDOUT, env=env)
+ try:
+ p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE,
+ cwd=cwd, stderr=subprocess.STDOUT, env=env)
+ except OSError, e:
+ level = ERROR
+ if halt_on_failure:
+ level = FATAL
+ self.log('caught OS error %s: %s while running %s' % (e.errno,
+ e.strerror, command), level=level)
+ return -1
if output_parser is None:
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=error_list)
View
48 mozharness/mozilla/testing/errors.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""Mozilla error lists for running tests.
+
+Error lists are used to parse output in mozharness.base.log.OutputParser.
+
+Each line of output is matched against each substring or regular expression
+in the error list. On a match, we determine the 'level' of that line,
+whether IGNORE, DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL.
+
+"""
+
+import re
+from mozharness.base.log import INFO
+
+# ErrorLists {{{1
+TinderBoxPrintRe = {
+ "mochitest_summary": {
+ 'regex': re.compile(r'''(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))'''),
+ 'pass_group': "Passed",
+ 'fail_group': "Failed",
+ 'known_fail_group': "Todo",
+ },
+ "reftest_summary": {
+ 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),
+ 'pass_group': "Successful",
+ 'fail_group': "Unexpected",
+ 'known_fail_group': "Known problems",
+ },
+ "xpcshell_summary": {
+ 'regex': re.compile(r'''INFO \| (Passed|Failed): (\d+)'''),
+ 'pass_group': "Passed",
+ 'fail_group': "Failed",
+ 'known_fail_group': None,
+ },
+ "harness_error": {
+ 'full_regex': re.compile(r"TEST-UNEXPECTED-FAIL \| .* \| (Browser crashed \(minidump found\)|missing output line for total leaks!|negative leaks caught!|leaked \d+ bytes during test execution)"),
+ 'minimum_regex': re.compile(r'''TEST-UNEXPECTED''')
+ },
+}
+
+TestPassed = [
+ {'regex': re.compile('''(TEST-INFO|TEST-KNOWN-FAIL|TEST-PASS|INFO \| )'''), 'level': INFO},
+]
View
15 mozharness/mozilla/testing/testbase.py
@@ -130,15 +130,20 @@ def _download_test_zip(self):
error_level=FATAL)
self.test_zip_path = os.path.realpath(source)
- def _extract_test_zip(self):
+ def _extract_test_zip(self, target_unzip_dirs=None):
dirs = self.query_abs_dirs()
unzip = self.query_exe("unzip")
test_install_dir = dirs.get('abs_test_install_dir',
os.path.join(dirs['abs_work_dir'], 'tests'))
self.mkdir_p(test_install_dir)
+ # adding overwrite flag otherwise subprocess.Popen hangs on waiting for
+ # input in a hidden pipe whenever this action is run twice without
+ # clobber
+ unzip_cmd = [unzip, '-o', self.test_zip_path]
+ if target_unzip_dirs:
+ unzip_cmd.extend(target_unzip_dirs)
# TODO error_list
- self.run_command([unzip, self.test_zip_path],
- cwd=test_install_dir)
+ self.run_command(unzip_cmd, cwd=test_install_dir)
def _download_installer(self):
file_name = None
@@ -153,7 +158,7 @@ def _download_installer(self):
def download_and_extract(self):
"""
- Create virtualenv and install dependencies
+ download and extract test zip / download installer
"""
if self.test_url:
self._download_test_zip()
@@ -192,7 +197,7 @@ def install(self):
dirs = self.query_abs_dirs()
target_dir = dirs.get('abs_app_install_dir',
os.path.join(dirs['abs_work_dir'],
- 'application'))
+ 'application'))
self.mkdir_p(target_dir)
cmd.extend([self.installer_path,
'--destination', target_dir])
View
490 scripts/desktop_unittest.py
@@ -0,0 +1,490 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""desktop_unittest.py
+The goal of this is to extract desktop unittestng from buildbot's factory.py
+
+author: Jordan Lund
+"""
+
+import os
+import sys
+import copy
+import platform
+import shutil
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.errors import BaseErrorList
+from mozharness.mozilla.testing.errors import TinderBoxPrintRe
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.base.log import OutputParser, WARNING, INFO
+from mozharness.mozilla.buildbot import TBPL_WARNING, TBPL_FAILURE
+from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_STATUS_DICT
+
+SUITE_CATEGORIES = ['mochitest', 'reftest', 'xpcshell']
+
+
+class DesktopUnittestOutputParser(OutputParser):
+ """
+ A class that extends OutputParser such that it can parse the number of
+ passed/failed/todo tests from the output.
+ """
+
+ def __init__(self, suite_category, **kwargs):
+ # worst_log_level defined already in DesktopUnittestOutputParser
+ # but is here to make pylint happy
+ self.worst_log_level = INFO
+ super(DesktopUnittestOutputParser, self).__init__(**kwargs)
+ self.summary_suite_re = TinderBoxPrintRe.get('%s_summary' % suite_category, {})
+ self.harness_error_re = TinderBoxPrintRe['harness_error']['minimum_regex']
+ self.full_harness_error_re = TinderBoxPrintRe['harness_error']['full_regex']
+ self.fail_count = -1
+ self.pass_count = -1
+ # known_fail_count does not exist for some suites
+ self.known_fail_count = self.summary_suite_re.get('known_fail_group') and -1
+ self.crashed, self.leaked = False, False
+ self.tbpl_status = TBPL_SUCCESS
+
+ def parse_single_line(self, line):
+ if self.summary_suite_re:
+ summary_m = self.summary_suite_re['regex'].match(line) # pass/fail/todo
+ if summary_m:
+ message = ' %s' % line
+ log_level = INFO
+ # remove all the none values in groups() so this will work
+ # with all suites including mochitest browser-chrome
+ summary_match_list = [group for group in summary_m.groups()
+ if group is not None]
+ r = summary_match_list[0]
+ if self.summary_suite_re['pass_group'] in r:
+ self.pass_count = int(summary_match_list[-1])
+ elif self.summary_suite_re['fail_group'] in r:
+ self.fail_count = int(summary_match_list[-1])
+ if self.fail_count > 0:
+ message += '\n One or more unittests failed.'
+ log_level = WARNING
+ # If self.summary_suite_re['known_fail_group'] == None,
+ # then r should not match it, # so this test is fine as is.
+ elif self.summary_suite_re['known_fail_group'] in r:
+ self.known_fail_count = int(summary_match_list[-1])
+ self.log(message, log_level)
+ return # skip harness check and base parse_single_line
+ harness_match = self.harness_error_re.match(line)
+ if harness_match:
+ self.warning(' %s\n This is a harness error.' % line)
+ self.worst_log_level = self.worst_level(WARNING, self.worst_log_level)
+ self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
+ levels=TBPL_STATUS_DICT.keys())
+ full_harness_match = self.full_harness_error_re.match(line)
+ if full_harness_match:
+ r = harness_match.group(1)
+ if r == "Browser crashed (minidump found)":
+ self.crashed = True
+ elif r == "missing output line for total leaks!":
+ self.leaked = None
+ else:
+ self.leaked = True
+ return # skip base parse_single_line
+ super(DesktopUnittestOutputParser, self).parse_single_line(line)
+
+ def evaluate_parser(self, return_code):
+ if self.num_errors: # mozharness ran into a script error
+ self.tbpl_status = TBPL_FAILURE
+
+ # I have to put this outside of parse_single_line because this checks not
+ # only if fail_count was more then 0 but also if fail_count is still -1
+ # (no fail summary line was found)
+ if self.fail_count != 0:
+ self.worst_log_level = self.worst_level(WARNING, self.worst_log_level)
+ self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
+ levels=TBPL_STATUS_DICT.keys())
+ # we can trust in parser.worst_log_level in either case
+ return (self.tbpl_status, self.worst_log_level)
+
+ def append_tinderboxprint_line(self, suite_name):
+ # We are duplicating a condition (fail_count) from evaluate_parser and
+ # parse parse_single_line but at little cost since we are not parsing
+ # the log more then once. I figured this method should stay isolated as
+ # it is only here for tbpl highlighted summaries and is not part of
+ # buildbot evaluation or result status IIUC.
+ emphasize_fail_text = '<em class="testfail">%s</em>'
+
+ if self.pass_count < 0 or self.fail_count < 0 or self.known_fail_count < 0:
+ summary = emphasize_fail_text % 'T-FAIL'
+ elif self.pass_count == 0 and self.fail_count == 0 and \
+ (self.known_fail_count == 0 or self.known_fail_count is None):
+ summary = emphasize_fail_text % 'T-FAIL'
+ else:
+ str_fail_count = str(self.fail_count)
+ if self.fail_count > 0:
+ str_fail_count = emphasize_fail_text % str_fail_count
+ summary = "%d/%s" % (self.pass_count, str_fail_count)
+ if self.known_fail_count is not None:
+ summary += "/%d" % self.known_fail_count
+ # Format the crash status.
+ if self.crashed:
+ summary += "&nbsp;%s" % emphasize_fail_text % "CRASH"
+ # Format the leak status.
+ if self.leaked is not False:
+ summary += "&nbsp;%s" % emphasize_fail_text % (
+ (self.leaked and "LEAK") or "L-FAIL")
+ # Return the summary.
+ self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, summary))
+
+
+# DesktopUnittest {{{1
+class DesktopUnittest(TestingMixin, MercurialScript):
+
+ config_options = [
+ [['--mochitest-suite', ], {
+ "action": "append",
+ "dest": "specified_mochitest_suites",
+ "type": "string",
+ "help": "Specify which mochi suite to run."
+ "Suites are defined in the config file.\n"
+ "Examples: 'all', 'plain1', 'plain5', 'chrome', or 'a11y'"}
+ ],
+ [['--reftest-suite', ], {
+ "action": "append",
+ "dest": "specified_reftest_suites",
+ "type": "string",
+ "help": "Specify which reftest suite to run."
+ "Suites are defined in the config file.\n"
+ "Examples: 'all', 'crashplan', or 'jsreftest'"}
+ ],
+ [['--xpcshell-suite', ], {
+ "action": "append",
+ "dest": "specified_xpcshell_suites",
+ "type": "string",
+ "help": "Specify which xpcshell suite to run."
+ "Suites are defined in the config file\n."
+ "Examples: 'xpcshell'"}
+ ],
+ [['--run-all-suites', ], {
+ "action": "store_true",
+ "dest": "run_all_suites",
+ "default": False,
+ "help": "This will run all suites that are specified"
+ "in the config file. You do not need to specify "
+ "any other suites.\nBeware, this may take a while ;)"}
+ ],
+ [['--enable-preflight-run-commands', ], {
+ "action": "store_false",
+ "dest": "preflight_run_commands_disabled",
+ "default": True,
+ "help": "This will enable any run commands that are specified"
+ "in the config file under: preflight_run_cmd_suites"}
+ ]
+ ] + copy.deepcopy(testing_config_options)
+
+ virtualenv_modules = [
+ "simplejson",
+ {'mozlog': os.path.join('tests', 'mozbase', 'mozlog')},
+ {'mozinfo': os.path.join('tests', 'mozbase', 'mozinfo')},
+ {'mozhttpd': os.path.join('tests', 'mozbase', 'mozhttpd')},
+ {'mozinstall': os.path.join('tests', 'mozbase', 'mozinstall')},
+ {'manifestdestiny': os.path.join('tests', 'mozbase', 'manifestdestiny')},
+ {'mozprofile': os.path.join('tests', 'mozbase', 'mozprofile')},
+ {'mozprocess': os.path.join('tests', 'mozbase', 'mozprocess')},
+ {'mozrunner': os.path.join('tests', 'mozbase', 'mozrunner')},
+ ]
+
+ def __init__(self, require_config_file=True):
+ # abs_dirs defined already in BaseScript but is here to make pylint happy
+ self.abs_dirs = None
+ MercurialScript.__init__(
+ self,
+ config_options=self.config_options,
+ all_actions=[
+ 'clobber',
+ 'read-buildbot-config',
+ 'download-and-extract',
+ 'pull',
+ 'create-virtualenv',
+ 'install',
+ 'run-tests',
+ ],
+ require_config_file=require_config_file,
+ config={'virtualenv_modules': self.virtualenv_modules,
+ 'require_test_zip': True})
+
+ c = self.config
+ self.global_test_options = []
+ self.installer_url = c.get('installer_url')
+ self.test_url = c.get('test_url')
+ self.symbols_url = c.get('symbols_url')
+ # this is so mozinstall in install() doesn't bug out if we don't run the
+ # download_and_extract action
+ self.installer_path = os.path.join(self.abs_dirs['abs_work_dir'],
+ c.get('installer_path'))
+ self.binary_path = os.path.join(self.abs_dirs['abs_app_install_dir'],
+ c.get('binary_path'))
+
+ ###### helper methods
+ def _pre_config_lock(self, rw_config):
+ c = self.config
+ if not c.get('run_all_suites'):
+ return # configs are valid
+ for category in SUITE_CATEGORIES:
+ specific_suites = c.get('specified_%s_suites' % (category))
+ if specific_suites:
+ if specific_suites != 'all':
+ self.fatal("Config options are not valid. Please ensure"
+ " that if the '--run-all-suites' flag was enabled,"
+ " then do not specify to run only specific suites "
+ "like:\n '--mochitest-suite browser-chrome'")
+
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = super(DesktopUnittest, self).query_abs_dirs()
+
+ c = self.config
+ dirs = {}
+ dirs['abs_app_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'application')
+ dirs['abs_app_dir'] = os.path.join(dirs['abs_app_install_dir'], c['app_name_dir'])
+ dirs['abs_app_plugins_dir'] = os.path.join(dirs['abs_app_dir'], 'plugins')
+ dirs['abs_app_components_dir'] = os.path.join(dirs['abs_app_dir'], 'components')
+ dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
+ dirs['abs_test_bin_dir'] = os.path.join(dirs['abs_test_install_dir'], 'bin')
+ dirs['abs_test_bin_plugins_dir'] = os.path.join(dirs['abs_test_bin_dir'],
+ 'plugins')
+ dirs['abs_test_bin_components_dir'] = os.path.join(dirs['abs_test_bin_dir'],
+ 'components')
+ dirs['abs_mochitest_dir'] = os.path.join(dirs['abs_test_install_dir'], "mochitest")
+ dirs['abs_reftest_dir'] = os.path.join(dirs['abs_test_install_dir'], "reftest")
+ dirs['abs_xpcshell_dir'] = os.path.join(dirs['abs_test_install_dir'], "xpcshell")
+
+ if os.path.isabs(c['virtualenv_path']):
+ dirs['abs_virtualenv_dir'] = c['virtualenv_path']
+ else:
+ dirs['abs_virtualenv_dir'] = os.path.join(abs_dirs['abs_work_dir'],
+ c['virtualenv_path'])
+ abs_dirs.update(dirs)
+ self.abs_dirs = abs_dirs
+
+ return self.abs_dirs
+
+ def _query_symbols_url(self):
+ """query the full symbols URL based upon binary URL"""
+ # may break with name convention changes but is one less 'input' for script
+ if self.symbols_url:
+ return self.symbols_url
+
+ symbols_url = None
+ self.info("finding symbols_url based upon self.installer_url")
+ if self.installer_url:
+ for ext in ['.zip', '.dmg', '.tar.bz2']:
+ if ext in self.installer_url:
+ symbols_url = self.installer_url.replace(
+ ext, '.crashreporter-symbols.zip')
+ if not symbols_url:
+ self.fatal("self.installer_url was found but symbols_url could \
+ not be determined")
+ else:
+ self.fatal("self.installer_url was not found in self.config")
+ self.info("setting symbols_url as %s" % (symbols_url))
+ self.symbols_url = symbols_url
+ return self.symbols_url
+
+ def _query_abs_base_cmd(self, suite_category):
+ if self.binary_path:
+ c = self.config
+ dirs = self.query_abs_dirs()
+ options = []
+ run_file = c['run_file_names'][suite_category]
+ base_cmd = [self.query_python_path('python'), '-u']
+ base_cmd.append(dirs["abs_%s_dir" % suite_category] + "/" + run_file)
+ str_format_values = {
+ 'binary_path': self.binary_path,
+ 'symbols_path': self._query_symbols_url()
+ }
+ if self.config['%s_options' % suite_category]:
+ for option in self.config['%s_options' % suite_category]:
+ options.append(option % str_format_values)
+ abs_base_cmd = base_cmd + options
+ return abs_base_cmd
+ else:
+ self.warning("Suite options for %s could not be determined."
+ "\nIf you meant to have options for this suite, "
+ "please make sure they are specified in your "
+ "config under %s_options" %
+ (suite_category, suite_category))
+ else:
+ self.fatal("'binary_path' could not be determined.\n This should"
+ "be like '/path/build/application/firefox/firefox'"
+ "\nIf you are running this script without the 'install' "
+ "action (where binary_path is set), please ensure you are"
+ " either:\n(1) specifying it in the config file under "
+ "binary_path\n(2) specifying it on command line with the"
+ " '--binary-path' flag")
+
+ def _query_specified_suites(self, category):
+ # logic goes: if at least one '--{category}-suite' was given,
+ # then run only that(those) given suite(s). Elif no suites were
+ # specified and the --run-all-suites flag was given,
+ # run all {category} suites. Anything else, run no suites.
+ c = self.config
+ all_suites = c.get('all_%s_suites' % (category))
+ specified_suites = c.get('specified_%s_suites' % (category)) # list
+ suites = None
+
+ if specified_suites:
+ if 'all' in specified_suites:
+ # useful if you want a quick way of saying run all suites
+ # of a specific category.
+ suites = all_suites
+ else:
+ # suites gets a dict of everything from all_suites where a key
+ # is also in specified_suites
+ suites = dict((key, all_suites.get(key)) for key in
+ specified_suites if key in all_suites.keys())
+ else:
+ if c.get('run_all_suites'): # needed if you dont specify any suites
+ suites = all_suites
+
+ return suites
+
+ # Actions {{{2
+
+ # clobber defined in BaseScript, deletes mozharness/build if exists
+ # read_buildbot_config is in BuildbotMixin.
+ # postflight_read_buildbot_config is in TestingMixin.
+ # preflight_download_and_extract is in TestingMixin.
+ # create_virtualenv is in VirtualenvMixin.
+ # preflight_install is in TestingMixin.
+ # install is in TestingMixin.
+
+ def download_and_extract(self):
+ """
+ download and extract test zip / download installer
+ optimizes which subfolders to extract from tests zip
+ """
+ c = self.config
+ unzip_tests_dirs = None
+
+ if c['specific_tests_zip_dirs']:
+ unzip_tests_dirs = c['minimum_tests_zip_dirs']
+ for category in c['specific_tests_zip_dirs'].keys():
+ if c['run_all_suites'] or self._query_specified_suites(category) \
+ or 'run-tests' not in self.actions:
+ unzip_tests_dirs.extend(c['specific_tests_zip_dirs'][category])
+ if self.test_url:
+ self._download_test_zip()
+ self._extract_test_zip(target_unzip_dirs=unzip_tests_dirs)
+ self._download_installer()
+
+ def pull(self):
+ dirs = self.query_abs_dirs()
+ c = self.config
+ if c.get('repos'):
+ dirs = self.query_abs_dirs()
+ self.vcs_checkout_repos(c['repos'],
+ parent_dir=dirs['abs_work_dir'])
+
+ def preflight_run_tests(self):
+ """preflight commands for all tests"""
+ c = self.config
+ dirs = self.query_abs_dirs()
+
+ if not c.get('preflight_run_commands_disabled'):
+ arch = platform.architecture()[0]
+ for suite in c['preflight_run_cmd_suites']:
+ # XXX platform.architecture() may give incorrect values for some
+ # platforms like mac as excutable files may be universal
+ # files containing multiple architectures
+ # NOTE 'enabled' is only here while we have unconsolidated configs
+ if suite['enabled'] and arch in suite['architectures']:
+ cmd = suite['cmd']
+ name = suite['name']
+ self.info("Running pre test command %(name)s with '%(cmd)s'"
+ % {'name': name, 'cmd': ' '.join(cmd)})
+ if self.buildbot_config: # this cmd is for buildbot
+ # TODO rather then checking for formatting on every string
+ # in every preflight enabled cmd: find a better solution!
+ # maybe I can implement WithProperties in mozharness?
+ cmd = [x % (self.buildbot_config.get('properties'))
+ for x in cmd]
+ self.run_command(cmd,
+ cwd=dirs['abs_work_dir'],
+ error_list=BaseErrorList,
+ halt_on_failure=suite['halt_on_failure'])
+ else:
+ self.warning("Proceeding without running prerun test commands."
+ " These are often OS specific and disabling them may"
+ " result in spurious test results!")
+
+ def run_tests(self):
+ self._run_category_suites('mochitest')
+ self._run_category_suites('reftest')
+ self._run_category_suites('xpcshell',
+ preflight_run_method=self.preflight_xpcshell)
+
+ def preflight_xpcshell(self, suites):
+ c = self.config
+ dirs = self.query_abs_dirs()
+ if suites: # there are xpcshell suites to run
+ self.mkdir_p(dirs['abs_app_plugins_dir'])
+ self.info('copying %s to %s' % (os.path.join(dirs['abs_test_bin_dir'],
+ c['xpcshell_name']), os.path.join(dirs['abs_app_dir'],
+ c['xpcshell_name'])))
+ shutil.copy2(os.path.join(dirs['abs_test_bin_dir'], c['xpcshell_name']),
+ os.path.join(dirs['abs_app_dir'], c['xpcshell_name']))
+ self.copytree(dirs['abs_test_bin_components_dir'],
+ dirs['abs_app_components_dir'],
+ overwrite='overwrite_if_exists')
+ self.copytree(dirs['abs_test_bin_plugins_dir'],
+ dirs['abs_app_plugins_dir'],
+ overwrite='overwrite_if_exists')
+
+ def _run_category_suites(self, suite_category, preflight_run_method=None):
+ """run suite(s) to a specific category"""
+ dirs = self.query_abs_dirs()
+ abs_base_cmd = self._query_abs_base_cmd(suite_category)
+ suites = self._query_specified_suites(suite_category)
+
+ if preflight_run_method:
+ preflight_run_method(suites)
+ if suites:
+ self.info('#### Running %s suites' % suite_category)
+ for suite in suites:
+ cmd = abs_base_cmd + suites[suite]
+ suite_name = suite_category + '-' + suite
+ tbpl_status, log_level = None, None
+ parser = DesktopUnittestOutputParser(suite_category,
+ config=self.config,
+ error_list=BaseErrorList,
+ log_obj=self.log_obj)
+
+ return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'],
+ output_parser=parser)
+
+ # mochitests, reftests, and xpcshell suites do not return
+ # appropriate return codes. Therefore, we must parse the output
+ # to determine what the tbpl_status and worst_log_level must
+ # be. We do this by:
+ # 1) checking to see if our mozharness script ran into any
+ # errors itself with 'num_errors' <- OutputParser
+ # 2) if num_errors is 0 then we look in the subclassed 'parser'
+ # findings for harness/suite errors <- DesktopUnittestOutputParser
+ tbpl_status, log_level = parser.evaluate_parser(return_code)
+ parser.append_tinderboxprint_line(suite_name)
+
+ self.buildbot_status(tbpl_status, level=log_level)
+ self.add_summary("The %s suite: %s ran with return status: %s" %
+ (suite_category, suite, tbpl_status),
+ level=log_level)
+ else:
+ self.debug('There were no suites to run for %s' % suite_category)
+
+
+# main {{{1
+if __name__ == '__main__':
+ desktop_unittest = DesktopUnittest()
+ desktop_unittest.run()
Please sign in to comment.
Something went wrong with that request. Please try again.