Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

bug 650880 - port desktop unit tests to mozharness. r=aki

--HG--
extra : rebase_source : 123e258
  • Loading branch information...
commit 90811981ba274894e95b83ce44b6d905f065d139 1 parent dea0f9b
Jordan Lund authored
4 .hgignore
@@ -10,4 +10,6 @@ test.py
10 10 .coverage
11 11 coverage
12 12 htmlcov
13   -venv
  13 +venv
  14 +build/
  15 +logs/
98 configs/unittests/linux_unittest.py
... ... @@ -0,0 +1,98 @@
  1 +#### OS Specifics ####
  2 +APP_NAME_DIR = "firefox"
  3 +BINARY_PATH = "firefox-bin"
  4 +INSTALLER_PATH = "installer.tar.bz2"
  5 +XPCSHELL_NAME = "xpcshell"
  6 +DISABLE_SCREEN_SAVER = True
  7 +ADJUST_MOUSE_AND_SCREEN = False
  8 +#####
  9 +config = {
  10 + ### BUILDBOT
  11 + "buildbot_json_path": "buildprops.json",
  12 + "exes": {
  13 + 'python': '/tools/buildbot/bin/python',
  14 + 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
  15 + },
  16 + ###
  17 + "app_name_dir": APP_NAME_DIR,
  18 + "installer_path": INSTALLER_PATH,
  19 + "binary_path": APP_NAME_DIR + "/" + BINARY_PATH,
  20 + "xpcshell_name": XPCSHELL_NAME,
  21 + "simplejson_url": "http://build.mozilla.org/talos/zips/simplejson-2.2.1.tar.gz",
  22 + "repos": [{
  23 + "repo": "http://hg.mozilla.org/build/tools",
  24 + "revision": "default",
  25 + "dest": "tools"
  26 + }],
  27 + "run_file_names": {
  28 + "mochitest": "runtests.py",
  29 + "reftest": "runreftest.py",
  30 + "xpcshell": "runxpcshelltests.py"
  31 + },
  32 + "minimum_tests_zip_dirs": ["bin/*", "certs/*", "modules/*", "mozbase/*"],
  33 + "specific_tests_zip_dirs": {
  34 + "mochitest": ["mochitest/*"],
  35 + "reftest": ["reftest/*", "jsreftest/*"],
  36 + "xpcshell": ["xpcshell/*"]
  37 + },
  38 + "reftest_options": [
  39 + "--appname=%(binary_path)s", "--utility-path=tests/bin",
  40 + "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
  41 + ],
  42 + "mochitest_options": [
  43 + "--appname=%(binary_path)s", "--utility-path=tests/bin",
  44 + "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
  45 + "--certificate-path=tests/certs", "--autorun", "--close-when-done",
  46 + "--console-level=INFO"
  47 + ],
  48 + "xpcshell_options": [
  49 + "--symbols-path=%(symbols_path)s"
  50 + ],
  51 + #local mochi suites
  52 + "all_mochitest_suites": {
  53 + "plain1": ["--total-chunks=5", "--this-chunk=1", "--chunk-by-dir=4"],
  54 + "plain2": ["--total-chunks=5", "--this-chunk=2", "--chunk-by-dir=4"],
  55 + "plain3": ["--total-chunks=5", "--this-chunk=3", "--chunk-by-dir=4"],
  56 + "plain4": ["--total-chunks=5", "--this-chunk=4", "--chunk-by-dir=4"],
  57 + "plain5": ["--total-chunks=5", "--this-chunk=5", "--chunk-by-dir=4"],
  58 + "chrome": ["--chrome"],
  59 + "browser-chrome": ["--browser-chrome"],
  60 + "a11y": ["--a11y"],
  61 + "plugins": ['--setpref=dom.ipc.plugins.enabled=false',
  62 + '--setpref=dom.ipc.plugins.enabled.x86_64=false',
  63 + '--ipcplugins']
  64 + },
  65 + #local reftests suites
  66 + "all_reftest_suites": {
  67 + "reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
  68 + "crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
  69 + "jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
  70 + },
  71 + "all_xpcshell_suites": {
  72 + "xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
  73 + "application/" + APP_NAME_DIR + "/" + XPCSHELL_NAME]
  74 + },
  75 + "preflight_run_cmd_suites": [
  76 + # NOTE 'enabled' is only here while we have unconsolidated configs
  77 + {
  78 + "name": "disable_screen_saver",
  79 + "cmd": ["xset", "s", "reset"],
  80 + "halt_on_failure": False,
  81 + "architectures": ["32bit", "64bit"],
  82 + "enabled": DISABLE_SCREEN_SAVER
  83 + },
  84 + {
  85 + "name": "run mouse & screen adjustment script",
  86 + "cmd": [
  87 + # when configs are consolidated this python path will only show
  88 + # for windows.
  89 + "python", "tools/scripts/support/mouse_and_screen_resolution.py",
  90 + "--configuration-url",
  91 + "http://hg.mozilla.org/%(branch)s/raw-file/%(revision)s/" +
  92 + "testing/machine-configuration.json"],
  93 + "architectures": ["32bit"],
  94 + "halt_on_failure": True,
  95 + "enabled": ADJUST_MOUSE_AND_SCREEN
  96 + },
  97 + ],
  98 +}
97 configs/unittests/mac_unittest.py
... ... @@ -0,0 +1,97 @@
  1 +#### OS Specifics ####
  2 +APP_NAME_DIR = "FirefoxNightly.app/Contents/MacOS"
  3 +BINARY_PATH = "firefox-bin"
  4 +INSTALLER_PATH = "installer.dmg"
  5 +XPCSHELL_NAME = 'xpcshell'
  6 +DISABLE_SCREEN_SAVER = False
  7 +ADJUST_MOUSE_AND_SCREEN = False
  8 +#####
  9 +config = {
  10 + ### BUILDBOT
  11 + "buildbot_json_path": "buildprops.json",
  12 + "exes": {
  13 + 'python': '/tools/buildbot/bin/python',
  14 + 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
  15 + },
  16 + ###
  17 + "app_name_dir": APP_NAME_DIR,
  18 + "installer_path": INSTALLER_PATH,
  19 + "binary_path": APP_NAME_DIR + "/" + BINARY_PATH,
  20 + "xpcshell_name": XPCSHELL_NAME,
  21 + "simplejson_url": "http://build.mozilla.org/talos/zips/simplejson-2.2.1.tar.gz",
  22 + "repos": [{
  23 + "repo": "http://hg.mozilla.org/build/tools",
  24 + "revision": "default",
  25 + "dest": "tools"
  26 + }],
  27 + "run_file_names": {
  28 + "mochitest": "runtests.py",
  29 + "reftest": "runreftest.py",
  30 + "xpcshell": "runxpcshelltests.py"
  31 + },
  32 + "minimum_tests_zip_dirs": ["bin/*", "certs/*", "modules/*", "mozbase/*"],
  33 + "specific_tests_zip_dirs": {
  34 + "mochitest": ["mochitest/*"],
  35 + "reftest": ["reftest/*", "jsreftest/*"],
  36 + "xpcshell": ["xpcshell/*"]
  37 + },
  38 + "reftest_options": [
  39 + "--appname=%(binary_path)s", "--utility-path=tests/bin",
  40 + "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
  41 + ],
  42 + "mochitest_options": [
  43 + "--appname=%(binary_path)s", "--utility-path=tests/bin",
  44 + "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
  45 + "--certificate-path=tests/certs", "--autorun", "--close-when-done",
  46 + "--console-level=INFO"
  47 + ],
  48 + "xpcshell_options": [
  49 + "--symbols-path=%(symbols_path)s"
  50 + ],
  51 + #local mochi suites
  52 + "all_mochitest_suites": {
  53 + "plain1": ["--total-chunks=5", "--this-chunk=1", "--chunk-by-dir=4"],
  54 + "plain2": ["--total-chunks=5", "--this-chunk=2", "--chunk-by-dir=4"],
  55 + "plain3": ["--total-chunks=5", "--this-chunk=3", "--chunk-by-dir=4"],
  56 + "plain4": ["--total-chunks=5", "--this-chunk=4", "--chunk-by-dir=4"],
  57 + "plain5": ["--total-chunks=5", "--this-chunk=5", "--chunk-by-dir=4"],
  58 + "chrome": ["--chrome"],
  59 + "browser-chrome": ["--browser-chrome"],
  60 + "plugins": ['--setpref=dom.ipc.plugins.enabled=false',
  61 + '--setpref=dom.ipc.plugins.enabled.x86_64=false',
  62 + '--ipcplugins']
  63 + },
  64 + #local reftests suites
  65 + "all_reftest_suites": {
  66 + "reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
  67 + "crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
  68 + "jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
  69 + },
  70 + "all_xpcshell_suites": {
  71 + "xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
  72 + "application/" + APP_NAME_DIR + "/" + XPCSHELL_NAME]
  73 + },
  74 + "preflight_run_cmd_suites": [
  75 + # NOTE 'enabled' is only here while we have unconsolidated configs
  76 + {
  77 + "name": "disable_screen_saver",
  78 + "cmd": ["xset", "s", "reset"],
  79 + "architectures": ["32bit", "64bit"],
  80 + "halt_on_failure": False,
  81 + "enabled": DISABLE_SCREEN_SAVER
  82 + },
  83 + {
  84 + "name": "run mouse & screen adjustment script",
  85 + "cmd": [
  86 + # when configs are consolidated this python path will only show
  87 + # for windows.
  88 + "python", "tools/scripts/support/mouse_and_screen_resolution.py",
  89 + "--configuration-url",
  90 + "http://hg.mozilla.org/%(branch)s/raw-file/%(revision)s/" +
  91 + "testing/machine-configuration.json"],
  92 + "architectures": ["32bit"],
  93 + "halt_on_failure": True,
  94 + "enabled": ADJUST_MOUSE_AND_SCREEN
  95 + },
  96 + ],
  97 +}
102 configs/unittests/win_unittest.py
... ... @@ -0,0 +1,102 @@
  1 +#### OS Specifics ####
  2 +APP_NAME_DIR = "firefox"
  3 +BINARY_PATH = "firefox.exe"
  4 +INSTALLER_PATH = "installer.zip"
  5 +XPCSHELL_NAME = 'xpcshell.exe'
  6 +DISABLE_SCREEN_SAVER = False
  7 +ADJUST_MOUSE_AND_SCREEN = True
  8 +#####
  9 +config = {
  10 + ### BUILDBOT
  11 + "buildbot_json_path": "buildprops.json",
  12 + "exes": {
  13 + 'python': 'c:/mozilla-build/python27/python',
  14 + 'virtualenv': ['c:/mozilla-build/python27/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
  15 + 'hg': 'c:/mozilla-build/hg/hg',
  16 + },
  17 + ###
  18 + "app_name_dir": APP_NAME_DIR,
  19 + "installer_path": INSTALLER_PATH,
  20 + "binary_path": APP_NAME_DIR + "/" + BINARY_PATH,
  21 + "xpcshell_name": XPCSHELL_NAME,
  22 + "virtualenv_path": 'c:/talos-slave/test/build/venv',
  23 + "virtualenv_python_dll": 'c:/mozilla-build/python27/python27.dll',
  24 + "simplejson_url": "http://build.mozilla.org/talos/zips/simplejson-2.2.1.tar.gz",
  25 + "repos": [{
  26 + "repo": "http://hg.mozilla.org/build/tools",
  27 + "revision": "default",
  28 + "dest": "tools"
  29 + }],
  30 + "run_file_names": {
  31 + "mochitest": "runtests.py",
  32 + "reftest": "runreftest.py",
  33 + "xpcshell": "runxpcshelltests.py"
  34 + },
  35 + "minimum_tests_zip_dirs": ["bin/*", "certs/*", "modules/*", "mozbase/*"],
  36 + "specific_tests_zip_dirs": {
  37 + "mochitest": ["mochitest/*"],
  38 + "reftest": ["reftest/*", "jsreftest/*"],
  39 + "xpcshell": ["xpcshell/*"]
  40 + },
  41 + "reftest_options": [
  42 + "--appname=%(binary_path)s", "--utility-path=tests/bin",
  43 + "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
  44 + ],
  45 + "mochitest_options": [
  46 + "--appname=%(binary_path)s", "--utility-path=tests/bin",
  47 + "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
  48 + "--certificate-path=tests/certs", "--autorun", "--close-when-done",
  49 + "--console-level=INFO"
  50 + ],
  51 + "xpcshell_options": [
  52 + "--symbols-path=%(symbols_path)s"
  53 + ],
  54 + #local mochi suites
  55 + "all_mochitest_suites":
  56 + {
  57 + "plain1": ["--total-chunks=5", "--this-chunk=1", "--chunk-by-dir=4"],
  58 + "plain2": ["--total-chunks=5", "--this-chunk=2", "--chunk-by-dir=4"],
  59 + "plain3": ["--total-chunks=5", "--this-chunk=3", "--chunk-by-dir=4"],
  60 + "plain4": ["--total-chunks=5", "--this-chunk=4", "--chunk-by-dir=4"],
  61 + "plain5": ["--total-chunks=5", "--this-chunk=5", "--chunk-by-dir=4"],
  62 + "chrome": ["--chrome"],
  63 + "browser-chrome": ["--browser-chrome"],
  64 + "a11y": ["--a11y"],
  65 + "plugins": ['--setpref=dom.ipc.plugins.enabled=false',
  66 + '--setpref=dom.ipc.plugins.enabled.x86_64=false',
  67 + '--ipcplugins']
  68 + },
  69 + #local reftests suites
  70 + "all_reftest_suites": {
  71 + "reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
  72 + "crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
  73 + "jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
  74 + },
  75 + "all_xpcshell_suites": {
  76 + "xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
  77 + "application/" + APP_NAME_DIR + "/" + XPCSHELL_NAME]
  78 + },
  79 + "preflight_run_cmd_suites": [
  80 + # NOTE 'enabled' is only here while we have unconsolidated configs
  81 + {
  82 + "name": "disable_screen_saver",
  83 + "cmd": ["xset", "s", "reset"],
  84 + "architectures": ["32bit", "64bit"],
  85 + "halt_on_failure": False,
  86 + "enabled": DISABLE_SCREEN_SAVER
  87 + },
  88 + {
  89 + "name": "run mouse & screen adjustment script",
  90 + "cmd": [
  91 + # when configs are consolidated this python path will only show
  92 + # for windows.
  93 + "C:\\mozilla-build\\python25\\python.exe", "tools/scripts/support/mouse_and_screen_resolution.py",
  94 + "--configuration-url",
  95 + "http://hg.mozilla.org/%(branch)s/raw-file/%(revision)s/" +
  96 + "testing/machine-configuration.json"],
  97 + "architectures": ["32bit"],
  98 + "halt_on_failure": True,
  99 + "enabled": ADJUST_MOUSE_AND_SCREEN
  100 + },
  101 + ],
  102 +}
50 mozharness/base/log.py
@@ -96,8 +96,6 @@ def critical(self, message):
96 96 def fatal(self, message, exit_code=-1):
97 97 self.log(message, level=FATAL, exit_code=exit_code)
98 98
99   -
100   -
101 99 # OutputParser {{{1
102 100 class OutputParser(LogMixin):
103 101 """ Helper object to parse command output.
@@ -118,26 +116,21 @@ class OutputParser(LogMixin):
118 116 buffered up to self.num_pre_context_lines (set to the largest
119 117 pre-context-line setting in error_list.)
120 118 """
121   - def __init__(self, config=None, log_obj=None, error_list=None,
122   - log_output=True):
  119 + def __init__(self, config=None, log_obj=None, error_list=None, log_output=True):
123 120 self.config = config
124 121 self.log_obj = log_obj
125   - self.error_list = error_list
  122 + self.error_list = error_list or []
126 123 self.log_output = log_output
127 124 self.num_errors = 0
  125 + self.num_warnings = 0
128 126 # TODO context_lines.
129 127 # Not in use yet, but will be based off error_list.
130 128 self.context_buffer = []
131 129 self.num_pre_context_lines = 0
132 130 self.num_post_context_lines = 0
133   - # TODO set self.error_level to the worst error level hit
134   - # (WARNING, ERROR, CRITICAL, FATAL)
135   - # self.error_level = INFO
  131 + self.worst_log_level = INFO
136 132
137 133 def parse_single_line(self, line):
138   - if not line or line.isspace():
139   - return
140   - line = line.decode("utf-8").rstrip()
141 134 for error_check in self.error_list:
142 135 # TODO buffer for context_lines.
143 136 match = False
@@ -148,33 +141,50 @@ def parse_single_line(self, line):
148 141 if error_check['regex'].search(line):
149 142 match = True
150 143 else:
151   - self.warn("error_list: 'substr' and 'regex' not in %s" % \
152   - error_check)
  144 + self.warning("error_list: 'substr' and 'regex' not in %s" %
  145 + error_check)
153 146 if match:
154   - level = error_check.get('level', INFO)
  147 + log_level = error_check.get('level', INFO)
155 148 if self.log_output:
156 149 message = ' %s' % line
157 150 if error_check.get('explanation'):
158 151 message += '\n %s' % error_check['explanation']
159 152 if error_check.get('summary'):
160   - self.add_summary(message, level=level)
  153 + self.add_summary(message, level=log_level)
161 154 else:
162   - self.log(message, level=level)
163   - if level in (ERROR, CRITICAL, FATAL):
  155 + self.log(message, level=log_level)
  156 + if log_level in (ERROR, CRITICAL, FATAL):
164 157 self.num_errors += 1
165   - # TODO set self.error_status (or something)
166   - # that sets the worst error level hit.
  158 + if log_level == WARNING:
  159 + self.num_warnings += 1
  160 + self.worst_log_level = self.worst_level(log_level,
  161 + self.worst_log_level)
167 162 break
168 163 else:
169 164 if self.log_output:
170 165 self.info(' %s' % line)
171 166
172 167 def add_lines(self, output):
173   - if str(output) == output:
  168 + if isinstance(output, basestring):
174 169 output = [output]
175 170 for line in output:
  171 + if not line or line.isspace():
  172 + continue
  173 + line = line.decode("utf-8").rstrip()
176 174 self.parse_single_line(line)
177 175
  176 + def worst_level(self, target_level, existing_level, levels=None):
  177 + """returns either existing_level or target level.
  178 + This depends on which is closest to levels[0]
  179 + By default, levels is the list of log levels"""
  180 + if not levels:
  181 + levels = [FATAL, CRITICAL, ERROR, WARNING, INFO, DEBUG, IGNORE]
  182 + if target_level not in levels:
  183 + self.fatal("'%s' not in %s'." % (target_level, levels))
  184 + for l in levels:
  185 + if l in (target_level, existing_level):
  186 + return l
  187 +
178 188
179 189 # BaseLogger {{{1
180 190 class BaseLogger(object):
69 mozharness/base/script.py
@@ -154,7 +154,7 @@ def download_file(self, url, file_name=None, parent_dir=None,
154 154 return
155 155 except urllib2.URLError, e:
156 156 self.log("URL Error: %s" % (url), level=error_level,
157   - exit_code=exit_code)
  157 + exit_code=exit_code)
158 158 return
159 159 return file_name
160 160
@@ -186,6 +186,59 @@ def copyfile(self, src, dest, log_level=INFO, error_level=ERROR):
186 186 level=error_level)
187 187 return -1
188 188
  189 + def copytree(self, src, dest, overwrite='no_overwrite', log_level=INFO,
  190 + error_level=ERROR):
  191 + """an implementation of shutil.copytree however it allows for
  192 + dest to exist and implements differen't overwrite levels.
  193 + overwrite uses:
  194 + 'no_overwrite' will keep all(any) existing files in destination tree
  195 + 'overwrite_if_exists' will only overwrite destination paths that have
  196 + the same path names relative to the root of the src and
  197 + destination tree
  198 + 'clobber' will replace the whole destination tree(clobber) if it exists"""
  199 +
  200 + self.info('copying tree: %s to %s' % (src, dest))
  201 + try:
  202 + if overwrite == 'clobber':
  203 + self.rmtree(dest)
  204 + shutil.copytree(src, dest)
  205 + elif overwrite == 'no_overwrite' or overwrite == 'overwrite_if_exists':
  206 + files = os.listdir(src)
  207 + for f in files:
  208 + abs_src_f = os.path.join(src, f)
  209 + abs_dest_f = os.path.join(dest, f)
  210 + if not os.path.exists(abs_dest_f):
  211 + if os.path.isdir(abs_src_f):
  212 + self.mkdir_p(abs_dest_f)
  213 + self.copytree(abs_src_f, abs_dest_f,
  214 + overwrite='clobber')
  215 + else:
  216 + shutil.copy2(abs_src_f, abs_dest_f)
  217 + elif overwrite == 'no_overwrite': # destination path exists
  218 + if os.path.isdir(abs_src_f) and os.path.isdir(abs_dest_f):
  219 + self.copytree(abs_src_f, abs_dest_f,
  220 + overwrite='no_overwrite')
  221 + else:
  222 + self.debug('ignoring path: %s as destination: \
  223 + %s exists' % (abs_src_f, abs_dest_f))
  224 + else: # overwrite == 'overwrite_if_exists' and destination exists
  225 + self.debug('overwriting: %s with: %s' %
  226 + (abs_dest_f, abs_src_f))
  227 + self.rmtree(abs_dest_f)
  228 +
  229 + if os.path.isdir(abs_src_f):
  230 + self.mkdir_p(abs_dest_f)
  231 + self.copytree(abs_src_f, abs_dest_f,
  232 + overwrite='overwrite_if_exists')
  233 + else:
  234 + shutil.copy2(abs_src_f, abs_dest_f)
  235 + else:
  236 + self.fatal("%s is not a valid argument for param overwrite" % (overwrite))
  237 + except (IOError, shutil.Error):
  238 + self.dump_exception("There was an error while copying %s to %s!" % (src, dest),
  239 + level=error_level)
  240 + return -1
  241 +
189 242 def write_to_file(self, file_path, contents, verbose=True,
190 243 open_mode='w', create_parent_dir=False,
191 244 error_level=ERROR):
@@ -372,8 +425,6 @@ def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
372 425 ]
373 426 (context_lines isn't written yet)
374 427 """
375   - if error_list is None:
376   - error_list = []
377 428 if success_codes is None:
378 429 success_codes = [0]
379 430 if cwd:
@@ -395,8 +446,16 @@ def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
395 446 shell = True
396 447 if isinstance(command, list):
397 448 shell = False
398   - p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE,
399   - cwd=cwd, stderr=subprocess.STDOUT, env=env)
  449 + try:
  450 + p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE,
  451 + cwd=cwd, stderr=subprocess.STDOUT, env=env)
  452 + except OSError, e:
  453 + level = ERROR
  454 + if halt_on_failure:
  455 + level = FATAL
  456 + self.log('caught OS error %s: %s while running %s' % (e.errno,
  457 + e.strerror, command), level=level)
  458 + return -1
400 459 if output_parser is None:
401 460 parser = OutputParser(config=self.config, log_obj=self.log_obj,
402 461 error_list=error_list)
48 mozharness/mozilla/testing/errors.py
... ... @@ -0,0 +1,48 @@
  1 +#!/usr/bin/env python
  2 +# ***** BEGIN LICENSE BLOCK *****
  3 +# This Source Code Form is subject to the terms of the Mozilla Public
  4 +# License, v. 2.0. If a copy of the MPL was not distributed with this file,
  5 +# You can obtain one at http://mozilla.org/MPL/2.0/.
  6 +# ***** END LICENSE BLOCK *****
  7 +"""Mozilla error lists for running tests.
  8 +
  9 +Error lists are used to parse output in mozharness.base.log.OutputParser.
  10 +
  11 +Each line of output is matched against each substring or regular expression
  12 +in the error list. On a match, we determine the 'level' of that line,
  13 +whether IGNORE, DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL.
  14 +
  15 +"""
  16 +
  17 +import re
  18 +from mozharness.base.log import INFO
  19 +
  20 +# ErrorLists {{{1
  21 +TinderBoxPrintRe = {
  22 + "mochitest_summary": {
  23 + 'regex': re.compile(r'''(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))'''),
  24 + 'pass_group': "Passed",
  25 + 'fail_group': "Failed",
  26 + 'known_fail_group': "Todo",
  27 + },
  28 + "reftest_summary": {
  29 + 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),
  30 + 'pass_group': "Successful",
  31 + 'fail_group': "Unexpected",
  32 + 'known_fail_group': "Known problems",
  33 + },
  34 + "xpcshell_summary": {
  35 + 'regex': re.compile(r'''INFO \| (Passed|Failed): (\d+)'''),
  36 + 'pass_group': "Passed",
  37 + 'fail_group': "Failed",
  38 + 'known_fail_group': None,
  39 + },
  40 + "harness_error": {
  41 + 'full_regex': re.compile(r"TEST-UNEXPECTED-FAIL \| .* \| (Browser crashed \(minidump found\)|missing output line for total leaks!|negative leaks caught!|leaked \d+ bytes during test execution)"),
  42 + 'minimum_regex': re.compile(r'''TEST-UNEXPECTED''')
  43 + },
  44 +}
  45 +
  46 +TestPassed = [
  47 + {'regex': re.compile('''(TEST-INFO|TEST-KNOWN-FAIL|TEST-PASS|INFO \| )'''), 'level': INFO},
  48 +]
15 mozharness/mozilla/testing/testbase.py
@@ -130,15 +130,20 @@ def _download_test_zip(self):
130 130 error_level=FATAL)
131 131 self.test_zip_path = os.path.realpath(source)
132 132
133   - def _extract_test_zip(self):
  133 + def _extract_test_zip(self, target_unzip_dirs=None):
134 134 dirs = self.query_abs_dirs()
135 135 unzip = self.query_exe("unzip")
136 136 test_install_dir = dirs.get('abs_test_install_dir',
137 137 os.path.join(dirs['abs_work_dir'], 'tests'))
138 138 self.mkdir_p(test_install_dir)
  139 + # adding overwrite flag otherwise subprocess.Popen hangs on waiting for
  140 + # input in a hidden pipe whenever this action is run twice without
  141 + # clobber
  142 + unzip_cmd = [unzip, '-o', self.test_zip_path]
  143 + if target_unzip_dirs:
  144 + unzip_cmd.extend(target_unzip_dirs)
139 145 # TODO error_list
140   - self.run_command([unzip, self.test_zip_path],
141   - cwd=test_install_dir)
  146 + self.run_command(unzip_cmd, cwd=test_install_dir)
142 147
143 148 def _download_installer(self):
144 149 file_name = None
@@ -153,7 +158,7 @@ def _download_installer(self):
153 158
154 159 def download_and_extract(self):
155 160 """
156   - Create virtualenv and install dependencies
  161 + download and extract test zip / download installer
157 162 """
158 163 if self.test_url:
159 164 self._download_test_zip()
@@ -192,7 +197,7 @@ def install(self):
192 197 dirs = self.query_abs_dirs()
193 198 target_dir = dirs.get('abs_app_install_dir',
194 199 os.path.join(dirs['abs_work_dir'],
195   - 'application'))
  200 + 'application'))
196 201 self.mkdir_p(target_dir)
197 202 cmd.extend([self.installer_path,
198 203 '--destination', target_dir])
490 scripts/desktop_unittest.py
... ... @@ -0,0 +1,490 @@
  1 +#!/usr/bin/env python
  2 +# ***** BEGIN LICENSE BLOCK *****
  3 +# This Source Code Form is subject to the terms of the Mozilla Public
  4 +# License, v. 2.0. If a copy of the MPL was not distributed with this file,
  5 +# You can obtain one at http://mozilla.org/MPL/2.0/.
  6 +# ***** END LICENSE BLOCK *****
  7 +"""desktop_unittest.py
  8 +The goal of this is to extract desktop unittestng from buildbot's factory.py
  9 +
  10 +author: Jordan Lund
  11 +"""
  12 +
  13 +import os
  14 +import sys
  15 +import copy
  16 +import platform
  17 +import shutil
  18 +
  19 +# load modules from parent dir
  20 +sys.path.insert(1, os.path.dirname(sys.path[0]))
  21 +
  22 +from mozharness.base.errors import BaseErrorList
  23 +from mozharness.mozilla.testing.errors import TinderBoxPrintRe
  24 +from mozharness.base.vcs.vcsbase import MercurialScript
  25 +from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
  26 +from mozharness.base.log import OutputParser, WARNING, INFO
  27 +from mozharness.mozilla.buildbot import TBPL_WARNING, TBPL_FAILURE
  28 +from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_STATUS_DICT
  29 +
  30 +SUITE_CATEGORIES = ['mochitest', 'reftest', 'xpcshell']
  31 +
  32 +
  33 +class DesktopUnittestOutputParser(OutputParser):
  34 + """
  35 + A class that extends OutputParser such that it can parse the number of
  36 + passed/failed/todo tests from the output.
  37 + """
  38 +
  39 + def __init__(self, suite_category, **kwargs):
  40 + # worst_log_level defined already in DesktopUnittestOutputParser
  41 + # but is here to make pylint happy
  42 + self.worst_log_level = INFO
  43 + super(DesktopUnittestOutputParser, self).__init__(**kwargs)
  44 + self.summary_suite_re = TinderBoxPrintRe.get('%s_summary' % suite_category, {})
  45 + self.harness_error_re = TinderBoxPrintRe['harness_error']['minimum_regex']
  46 + self.full_harness_error_re = TinderBoxPrintRe['harness_error']['full_regex']
  47 + self.fail_count = -1
  48 + self.pass_count = -1
  49 + # known_fail_count does not exist for some suites
  50 + self.known_fail_count = self.summary_suite_re.get('known_fail_group') and -1
  51 + self.crashed, self.leaked = False, False
  52 + self.tbpl_status = TBPL_SUCCESS
  53 +
  54 + def parse_single_line(self, line):
  55 + if self.summary_suite_re:
  56 + summary_m = self.summary_suite_re['regex'].match(line) # pass/fail/todo
  57 + if summary_m:
  58 + message = ' %s' % line
  59 + log_level = INFO
  60 + # remove all the none values in groups() so this will work
  61 + # with all suites including mochitest browser-chrome
  62 + summary_match_list = [group for group in summary_m.groups()
  63 + if group is not None]
  64 + r = summary_match_list[0]
  65 + if self.summary_suite_re['pass_group'] in r:
  66 + self.pass_count = int(summary_match_list[-1])
  67 + elif self.summary_suite_re['fail_group'] in r:
  68 + self.fail_count = int(summary_match_list[-1])
  69 + if self.fail_count > 0:
  70 + message += '\n One or more unittests failed.'
  71 + log_level = WARNING
  72 + # If self.summary_suite_re['known_fail_group'] == None,
  73 + # then r should not match it, # so this test is fine as is.
  74 + elif self.summary_suite_re['known_fail_group'] in r:
  75 + self.known_fail_count = int(summary_match_list[-1])
  76 + self.log(message, log_level)
  77 + return # skip harness check and base parse_single_line
  78 + harness_match = self.harness_error_re.match(line)
  79 + if harness_match:
  80 + self.warning(' %s\n This is a harness error.' % line)
  81 + self.worst_log_level = self.worst_level(WARNING, self.worst_log_level)
  82 + self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
  83 + levels=TBPL_STATUS_DICT.keys())
  84 + full_harness_match = self.full_harness_error_re.match(line)
  85 + if full_harness_match:
  86 + r = harness_match.group(1)
  87 + if r == "Browser crashed (minidump found)":
  88 + self.crashed = True
  89 + elif r == "missing output line for total leaks!":
  90 + self.leaked = None
  91 + else:
  92 + self.leaked = True
  93 + return # skip base parse_single_line
  94 + super(DesktopUnittestOutputParser, self).parse_single_line(line)
  95 +
  96 + def evaluate_parser(self, return_code):
  97 + if self.num_errors: # mozharness ran into a script error
  98 + self.tbpl_status = TBPL_FAILURE
  99 +
  100 + # I have to put this outside of parse_single_line because this checks not
  101 + # only if fail_count was more then 0 but also if fail_count is still -1
  102 + # (no fail summary line was found)
  103 + if self.fail_count != 0:
  104 + self.worst_log_level = self.worst_level(WARNING, self.worst_log_level)
  105 + self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
  106 + levels=TBPL_STATUS_DICT.keys())
  107 + # we can trust in parser.worst_log_level in either case
  108 + return (self.tbpl_status, self.worst_log_level)
  109 +
  110 + def append_tinderboxprint_line(self, suite_name):
  111 + # We are duplicating a condition (fail_count) from evaluate_parser and
  112 + # parse parse_single_line but at little cost since we are not parsing
  113 + # the log more then once. I figured this method should stay isolated as
  114 + # it is only here for tbpl highlighted summaries and is not part of
  115 + # buildbot evaluation or result status IIUC.
  116 + emphasize_fail_text = '<em class="testfail">%s</em>'
  117 +
  118 + if self.pass_count < 0 or self.fail_count < 0 or self.known_fail_count < 0:
  119 + summary = emphasize_fail_text % 'T-FAIL'
  120 + elif self.pass_count == 0 and self.fail_count == 0 and \
  121 + (self.known_fail_count == 0 or self.known_fail_count is None):
  122 + summary = emphasize_fail_text % 'T-FAIL'
  123 + else:
  124 + str_fail_count = str(self.fail_count)
  125 + if self.fail_count > 0:
  126 + str_fail_count = emphasize_fail_text % str_fail_count
  127 + summary = "%d/%s" % (self.pass_count, str_fail_count)
  128 + if self.known_fail_count is not None:
  129 + summary += "/%d" % self.known_fail_count
  130 + # Format the crash status.
  131 + if self.crashed:
  132 + summary += "&nbsp;%s" % emphasize_fail_text % "CRASH"
  133 + # Format the leak status.
  134 + if self.leaked is not False:
  135 + summary += "&nbsp;%s" % emphasize_fail_text % (
  136 + (self.leaked and "LEAK") or "L-FAIL")
  137 + # Return the summary.
  138 + self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, summary))
  139 +
  140 +
  141 +# DesktopUnittest {{{1
  142 +class DesktopUnittest(TestingMixin, MercurialScript):
  143 +
  144 + config_options = [
  145 + [['--mochitest-suite', ], {
  146 + "action": "append",
  147 + "dest": "specified_mochitest_suites",
  148 + "type": "string",
  149 + "help": "Specify which mochi suite to run."
  150 + "Suites are defined in the config file.\n"
  151 + "Examples: 'all', 'plain1', 'plain5', 'chrome', or 'a11y'"}
  152 + ],
  153 + [['--reftest-suite', ], {
  154 + "action": "append",
  155 + "dest": "specified_reftest_suites",
  156 + "type": "string",
  157 + "help": "Specify which reftest suite to run."
  158 + "Suites are defined in the config file.\n"
  159 + "Examples: 'all', 'crashplan', or 'jsreftest'"}
  160 + ],
  161 + [['--xpcshell-suite', ], {
  162 + "action": "append",
  163 + "dest": "specified_xpcshell_suites",
  164 + "type": "string",
  165 + "help": "Specify which xpcshell suite to run."
  166 + "Suites are defined in the config file\n."
  167 + "Examples: 'xpcshell'"}
  168 + ],
  169 + [['--run-all-suites', ], {
  170 + "action": "store_true",
  171 + "dest": "run_all_suites",
  172 + "default": False,
  173 + "help": "This will run all suites that are specified"
  174 + "in the config file. You do not need to specify "
  175 + "any other suites.\nBeware, this may take a while ;)"}
  176 + ],
  177 + [['--enable-preflight-run-commands', ], {
  178 + "action": "store_false",
  179 + "dest": "preflight_run_commands_disabled",
  180 + "default": True,
  181 + "help": "This will enable any run commands that are specified"
  182 + "in the config file under: preflight_run_cmd_suites"}
  183 + ]
  184 + ] + copy.deepcopy(testing_config_options)
  185 +
  186 + virtualenv_modules = [
  187 + "simplejson",
  188 + {'mozlog': os.path.join('tests', 'mozbase', 'mozlog')},
  189 + {'mozinfo': os.path.join('tests', 'mozbase', 'mozinfo')},
  190 + {'mozhttpd': os.path.join('tests', 'mozbase', 'mozhttpd')},
  191 + {'mozinstall': os.path.join('tests', 'mozbase', 'mozinstall')},
  192 + {'manifestdestiny': os.path.join('tests', 'mozbase', 'manifestdestiny')},
  193 + {'mozprofile': os.path.join('tests', 'mozbase', 'mozprofile')},
  194 + {'mozprocess': os.path.join('tests', 'mozbase', 'mozprocess')},
  195 + {'mozrunner': os.path.join('tests', 'mozbase', 'mozrunner')},
  196 + ]
  197 +
  198 + def __init__(self, require_config_file=True):
  199 + # abs_dirs defined already in BaseScript but is here to make pylint happy
  200 + self.abs_dirs = None
  201 + MercurialScript.__init__(
  202 + self,
  203 + config_options=self.config_options,
  204 + all_actions=[
  205 + 'clobber',
  206 + 'read-buildbot-config',
  207 + 'download-and-extract',
  208 + 'pull',
  209 + 'create-virtualenv',
  210 + 'install',
  211 + 'run-tests',
  212 + ],
  213 + require_config_file=require_config_file,
  214 + config={'virtualenv_modules': self.virtualenv_modules,
  215 + 'require_test_zip': True})
  216 +
  217 + c = self.config
  218 + self.global_test_options = []
  219 + self.installer_url = c.get('installer_url')
  220 + self.test_url = c.get('test_url')
  221 + self.symbols_url = c.get('symbols_url')
  222 + # this is so mozinstall in install() doesn't bug out if we don't run the
  223 + # download_and_extract action
  224 + self.installer_path = os.path.join(self.abs_dirs['abs_work_dir'],
  225 + c.get('installer_path'))
  226 + self.binary_path = os.path.join(self.abs_dirs['abs_app_install_dir'],
  227 + c.get('binary_path'))
  228 +
  229 + ###### helper methods
  230 + def _pre_config_lock(self, rw_config):
  231 + c = self.config
  232 + if not c.get('run_all_suites'):
  233 + return # configs are valid
  234 + for category in SUITE_CATEGORIES:
  235 + specific_suites = c.get('specified_%s_suites' % (category))
  236 + if specific_suites:
  237 + if specific_suites != 'all':
  238 + self.fatal("Config options are not valid. Please ensure"
  239 + " that if the '--run-all-suites' flag was enabled,"
  240 + " then do not specify to run only specific suites "
  241 + "like:\n '--mochitest-suite browser-chrome'")
  242 +
  243 + def query_abs_dirs(self):
  244 + if self.abs_dirs:
  245 + return self.abs_dirs
  246 + abs_dirs = super(DesktopUnittest, self).query_abs_dirs()
  247 +
  248 + c = self.config
  249 + dirs = {}
  250 + dirs['abs_app_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'application')
  251 + dirs['abs_app_dir'] = os.path.join(dirs['abs_app_install_dir'], c['app_name_dir'])
  252 + dirs['abs_app_plugins_dir'] = os.path.join(dirs['abs_app_dir'], 'plugins')
  253 + dirs['abs_app_components_dir'] = os.path.join(dirs['abs_app_dir'], 'components')
  254 + dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
  255 + dirs['abs_test_bin_dir'] = os.path.join(dirs['abs_test_install_dir'], 'bin')
  256 + dirs['abs_test_bin_plugins_dir'] = os.path.join(dirs['abs_test_bin_dir'],
  257 + 'plugins')
  258 + dirs['abs_test_bin_components_dir'] = os.path.join(dirs['abs_test_bin_dir'],
  259 + 'components')
  260 + dirs['abs_mochitest_dir'] = os.path.join(dirs['abs_test_install_dir'], "mochitest")
  261 + dirs['abs_reftest_dir'] = os.path.join(dirs['abs_test_install_dir'], "reftest")
  262 + dirs['abs_xpcshell_dir'] = os.path.join(dirs['abs_test_install_dir'], "xpcshell")
  263 +
  264 + if os.path.isabs(c['virtualenv_path']):
  265 + dirs['abs_virtualenv_dir'] = c['virtualenv_path']
  266 + else:
  267 + dirs['abs_virtualenv_dir'] = os.path.join(abs_dirs['abs_work_dir'],
  268 + c['virtualenv_path'])
  269 + abs_dirs.update(dirs)
  270 + self.abs_dirs = abs_dirs
  271 +
  272 + return self.abs_dirs
  273 +
  274 + def _query_symbols_url(self):
  275 + """query the full symbols URL based upon binary URL"""
  276 + # may break with name convention changes but is one less 'input' for script
  277 + if self.symbols_url:
  278 + return self.symbols_url
  279 +
  280 + symbols_url = None
  281 + self.info("finding symbols_url based upon self.installer_url")
  282 + if self.installer_url:
  283 + for ext in ['.zip', '.dmg', '.tar.bz2']:
  284 + if ext in self.installer_url:
  285 + symbols_url = self.installer_url.replace(
  286 + ext, '.crashreporter-symbols.zip')
  287 + if not symbols_url:
  288 + self.fatal("self.installer_url was found but symbols_url could \
  289 + not be determined")
  290 + else:
  291 + self.fatal("self.installer_url was not found in self.config")
  292 + self.info("setting symbols_url as %s" % (symbols_url))
  293 + self.symbols_url = symbols_url
  294 + return self.symbols_url
  295 +
  296 + def _query_abs_base_cmd(self, suite_category):
  297 + if self.binary_path:
  298 + c = self.config
  299 + dirs = self.query_abs_dirs()
  300 + options = []
  301 + run_file = c['run_file_names'][suite_category]
  302 + base_cmd = [self.query_python_path('python'), '-u']
  303 + base_cmd.append(dirs["abs_%s_dir" % suite_category] + "/" + run_file)
  304 + str_format_values = {
  305 + 'binary_path': self.binary_path,
  306 + 'symbols_path': self._query_symbols_url()
  307 + }
  308 + if self.config['%s_options' % suite_category]:
  309 + for option in self.config['%s_options' % suite_category]:
  310 + options.append(option % str_format_values)
  311 + abs_base_cmd = base_cmd + options
  312 + return abs_base_cmd
  313 + else:
  314 + self.warning("Suite options for %s could not be determined."
  315 + "\nIf you meant to have options for this suite, "
  316 + "please make sure they are specified in your "
  317 + "config under %s_options" %
  318 + (suite_category, suite_category))
  319 + else:
  320 + self.fatal("'binary_path' could not be determined.\n This should"
  321 + "be like '/path/build/application/firefox/firefox'"
  322 + "\nIf you are running this script without the 'install' "
  323 + "action (where binary_path is set), please ensure you are"
  324 + " either:\n(1) specifying it in the config file under "
  325 + "binary_path\n(2) specifying it on command line with the"
  326 + " '--binary-path' flag")
  327 +
  328 + def _query_specified_suites(self, category):
  329 + # logic goes: if at least one '--{category}-suite' was given,
  330 + # then run only that(those) given suite(s). Elif no suites were
  331 + # specified and the --run-all-suites flag was given,
  332 + # run all {category} suites. Anything else, run no suites.
  333 + c = self.config
  334 + all_suites = c.get('all_%s_suites' % (category))
  335 + specified_suites = c.get('specified_%s_suites' % (category)) # list
  336 + suites = None
  337 +
  338 + if specified_suites:
  339 + if 'all' in specified_suites:
  340 + # useful if you want a quick way of saying run all suites
  341 + # of a specific category.
  342 + suites = all_suites
  343 + else:
  344 + # suites gets a dict of everything from all_suites where a key
  345 + # is also in specified_suites
  346 + suites = dict((key, all_suites.get(key)) for key in
  347 + specified_suites if key in all_suites.keys())
  348 + else:
  349 + if c.get('run_all_suites'): # needed if you dont specify any suites
  350 + suites = all_suites
  351 +
  352 + return suites
  353 +
  354 + # Actions {{{2
  355 +
  356 + # clobber defined in BaseScript, deletes mozharness/build if exists
  357 + # read_buildbot_config is in BuildbotMixin.
  358 + # postflight_read_buildbot_config is in TestingMixin.
  359 + # preflight_download_and_extract is in TestingMixin.
  360 + # create_virtualenv is in VirtualenvMixin.
  361 + # preflight_install is in TestingMixin.
  362 + # install is in TestingMixin.
  363 +
  364 + def download_and_extract(self):
  365 + """
  366 + download and extract test zip / download installer
  367 + optimizes which subfolders to extract from tests zip
  368 + """
  369 + c = self.config
  370 + unzip_tests_dirs = None
  371 +
  372 + if c['specific_tests_zip_dirs']:
  373 + unzip_tests_dirs = c['minimum_tests_zip_dirs']
  374 + for category in c['specific_tests_zip_dirs'].keys():
  375 + if c['run_all_suites'] or self._query_specified_suites(category) \
  376 + or 'run-tests' not in self.actions:
  377 + unzip_tests_dirs.extend(c['specific_tests_zip_dirs'][category])
  378 + if self.test_url:
  379 + self._download_test_zip()
  380 + self._extract_test_zip(target_unzip_dirs=unzip_tests_dirs)
  381 + self._download_installer()
  382 +
  383 + def pull(self):
  384 + dirs = self.query_abs_dirs()
  385 + c = self.config
  386 + if c.get('repos'):
  387 + dirs = self.query_abs_dirs()
  388 + self.vcs_checkout_repos(c['repos'],
  389 + parent_dir=dirs['abs_work_dir'])
  390 +
  391 + def preflight_run_tests(self):
  392 + """preflight commands for all tests"""
  393 + c = self.config
  394 + dirs = self.query_abs_dirs()
  395 +
  396 + if not c.get('preflight_run_commands_disabled'):
  397 + arch = platform.architecture()[0]
  398 + for suite in c['preflight_run_cmd_suites']:
  399 + # XXX platform.architecture() may give incorrect values for some
  400 + # platforms like mac as excutable files may be universal
  401 + # files containing multiple architectures
  402 + # NOTE 'enabled' is only here while we have unconsolidated configs
  403 + if suite['enabled'] and arch in suite['architectures']:
  404 + cmd = suite['cmd']
  405 + name = suite['name']
  406 + self.info("Running pre test command %(name)s with '%(cmd)s'"
  407 + % {'name': name, 'cmd': ' '.join(cmd)})
  408 + if self.buildbot_config: # this cmd is for buildbot
  409 + # TODO rather then checking for formatting on every string
  410 + # in every preflight enabled cmd: find a better solution!
  411 + # maybe I can implement WithProperties in mozharness?
  412 + cmd = [x % (self.buildbot_config.get('properties'))
  413 + for x in cmd]
  414 + self.run_command(cmd,
  415 + cwd=dirs['abs_work_dir'],
  416 + error_list=BaseErrorList,
  417 + halt_on_failure=suite['halt_on_failure'])
  418 + else:
  419 + self.warning("Proceeding without running prerun test commands."
  420 + " These are often OS specific and disabling them may"
  421 + " result in spurious test results!")
  422 +
  423 + def run_tests(self):
  424 + self._run_category_suites('mochitest')
  425 + self._run_category_suites('reftest')
  426 + self._run_category_suites('xpcshell',
  427 + preflight_run_method=self.preflight_xpcshell)
  428 +
  429 + def preflight_xpcshell(self, suites):
  430 + c = self.config
  431 + dirs = self.query_abs_dirs()
  432 + if suites: # there are xpcshell suites to run
  433 + self.mkdir_p(dirs['abs_app_plugins_dir'])
  434 + self.info('copying %s to %s' % (os.path.join(dirs['abs_test_bin_dir'],
  435 + c['xpcshell_name']), os.path.join(dirs['abs_app_dir'],
  436 + c['xpcshell_name'])))
  437 + shutil.copy2(os.path.join(dirs['abs_test_bin_dir'], c['xpcshell_name']),
  438 + os.path.join(dirs['abs_app_dir'], c['xpcshell_name']))
  439 + self.copytree(dirs['abs_test_bin_components_dir'],
  440 + dirs['abs_app_components_dir'],
  441 + overwrite='overwrite_if_exists')
  442 + self.copytree(dirs['abs_test_bin_plugins_dir'],
  443 + dirs['abs_app_plugins_dir'],
  444 + overwrite='overwrite_if_exists')
  445 +
  446 + def _run_category_suites(self, suite_category, preflight_run_method=None):
  447 + """run suite(s) to a specific category"""
  448 + dirs = self.query_abs_dirs()
  449 + abs_base_cmd = self._query_abs_base_cmd(suite_category)
  450 + suites = self._query_specified_suites(suite_category)
  451 +
  452 + if preflight_run_method:
  453 + preflight_run_method(suites)
  454 + if suites:
  455 + self.info('#### Running %s suites' % suite_category)
  456 + for suite in suites:
  457 + cmd = abs_base_cmd + suites[suite]
  458 + suite_name = suite_category + '-' + suite
  459 + tbpl_status, log_level = None, None
  460 + parser = DesktopUnittestOutputParser(suite_category,
  461 + config=self.config,
  462 + error_list=BaseErrorList,
  463 + log_obj=self.log_obj)
  464 +
  465 + return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'],
  466 + output_parser=parser)
  467 +
  468 + # mochitests, reftests, and xpcshell suites do not return
  469 + # appropriate return codes. Therefore, we must parse the output
  470 + # to determine what the tbpl_status and worst_log_level must
  471 + # be. We do this by:
  472 + # 1) checking to see if our mozharness script ran into any
  473 + # errors itself with 'num_errors' <- OutputParser
  474 + # 2) if num_errors is 0 then we look in the subclassed 'parser'
  475 + # findings for harness/suite errors <- DesktopUnittestOutputParser
  476 + tbpl_status, log_level = parser.evaluate_parser(return_code)
  477 + parser.append_tinderboxprint_line(suite_name)
  478 +
  479 + self.buildbot_status(tbpl_status, level=log_level)
  480 + self.add_summary("The %s suite: %s ran with return status: %s" %
  481 + (suite_category, suite, tbpl_status),
  482 + level=log_level)
  483 + else:
  484 + self.debug('There were no suites to run for %s' % suite_category)
  485 +
  486 +
  487 +# main {{{1
  488 +if __name__ == '__main__':
  489 + desktop_unittest = DesktopUnittest()
  490 + desktop_unittest.run()

Git Notes

Upstream source: https://hg.mozilla.org/build/mozharness/rev/ca3b37a3b8fff979f36883d283cdafd2adfb8d1e

0 comments on commit 9081198

Please sign in to comment.
Something went wrong with that request. Please try again.