Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Bug 777841 - Add mozharness script for Marionette, r=aki

  • Loading branch information...
commit d53e34c5beb8d8b46f18bcf40e4c65d4a7610343 1 parent 0e81b99
@jonallengriffin jonallengriffin authored
View
70 mozharness/base/log.py
@@ -134,44 +134,46 @@ def __init__(self, config=None, log_obj=None, error_list=None,
# (WARNING, ERROR, CRITICAL, FATAL)
# self.error_level = INFO
+ def parse_single_line(self, line):
+ if not line or line.isspace():
+ return
+ line = line.decode("utf-8").rstrip()
+ for error_check in self.error_list:
+ # TODO buffer for context_lines.
+ match = False
+ if 'substr' in error_check:
+ if error_check['substr'] in line:
+ match = True
+ elif 'regex' in error_check:
+ if error_check['regex'].search(line):
+ match = True
+ else:
+ self.warn("error_list: 'substr' and 'regex' not in %s" % \
+ error_check)
+ if match:
+ level = error_check.get('level', INFO)
+ if self.log_output:
+ message = ' %s' % line
+ if error_check.get('explanation'):
+ message += '\n %s' % error_check['explanation']
+ if error_check.get('summary'):
+ self.add_summary(message, level=level)
+ else:
+ self.log(message, level=level)
+ if level in (ERROR, CRITICAL, FATAL):
+ self.num_errors += 1
+ # TODO set self.error_status (or something)
+ # that sets the worst error level hit.
+ break
+ else:
+ if self.log_output:
+ self.info(' %s' % line)
+
def add_lines(self, output):
if str(output) == output:
output = [output]
for line in output:
- if not line or line.isspace():
- continue
- line = line.decode("utf-8").rstrip()
- for error_check in self.error_list:
- # TODO buffer for context_lines.
- match = False
- if 'substr' in error_check:
- if error_check['substr'] in line:
- match = True
- elif 'regex' in error_check:
- if error_check['regex'].search(line):
- match = True
- else:
- self.warn("error_list: 'substr' and 'regex' not in %s" % \
- error_check)
- if match:
- level = error_check.get('level', INFO)
- if self.log_output:
- message = ' %s' % line
- if error_check.get('explanation'):
- message += '\n %s' % error_check['explanation']
- if error_check.get('summary'):
- self.add_summary(message, level=level)
- else:
- self.log(message, level=level)
- if level in (ERROR, CRITICAL, FATAL):
- self.num_errors += 1
- # TODO set self.error_status (or something)
- # that sets the worst error level hit.
- break
- else:
- if self.log_output:
- self.info(' %s' % line)
-
+ self.parse_single_line(line)
# BaseLogger {{{1
View
13 mozharness/base/script.py
@@ -352,7 +352,8 @@ def query_exe(self, exe_name, exe_dict='exes', default=None,
def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
halt_on_failure=False, success_codes=None,
- env=None, return_type='status', throw_exception=False):
+ env=None, return_type='status', throw_exception=False,
+ output_parser=None):
"""Run a command, with logging and error parsing.
TODO: parse_at_end, context_lines
@@ -361,6 +362,9 @@ def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
TODO: Add a copy-pastable version of |command| if it's a list.
TODO: print env if set
+ output_parser lets you provide an instance of your own OutputParser
+ subclass, or pass None to use OutputParser.
+
error_list example:
[{'regex': re.compile('^Error: LOL J/K'), level=IGNORE},
{'regex': re.compile('^Error:'), level=ERROR, contextLines='5:5'},
@@ -393,8 +397,11 @@ def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
shell = False
p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE,
cwd=cwd, stderr=subprocess.STDOUT, env=env)
- parser = OutputParser(config=self.config, log_obj=self.log_obj,
- error_list=error_list)
+ if output_parser is None:
+ parser = OutputParser(config=self.config, log_obj=self.log_obj,
+ error_list=error_list)
+ else:
+ parser = output_parser
loop = True
while loop:
if p.poll() is not None:
View
195 scripts/marionette.py
@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+
+import copy
+import os
+import re
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.errors import PythonErrorList
+from mozharness.base.log import INFO, ERROR, OutputParser
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+
+
+class MarionetteOutputParser(OutputParser):
+ """
+ A class that extends OutputParser such that it can parse the number of
+ passed/failed/todo tests from the output.
+ """
+
+ summary = re.compile(r'(passed|failed|todo): (\d+)')
+
+ def __init__(self, **kwargs):
+ self.failed = 0
+ self.passed = 0
+ self.todo = 0
+ super(MarionetteOutputParser, self).__init__(**kwargs)
+
+ def parse_single_line(self, line):
+ m = self.summary.match(line)
+ if m:
+ try:
+ setattr(self, m.group(1), int(m.group(2)))
+ except ValueError:
+ # ignore bad values
+ pass
+ super(MarionetteOutputParser, self).parse_single_line(line)
+
+
+class MarionetteTest(TestingMixin, BaseScript):
+ config_options = [
+ [["--type"],
+ {"action": "store",
+ "dest": "test_type",
+ "default": "browser",
+ "help": "The type of tests to run",
+ }],
+ [["--marionette-address"],
+ {"action": "store",
+ "dest": "marionette_address",
+ "default": "localhost:2828",
+ "help": "The host:port of the Marionette server running inside Gecko",
+ }],
+ [["--test-manifest"],
+ {"action": "store",
+ "dest": "test_manifest",
+ "default": None,
+ "help": "Path to test manifest to run",
+ }]] + copy.deepcopy(testing_config_options)
+
+ error_list = [
+ {'substr': r'''FAILED (errors=''', 'level': ERROR},
+ ]
+
+ virtualenv_modules = [
+ 'datazilla',
+ 'mozinstall',
+ 'mozhttpd',
+ 'manifestdestiny',
+ 'mozprofile',
+ 'mozprocess',
+ 'mozrunner',
+ ]
+
+ def __init__(self, require_config_file=False):
+ super(MarionetteTest, self).__init__(
+ config_options=self.config_options,
+ all_actions=['clobber',
+ 'read-buildbot-config',
+ 'download-and-extract',
+ 'create-virtualenv',
+ 'install',
+ 'run-marionette'],
+ default_actions=['clobber',
+ 'download-and-extract',
+ 'create-virtualenv',
+ 'install',
+ 'run-marionette'],
+ require_config_file=require_config_file,
+ config={'virtualenv_modules': self.virtualenv_modules,
+ 'require_test_zip': True,})
+
+ # these are necessary since self.config is read only
+ c = self.config
+ self.installer_url = c.get('installer_url')
+ self.installer_path = c.get('installer_path')
+ self.binary_path = c.get('binary_path')
+ self.test_url = self.config.get('test_url')
+
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = super(MarionetteTest, self).query_abs_dirs()
+ dirs = {}
+ dirs['abs_test_install_dir'] = os.path.join(
+ abs_dirs['abs_work_dir'], 'tests')
+ dirs['abs_marionette_dir'] = os.path.join(
+ dirs['abs_test_install_dir'], 'marionette', 'marionette')
+ dirs['abs_marionette_tests_dir'] = os.path.join(
+ dirs['abs_test_install_dir'], 'marionette', 'tests', 'testing',
+ 'marionette', 'client', 'marionette', 'tests')
+ for key in dirs.keys():
+ if key not in abs_dirs:
+ abs_dirs[key] = dirs[key]
+ self.abs_dirs = abs_dirs
+ return self.abs_dirs
+
+ def _build_arg(self, option, value):
+ """
+ Build a command line argument
+ """
+ if not value:
+ return []
+ return [str(option), str(value)]
+
+ def run_marionette(self):
+ """
+ Run the Marionette tests
+ """
+ dirs = self.query_abs_dirs()
+
+ error_list = self.error_list
+ error_list.extend(PythonErrorList)
+
+ # build the marionette command arguments
+ python = self.query_python_path('python')
+ cmd = [python, '-u', os.path.join(dirs['abs_marionette_dir'],
+ 'runtests.py')]
+ cmd.extend(self._build_arg('--binary', self.binary_path))
+ cmd.extend(self._build_arg('--type', self.config['test_type']))
+ cmd.extend(self._build_arg('--address', self.config['marionette_address']))
+ manifest = self.config.get('test_manifest')
+ if not manifest:
+ manifest = os.path.join(dirs['abs_marionette_tests_dir'],
+ 'unit-tests.ini')
+ cmd.append(manifest)
+
+ marionette_parser = MarionetteOutputParser(config=self.config,
+ log_obj=self.log_obj,
+ error_list=error_list)
+ code = self.run_command(cmd,
+ output_parser=marionette_parser)
+ level = INFO
+ if code == 0:
+ status = "success"
+ tbpl_status = TBPL_SUCCESS
+ elif code == 10:
+ status = "test failures"
+ tbpl_status = TBPL_WARNING
+ else:
+ status = "harness failures"
+ level = ERROR
+ tbpl_status = TBPL_FAILURE
+
+ # generate the TinderboxPrint line for TBPL
+ emphasize_fail_text = '<em class="testfail">%s</em>'
+ if marionette_parser.passed == 0 and marionette_parser.failed == 0:
+ tsummary = emphasize_fail_text % "T-FAIL"
+ else:
+ failed = "0"
+ if marionette_parser.failed > 0:
+ failed = emphasize_fail_text % str(marionette_parser.failed)
+ tsummary = "%d/%s/%d" % (marionette_parser.passed,
+ failed,
+ marionette_parser.todo)
+ self.info("TinderboxPrint: marionette<br/>%s\n" % tsummary)
+
+ self.add_summary("Marionette exited with return code %s: %s" % (code,
+ status),
+ level=level)
+ self.buildbot_status(tbpl_status)
+
+
+if __name__ == '__main__':
+ marionetteTest = MarionetteTest()
+ marionetteTest.run()
+
Please sign in to comment.
Something went wrong with that request. Please try again.