From 943f77330f499297be36e403f8812fece50cbd4a Mon Sep 17 00:00:00 2001 From: Henrik Skupin Date: Tue, 17 Nov 2015 17:43:13 +0100 Subject: [PATCH] Switch to mozharness as test runner (#642) --- config/production/jenkins.patch | 6 +- jenkins-master/config.xml | 2 +- .../jobs/mozilla-aurora_functional/config.xml | 38 +--- .../jobs/mozilla-aurora_update/config.xml | 38 +--- .../mozilla-central_functional/config.xml | 38 +--- .../jobs/mozilla-central_update/config.xml | 38 +--- .../config.xml | 38 +--- .../config.xml | 38 +--- .../config.xml | 38 +--- .../jobs/scripts/workspace/buildbot.py | 20 ++ .../jobs/scripts/workspace/config.py | 26 ++- .../jobs/scripts/workspace/runtests.py | 199 +++++++++++------- .../jobs/scripts/workspace/submission.py | 140 ++++++------ 13 files changed, 281 insertions(+), 378 deletions(-) create mode 100644 jenkins-master/jobs/scripts/workspace/buildbot.py diff --git a/config/production/jenkins.patch b/config/production/jenkins.patch index e9acfcc7..b5496e05 100644 --- a/config/production/jenkins.patch +++ b/config/production/jenkins.patch @@ -1,5 +1,5 @@ diff --git a/jenkins-master/config.xml b/jenkins-master/config.xml -index a2d98a6..9fd2a63 100644 +index bd544ad..7a9ae47 100644 --- a/jenkins-master/config.xml +++ b/jenkins-master/config.xml @@ -17,16 +17,708 @@ @@ -798,7 +798,7 @@ index e6be56d..785729a 100644 -1 diff --git a/jenkins-master/jobs/mozilla-central_functional/config.xml b/jenkins-master/jobs/mozilla-central_functional/config.xml -index 92a9cf5..bcb8fdd 100644 +index 615040d..5fe5c26 100644 --- a/jenkins-master/jobs/mozilla-central_functional/config.xml +++ b/jenkins-master/jobs/mozilla-central_functional/config.xml @@ -3,8 +3,8 @@ @@ -813,7 +813,7 @@ index 92a9cf5..bcb8fdd 100644 -1 diff --git a/jenkins-master/jobs/mozilla-central_update/config.xml b/jenkins-master/jobs/mozilla-central_update/config.xml -index dcdeb1a..b4e6997 100644 +index 47c5f4d..94fbbed 100644 --- a/jenkins-master/jobs/mozilla-central_update/config.xml +++ b/jenkins-master/jobs/mozilla-central_update/config.xml @@ -3,8 +3,8 @@ diff --git a/jenkins-master/config.xml b/jenkins-master/config.xml index 1959905a..d21929c7 100644 --- a/jenkins-master/config.xml +++ b/jenkins-master/config.xml @@ -234,7 +234,7 @@ 6 MOZHARNESS_REVISION - 49b1fe014eea + d9243e369c22 MOZMILL_AUTOMATION_VERSION 2.0.10.2 NOTIFICATION_ADDRESS diff --git a/jenkins-master/jobs/mozilla-aurora_functional/config.xml b/jenkins-master/jobs/mozilla-aurora_functional/config.xml index ff157d26..ad4b5794 100644 --- a/jenkins-master/jobs/mozilla-aurora_functional/config.xml +++ b/jenkins-master/jobs/mozilla-aurora_functional/config.xml @@ -48,32 +48,9 @@ - - 2 - - - https://github.com/mozilla/firefox-ui-tests.git - - - - - */mozilla-aurora - - - false - - - - mozilla-aurora - - - true - - - - firefox-ui-tests - - + + get_mozharness + Successful true false @@ -86,20 +63,22 @@ scripts - + + true + python -u submission.py --test-type=functional --build-state=running --repository=mozilla-aurora --revision=$REVISION --locale=$LOCALE treeherder_venv false - python runtests.py --type=functional --installer-url=$INSTALLER_URL + python -u runtests.py --test-type=functional --repository=mozilla-aurora --installer-url=$INSTALLER_URL false - build/upload/*.* + build/upload/**,minidumps/** true false false @@ -195,7 +174,6 @@ ${FAILED_TESTS} absolute 3 - xterm diff --git a/jenkins-master/jobs/mozilla-aurora_update/config.xml b/jenkins-master/jobs/mozilla-aurora_update/config.xml index e6be56da..ba2bd596 100644 --- a/jenkins-master/jobs/mozilla-aurora_update/config.xml +++ b/jenkins-master/jobs/mozilla-aurora_update/config.xml @@ -65,32 +65,9 @@ NSPR_LOG_FILE=http.log - - 2 - - - https://github.com/mozilla/firefox-ui-tests.git - - - - - */mozilla-aurora - - - false - - - - mozilla-aurora - - - true - - - - firefox-ui-tests - - + + get_mozharness + Successful true false @@ -103,20 +80,22 @@ NSPR_LOG_FILE=http.log scripts - + + true + python -u submission.py --test-type=update --build-state=running --repository=mozilla-aurora --revision=$REVISION --locale=$LOCALE --update-number=$UPDATE_NUMBER treeherder_venv false - python runtests.py --type=update --installer-url=$INSTALLER_URL --update-channel=$CHANNEL --update-target-build-id=$TARGET_BUILD_ID + python -u runtests.py --test-type=update --repository=mozilla-aurora --installer-url=$INSTALLER_URL --update-channel=$CHANNEL --update-target-build-id=$TARGET_BUILD_ID false - build/upload/*.* + build/http.log,build/upload/**,minidumps/** true false false @@ -212,7 +191,6 @@ ${FAILED_TESTS} absolute 3 - xterm diff --git a/jenkins-master/jobs/mozilla-central_functional/config.xml b/jenkins-master/jobs/mozilla-central_functional/config.xml index 92a9cf5d..4fe2aa39 100644 --- a/jenkins-master/jobs/mozilla-central_functional/config.xml +++ b/jenkins-master/jobs/mozilla-central_functional/config.xml @@ -48,32 +48,9 @@ - - 2 - - - https://github.com/mozilla/firefox-ui-tests.git - - - - - */mozilla-central - - - false - - - - mozilla-central - - - true - - - - firefox-ui-tests - - + + get_mozharness + Successful true false @@ -86,20 +63,22 @@ scripts - + + true + python -u submission.py --test-type=functional --build-state=running --repository=mozilla-central --revision=$REVISION --locale=$LOCALE treeherder_venv false - python runtests.py --type=functional --installer-url=$INSTALLER_URL + python -u runtests.py --test-type=functional --repository=mozilla-central --installer-url=$INSTALLER_URL false - build/upload/*.* + build/upload/**,minidumps/** true false false @@ -195,7 +174,6 @@ ${FAILED_TESTS} absolute 3 - xterm diff --git a/jenkins-master/jobs/mozilla-central_update/config.xml b/jenkins-master/jobs/mozilla-central_update/config.xml index dcdeb1ad..260a520d 100644 --- a/jenkins-master/jobs/mozilla-central_update/config.xml +++ b/jenkins-master/jobs/mozilla-central_update/config.xml @@ -65,32 +65,9 @@ NSPR_LOG_FILE=http.log - - 2 - - - https://github.com/mozilla/firefox-ui-tests.git - - - - - */mozilla-central - - - false - - - - master - - - true - - - - firefox-ui-tests - - + + get_mozharness + Successful true false @@ -103,20 +80,22 @@ NSPR_LOG_FILE=http.log scripts - + + true + python -u submission.py --test-type=update --build-state=running --repository=mozilla-central --revision=$REVISION --locale=$LOCALE --update-number=$UPDATE_NUMBER treeherder_venv false - python runtests.py --type=update --installer-url=$INSTALLER_URL --update-channel=$CHANNEL --update-target-build-id=$TARGET_BUILD_ID + python -u runtests.py --test-type=update --repository=mozilla-central --installer-url=$INSTALLER_URL --update-channel=$CHANNEL --update-target-build-id=$TARGET_BUILD_ID false - build/upload/*.* + build/http.log,build/upload/**,minidumps/** true false false @@ -212,7 +191,6 @@ ${FAILED_TESTS} absolute 3 - xterm diff --git a/jenkins-master/jobs/release-mozilla-beta_functional/config.xml b/jenkins-master/jobs/release-mozilla-beta_functional/config.xml index 144b5033..130a1616 100644 --- a/jenkins-master/jobs/release-mozilla-beta_functional/config.xml +++ b/jenkins-master/jobs/release-mozilla-beta_functional/config.xml @@ -48,32 +48,9 @@ - - 2 - - - https://github.com/mozilla/firefox-ui-tests.git - - - - - */mozilla-beta - - - false - - - - mozilla-beta - - - true - - - - firefox-ui-tests - - + + get_mozharness + Successful true false @@ -86,20 +63,22 @@ scripts - + + true + python -u submission.py --test-type=functional --build-state=running --repository=mozilla-beta --revision=$REVISION --locale=$LOCALE treeherder_venv false - python runtests.py --type=functional --installer-url=$INSTALLER_URL + python -u runtests.py --test-type=functional --repository=mozilla-beta --installer-url=$INSTALLER_URL false - build/upload/*.* + build/upload/**,minidumps/** true false false @@ -195,7 +174,6 @@ ${FAILED_TESTS} absolute 3 - xterm diff --git a/jenkins-master/jobs/release-mozilla-esr38_functional/config.xml b/jenkins-master/jobs/release-mozilla-esr38_functional/config.xml index c45fd3c5..1f37814f 100644 --- a/jenkins-master/jobs/release-mozilla-esr38_functional/config.xml +++ b/jenkins-master/jobs/release-mozilla-esr38_functional/config.xml @@ -48,32 +48,9 @@ - - 2 - - - https://github.com/mozilla/firefox-ui-tests.git - - - - - */mozilla-esr38 - - - false - - - - mozilla-esr38 - - - true - - - - firefox-ui-tests - - + + get_mozharness + Successful true false @@ -86,20 +63,22 @@ scripts - + + true + python -u submission.py --test-type=functional --build-state=running --repository=mozilla-esr38 --revision=$REVISION --locale=$LOCALE treeherder_venv false - python runtests.py --type=functional --installer-url=$INSTALLER_URL + python -u runtests.py --test-type=functional --repository=mozilla-esr38 --installer-url=$INSTALLER_URL false - build/upload/*.* + build/upload/**,minidumps/** true false false @@ -195,7 +174,6 @@ ${FAILED_TESTS} absolute 3 - xterm diff --git a/jenkins-master/jobs/release-mozilla-release_functional/config.xml b/jenkins-master/jobs/release-mozilla-release_functional/config.xml index c0319bd4..a25f6901 100644 --- a/jenkins-master/jobs/release-mozilla-release_functional/config.xml +++ b/jenkins-master/jobs/release-mozilla-release_functional/config.xml @@ -48,32 +48,9 @@ - - 2 - - - https://github.com/mozilla/firefox-ui-tests.git - - - - - */mozilla-release - - - false - - - - mozilla-release - - - true - - - - firefox-ui-tests - - + + get_mozharness + Successful true false @@ -86,20 +63,22 @@ scripts - + + true + python -u submission.py --test-type=functional --build-state=running --repository=mozilla-release --revision=$REVISION --locale=$LOCALE treeherder_venv false - python runtests.py --type=functional --installer-url=$INSTALLER_URL + python -u runtests.py --test-type=functional --repository=mozilla-release --installer-url=$INSTALLER_URL false - build/upload/*.* + build/upload/**,minidumps/** true false false @@ -195,7 +174,6 @@ ${FAILED_TESTS} absolute 3 - xterm diff --git a/jenkins-master/jobs/scripts/workspace/buildbot.py b/jenkins-master/jobs/scripts/workspace/buildbot.py new file mode 100644 index 00000000..58ec5de7 --- /dev/null +++ b/jenkins-master/jobs/scripts/workspace/buildbot.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +class Enum(tuple): + __getattr__ = tuple.index + + +# Build exit code as defined by buildbot +# From: https://github.com/mozilla/treeherder/blob/master/treeherder/etl/buildbot.py#L3 +BuildExitCode = Enum([ + 'success', # 0 + 'testfailed', # 1 + 'busted', # 2 + 'skipped', # 3 + 'exception', # 4 + 'retry', # 5 + 'usercancel' # 6 +]) diff --git a/jenkins-master/jobs/scripts/workspace/config.py b/jenkins-master/jobs/scripts/workspace/config.py index 38f13858..2bb9e872 100644 --- a/jenkins-master/jobs/scripts/workspace/config.py +++ b/jenkins-master/jobs/scripts/workspace/config.py @@ -10,32 +10,38 @@ config = { 'test_types': { 'functional': { + 'harness_config': os.path.join('firefox_ui_tests', 'qa_jenkins.py'), + 'harness_script': os.path.join('firefox_ui_tests', 'functional.py'), 'treeherder': { 'group_name': 'Firefox UI Tests - functional', 'group_symbol': 'Ff', 'job_name': 'Firefox UI Tests - functional ({locale})', 'job_symbol': '{locale}', 'tier': 3, + 'artifacts': { + 'log_info.log': os.path.join(here, 'build', 'upload', 'logs', 'log_info.log'), + 'report.html': os.path.join(here, 'build', 'upload', 'reports', 'report.html'), + }, + 'log_reference': 'log_info.log', }, - 'logs': { - 'gecko.log': os.path.join(here, 'upload', 'logs', 'gecko.log'), - 'tbpl.log': os.path.join(here, 'upload', 'logs', 'tbpl.log'), - } }, 'update': { + 'harness_config': os.path.join('firefox_ui_tests', 'qa_jenkins.py'), + 'harness_script': os.path.join('firefox_ui_tests', 'update.py'), 'treeherder': { 'group_name': 'Firefox UI Tests - update', 'group_symbol': 'Fu', 'job_name': 'Firefox UI Tests - update ({locale}-{update_number})', 'job_symbol': '{locale}-{update_number}', 'tier': 3, + 'artifacts': { + 'log_info.log': os.path.join(here, 'build', 'upload', 'logs', 'log_info.log'), + 'report.html': os.path.join(here, 'build', 'upload', 'reports', 'report.html'), + # TODO: Bug 1210753: Move generation of log as option to mozharness + 'http.log': os.path.join(here, 'build', 'http.log'), + }, + 'log_reference': 'log_info.log', }, - 'logs': { - # Currently we don't have a gecko.log because we log to the console (bug 1174766) - # 'gecko': os.path.join(here, 'build', 'upload', 'logs', 'gecko.log'), - 'http.log': os.path.join(here, 'upload', 'logs', 'http.log'), - 'tbpl.log': os.path.join(here, 'upload', 'logs', 'tbpl.log'), - } }, }, } diff --git a/jenkins-master/jobs/scripts/workspace/runtests.py b/jenkins-master/jobs/scripts/workspace/runtests.py index 3b2d4c7b..78768d28 100755 --- a/jenkins-master/jobs/scripts/workspace/runtests.py +++ b/jenkins-master/jobs/scripts/workspace/runtests.py @@ -5,108 +5,136 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import argparse +import copy import os -import shutil import subprocess import sys -here = os.path.dirname(os.path.abspath(__file__)) -venv_path = os.path.join(here, 'tests_venv') +from buildbot import BuildExitCode +from config import config +from jenkins import JenkinsDefaultValueAction -# Create the test environment and activate it -# TODO remove once we make use of the mozharness script -import environment -command = ['python', os.path.join(here, 'firefox-ui-tests', 'create_venv.py'), - '--strict', '--with-optional', venv_path] -print('Run command to create virtual environment for Firefox UI Tests: %s' % command) -subprocess.check_call(command) -environment.activate(venv_path) +here = os.path.dirname(os.path.abspath(__file__)) -# Can only be imported after the environment has been activated -import firefox_ui_tests -import firefox_puppeteer +# Purge unwanted environment variables like credentials +ENV_VARS_TO_PURGE = [ + 'AWS_BUCKET', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', + 'TREEHERDER_URL', 'TREEHERDER_CLIENT_ID', 'TREEHERDER_SECRET', +] -from mozdownload import FactoryScraper -from config import config -from jenkins import JenkinsDefaultValueAction +class BaseRunner(object): + """Base class for different kinds of test runners.""" + + def __init__(self, settings, **kwargs): + """Creates new instance of the base runner class. + :param settings: Settings for the Runner as retrieved from the config file. + :param installer_url: URL of the build to download. + :param repository: Name of the repository the build has been built from. + """ + self.installer_url = kwargs['installer_url'] + self.repository = kwargs['repository'] + self.settings = settings -class Runner(object): - - def run_tests(self, args): - settings = config['test_types'][args.type] - - # In case the log folder does not exist yet we have to create it because - # otherwise Marionette will fail for e.g. gecko.log (see bug 1211666) - if settings['logs'].get('gecko.log'): - try: - os.makedirs(os.path.dirname(settings['logs']['gecko.log'])) - except OSError: - print('Failed to create log folder for {}'.format(settings['logs']['gecko.log'])) - - print('Downloading the installer: {}'.format(args.installer_url)) - scraper = FactoryScraper('direct', - url=args.installer_url, - retry_attempts=5, - retry_delay=30, - ) - installer_path = scraper.download() - - command = [ - 'firefox-ui-update' if args.type == 'update' else 'firefox-ui-tests', - '--installer', installer_path, - '--workspace', os.getcwd(), - '--log-tbpl', settings['logs']['tbpl.log'], + def query_args(self): + """Returns all required and optional command line arguments.""" + return [ + '--cfg', self.settings['harness_config'], + '--installer-url', self.installer_url, ] - if args.type == 'update': - # Enable Gecko log to the console because the file would be overwritten - # by the second update test - command.extend(['--gecko-log', '-']) + def run(self): + """Executes the tests. - if args.update_channel: - command.extend(['--update-channel', args.update_channel]) - if args.update_target_version: - command.extend(['--update-target-version', args.update_target_version]) - if args.update_target_build_id: - command.extend(['--update-target-buildid', args.update_target_build_id]) + It also ensures to save the return code of the subprocess to a file, which + is used by the submission script to check the build status. - elif args.type == 'functional': - command.extend(['--gecko-log', settings['logs']['gecko.log']]) + """ + # Purge unwanted environment variables (Treeherder and AWS credentials) + env = copy.copy(os.environ) + for var in ENV_VARS_TO_PURGE: + env.pop(var, None) - manifests = [firefox_puppeteer.manifest, firefox_ui_tests.manifest_functional] - command.extend(manifests) + # Set environment variable to let mozcrash save a copy of the minidump files + env.update({'MINIDUMP_SAVE_PATH': os.path.join(here, 'minidumps')}) - retval = 0 + command = [sys.executable, + os.path.join('mozharness', 'scripts', self.settings['harness_script'])] + command.extend(self.query_args()) print('Calling command to execute tests: {}'.format(command)) - retval = subprocess.call(command) + return subprocess.call(command, env=env) - # Save exit code into file for further processing in report submission - try: - with file('retval.txt', 'w') as f: - f.write(str(retval)) - except OSError as e: - print('Failed to save return value: {}'.format(e)) - # Delete http.log if tests were passing - if not retval and settings['logs'].get('http.log'): - shutil.rmtree(settings['logs']['http.log'], ignore_errors=True) +class FunctionalRunner(BaseRunner): + """Runner class for functional ui tests.""" + def __init__(self, *args, **kwargs): + """Creates new instance of the functional runner class.""" + BaseRunner.__init__(self, *args, **kwargs) -def main(): + if not kwargs['repository']: + raise TypeError('Repository information have not been specified.') + + def query_args(self): + """Returns additional required and optional command line arguments.""" + args = BaseRunner.query_args(self) + args.extend(['--firefox-ui-branch', self.repository]) + + return args + + +class UpdateRunner(BaseRunner): + """Runner class for update tests.""" + + def __init__(self, *args, **kwargs): + """Creates new instance of the update runner class. + + :param update_channel: The channel which is checked for available updates. + :param update_target_version: The expected target version of the application. + :param update_target_build_id: The expected target build id of the application. + + """ + BaseRunner.__init__(self, *args, **kwargs) + + if not kwargs['repository']: + raise TypeError('Repository information have not been specified.') + + self.channel = kwargs['update_channel'] + self.target_version = kwargs['update_target_version'] + self.target_build_id = kwargs['update_target_build_id'] + + def query_args(self): + """Returns all required and optional command line arguments.""" + args = BaseRunner.query_args(self) + + args.extend(['--firefox-ui-branch', self.repository]) + + if self.channel: + args.extend(['--update-channel', self.channel]) + if self.target_version: + args.extend(['--update-target-version', self.target_version]) + if self.target_build_id: + args.extend(['--update-target-build-id', self.target_build_id]) + + return args + + +def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--type', + parser.add_argument('--test-type', required=True, choices=config['test_types'].keys(), help='The type of tests to execute') parser.add_argument('--installer-url', required=True, help='The URL of the build installer.') + parser.add_argument('--repository', + help='The repository name the build was created from.') - update_group = parser.add_argument_group('update', 'Update test specific options') + update_group = parser.add_argument_group('Update Tests', 'Update test specific options') update_group.add_argument('--update-channel', action=JenkinsDefaultValueAction, help='The update channel to use for the update test') @@ -116,13 +144,32 @@ def main(): update_group.add_argument('--update-target-version', action=JenkinsDefaultValueAction, help='The expected version of the updated build') - args = parser.parse_args() + return parser.parse_args() + + +def main(): + # Default exit code to `busted` state + retval = BuildExitCode.busted + + # Maps the CLI test types to runner classes + runner_map = { + 'functional': FunctionalRunner, + 'update': UpdateRunner, + } try: - runner = Runner() - runner.run_tests(args) - except subprocess.CalledProcessError as e: - sys.exit(e) + kwargs = vars(parse_args()) + settings = config['test_types'].get(kwargs['test_type']) + runner = runner_map[kwargs['test_type']](settings, **kwargs) + retval = runner.run() + + finally: + # Save exit code into file for further processing in report submission + try: + with file('retval.txt', 'w') as f: + f.write(str(retval)) + except OSError as e: + print('Failed to save return value: {}'.format(e)) if __name__ == '__main__': main() diff --git a/jenkins-master/jobs/scripts/workspace/submission.py b/jenkins-master/jobs/scripts/workspace/submission.py index 06282418..cbc3ffcd 100755 --- a/jenkins-master/jobs/scripts/workspace/submission.py +++ b/jenkins-master/jobs/scripts/workspace/submission.py @@ -6,12 +6,12 @@ import argparse import os -import re import socket import time from urlparse import urljoin, urlparse import uuid +from buildbot import BuildExitCode from config import config from jenkins import JenkinsDefaultValueAction @@ -24,60 +24,21 @@ BUILD_STATES = ['running', 'completed'] -class FirefoxUIResultParser(object): - - BUSTED = 'busted' - SUCCESS = 'success' - TESTFAILED = 'testfailed' - UNKNOWN = 'unknown' - - def __init__(self, retval, log_file): - self.retval = retval - self.log_file = log_file - self.failure_re = re.compile(r'(^TEST-UNEXPECTED-FAIL|TEST-UNEXPECTED-ERROR)|' - r'(.*CRASH: )|' - r'(Crash reason: )') - self.failures = [] - self.parse() - - def parse(self): - try: - with open(self.log_file, 'r') as f: - for line in f.readlines(): - if self.failure_re.match(line): - self.failures.append(line) - except IOError: - print('Cannot parse failures due to missing log file: {}'.format(self.log_file)) - - @property - def status(self): - status = self.UNKNOWN - - # retval.txt was not written - so most likely an abort - if self.retval is None or (self.retval and not self.failures): - status = self.BUSTED - - elif not self.failures: - status = self.SUCCESS - - elif self.failures: - status = self.TESTFAILED - - return status - - def failures_as_json(self): - failures = {'all_errors': [], 'errors_truncated': True} - - for failure in self.failures: - failures['all_errors'].append({'line': failure, 'linenumber': 1}) - - return failures - - class Submission(object): + """Class for submitting reports to Treeherder.""" def __init__(self, repository, revision, settings, treeherder_url=None, treeherder_client_id=None, treeherder_secret=None): + """Creates new instance of the submission class. + + :param repository: Name of the repository the build has been built from. + :param revision: Changeset of the repository the build has been built from. + :param settings: Settings for the Treeherder job as retrieved from the config file. + :param treeherder_url: URL of the Treeherder instance. + :param treeherder_client_id: The client ID necessary for the Hawk authentication. + :param treeherder_secret: The secret key necessary for the Hawk authentication. + + """ self.repository = repository self.revision = revision self.settings = settings @@ -92,6 +53,7 @@ def __init__(self, repository, revision, settings, raise ValueError('The client_id and secret for Treeherder must be set.') def _get_treeherder_platform(self): + """Returns the Treeherder equivalent platform identifier of the current platform.""" platform = None info = mozinfo.info @@ -112,6 +74,13 @@ def _get_treeherder_platform(self): return platform def create_job(self, guid, **kwargs): + """Creates a new instance of a Treeherder job for submission. + + :param guid: Unique identifier of the job. + :param kwargs: Dictionary of necessary values to build the job details. The + properties correlate to the placeholders in config.py. + + """ job = TreeherderJob() job.add_job_guid(guid) @@ -147,6 +116,7 @@ def create_job(self, guid, **kwargs): return job def retrieve_revision_hash(self): + """Retrieves the unique hash for the current revision.""" if not self.url: raise ValueError('URL for Treeherder is missing.') @@ -166,9 +136,12 @@ def retrieve_revision_hash(self): return response.json()['results'][0]['revision_hash'] - def submit(self, job, logs=None): - logs = logs or [] + def submit(self, job): + """Submit the job to treeherder. + + :param job: Treeherder job instance to use for submission. + """ job.add_submit_timestamp(int(time.time())) # We can only submit job info once, so it has to be done in completed @@ -189,30 +162,34 @@ def submit(self, job, logs=None): JOB_FRAGMENT.format(repository=self.repository, revision=self.revision)))) def submit_running_job(self, job): - job.add_state('running') + """Submit job as state running. + + :param job: Treeherder job instance to use for submission. + """ + job.add_state('running') self.submit(job) def submit_completed_job(self, job, retval, uploaded_logs): - """Update the status of a job to completed. - """ - # Parse TBPL log for failures - tbpl_log = uploaded_logs.get('tbpl.log', {}) + """Submit job as state completed. - parser = FirefoxUIResultParser(retval, self.settings['logs']['tbpl.log']) - job.add_result(parser.status) + :param job: Treeherder job instance to use for submission. + :param retval: Return value of the build process to determine build state. + :param uploaded_logs: List of uploaded logs to reference in the job. - if os.path.isfile(self.settings['logs']['tbpl.log']): - job.add_log_reference(self.settings['logs']['tbpl.log'], - tbpl_log.get('url'), - parse_status='parsed') - job.add_artifact(name='text_log_summary', artifact_type='json', - blob={'step_data': parser.failures_as_json(), - 'logurl': tbpl_log.get('url')}) + """ + job.add_state('completed') + job.add_result(BuildExitCode[retval]) + job.add_end_timestamp(int(time.time())) + + # Add reference to the log which will be parsed by Treeherder + log_reference = uploaded_logs.get(self.settings['treeherder']['log_reference']) + if log_reference: + job.add_log_reference(name='buildbot_text', url=log_reference.get('url')) # If the Jenkins BUILD_URL environment variable is present add it as artifact # TODO: Figure out how to send it already for running state. If I do so right - # now the logs won't be uploaded. + # now the report will not be submitted. if os.environ.get('BUILD_URL'): self._job_details.append({ 'title': 'Inspect Jenkins Build (VPN required)', @@ -230,16 +207,21 @@ def submit_completed_job(self, job, retval, uploaded_logs): 'url': uploaded_logs[log]['url'], }) - job.add_state('completed') - job.add_end_timestamp(int(time.time())) - self.submit(job) def upload_log_files(guid, logs, bucket_name=None, access_key_id=None, access_secret_key=None): - # If AWS credentials are given we want to upload the log files and attach - # as artifacts to the treeherder job results + """Upload all specified logs to Amazon S3. + + :param guid: Unique ID which is used as subfolder name for all log files. + :param logs: List of log files to upload. + :param bucket_name: Name of the S3 bucket. + :param access_key_id: Client ID used for authentication. + :param access_secret_key: Secret key for authentication. + + """ + # If no AWS credentials are given we don't upload anything. if not bucket_name: print('No AWS Bucket name specified - skipping upload of artifacts.') return {} @@ -346,11 +328,12 @@ def parse_args(): # State 'running' if kwargs['build_state'] == BUILD_STATES[0]: job_guid = str(uuid.uuid4()) - job = th.create_job(job_guid, **kwargs) - th.submit_running_job(job) with file('job_guid.txt', 'w') as f: f.write(job_guid) + job = th.create_job(job_guid, **kwargs) + th.submit_running_job(job) + # State 'completed' elif kwargs['build_state'] == BUILD_STATES[1]: # Read in job guid to update the report @@ -364,11 +347,12 @@ def parse_args(): try: with file('retval.txt', 'r') as f: retval = int(f.read()) - except IOError: - retval = None + except: + # Default reval to `busted` state + retval = BuildExitCode.busted job = th.create_job(job_guid, **kwargs) - uploaded_logs = upload_log_files(job_guid, settings['logs'], + uploaded_logs = upload_log_files(job_guid, settings['treeherder']['artifacts'], bucket_name=kwargs.get('aws_bucket'), access_key_id=kwargs.get('aws_key'), access_secret_key=kwargs.get('aws_secret'),)