Browse files

Use pulsebuildmonitor to handle build notifications (#176)

  • Loading branch information...
1 parent 0a4bb64 commit b02a5759038e4d213095c76b0366b23706da0b69 @whimboo whimboo committed Apr 4, 2013
View
2 README.md
@@ -6,7 +6,7 @@ Before you can start the system the following commands have to be performed:
git clone git://github.com/whimboo/mozmill-ci.git
cd mozmill-ci
- ./setup/configure.sh
+ ./setup.sh
## Startup
The two components (Pulse consumer and Jenkins master) have to be started separately in two different terminals. As first step we have to setup the Jenkins master:
View
21 config/production/daily.json
@@ -1,24 +1,26 @@
{
"pulse": {
"applabel": "qa-auto@mozilla.com|mozmill_daily",
- "routing_key_regex": "build\\..+(-l10n)?-nightly\\.\\d+\\.finished",
- "platforms": [
- "macosx",
- "macosx64"
- ],
"branches": [
"mozilla-central",
"mozilla-aurora",
"mozilla-esr17"
],
- "products": [
- "firefox"
- ],
"locales": [
"de",
"en-US",
"fr",
"it"
+ ],
+ "platforms": [
+ "macosx",
+ "macosx64"
+ ],
+ "products": [
+ "firefox"
+ ],
+ "tags": [
+ "nightly"
]
},
"jenkins": {
@@ -36,8 +38,7 @@
"value": "daily"
},
"LOCALE": {
- "key": "locale",
- "default": "en-US"
+ "key": "locale"
},
"PLATFORM": {
"key": "platform",
View
27 config/production/l10n.json
@@ -1,20 +1,21 @@
{
"pulse": {
"applabel": "qa-auto@mozilla.com|mozmill_l10n",
- "routing_key_regex": "build\\..+-l10n-dep\\.\\d+\\.finished",
- "platforms": [
- "linux",
- "macosx",
- "macosx64",
- "win32"
- ],
"branches": [
"mozilla-aurora"
],
+ "locales": [
+ ],
+ "platforms": [
+ "macosx",
+ "macosx64"
+ ],
"products": [
"firefox"
],
- "locales": [
+ "tags": [
+ "dep",
+ "l10n"
]
},
"jenkins": {
@@ -32,8 +33,7 @@
"value": "tinderbox"
},
"LOCALE": {
- "key": "locale",
- "default": "en-US"
+ "key": "locale"
},
"PLATFORM": {
"key": "platform",
@@ -52,13 +52,6 @@
"platforms": {
"mac": [
"mac && 10.7 && 64bit"
- ],
- "win32": [
- "windows && xp && 32bit",
- "windows && 7 && 32bit"
- ],
- "linux": [
- "linux && ubuntu && 32bit"
]
}
}
View
20 config/production/release.json
@@ -1,24 +1,25 @@
{
"pulse": {
"applabel": "qa-auto@mozilla.com|mozmill_release",
- "routing_key_regex": "build\\..+_build\\.\\d+\\.finished",
- "platforms": [
- "macosx",
- "macosx64"
- ],
"branches": [
"release-mozilla-beta",
"release-mozilla-release",
"release-mozilla-esr17"
],
- "products": [
- "firefox"
- ],
"locales": [
"de",
"en-US",
"fr",
"it"
+ ],
+ "platforms": [
+ "macosx",
+ "macosx64"
+ ],
+ "products": [
+ "firefox"
+ ],
+ "tags": [
]
},
"jenkins": {
@@ -36,8 +37,7 @@
"value": "candidate"
},
"LOCALE": {
- "key": "locale",
- "default": "en-US"
+ "key": "locale"
},
"PLATFORM": {
"key": "platform",
View
23 config/production_new/daily.json
@@ -1,25 +1,27 @@
{
"pulse": {
"applabel": "qa-auto@mozilla.com|mozmill_daily",
- "routing_key_regex": "build\\..+(-l10n)?-nightly\\.\\d+\\.finished",
- "platforms": [
- "linux",
- "linux64",
- "win32"
- ],
"branches": [
"mozilla-central",
"mozilla-aurora",
"mozilla-esr17"
],
- "products": [
- "firefox"
- ],
"locales": [
"de",
"en-US",
"fr",
"it"
+ ],
+ "platforms": [
+ "linux",
+ "linux64",
+ "win32"
+ ],
+ "products": [
+ "firefox"
+ ],
+ "tags": [
+ "nightly"
]
},
"jenkins": {
@@ -37,8 +39,7 @@
"value": "daily"
},
"LOCALE": {
- "key": "locale",
- "default": "en-US"
+ "key": "locale"
},
"PLATFORM": {
"key": "platform",
View
16 config/production_new/l10n.json
@@ -1,18 +1,21 @@
{
"pulse": {
"applabel": "qa-auto@mozilla.com|mozmill_l10n",
- "routing_key_regex": "build\\..+-l10n-dep\\.\\d+\\.finished",
+ "branches": [
+ "mozilla-aurora"
+ ],
+ "locales": [
+ ],
"platforms": [
"linux",
"win32"
],
- "branches": [
- "mozilla-aurora"
- ],
"products": [
"firefox"
],
- "locales": [
+ "tags": [
+ "dep",
+ "l10n"
]
},
"jenkins": {
@@ -30,8 +33,7 @@
"value": "tinderbox"
},
"LOCALE": {
- "key": "locale",
- "default": "en-US"
+ "key": "locale"
},
"PLATFORM": {
"key": "platform",
View
22 config/production_new/release.json
@@ -1,25 +1,26 @@
{
"pulse": {
"applabel": "qa-auto@mozilla.com|mozmill_release",
- "routing_key_regex": "build\\..+_build\\.\\d+\\.finished",
- "platforms": [
- "linux",
- "linux64",
- "win32"
- ],
"branches": [
"release-mozilla-beta",
"release-mozilla-release",
"release-mozilla-esr17"
],
- "products": [
- "firefox"
- ],
"locales": [
"de",
"en-US",
"fr",
"it"
+ ],
+ "platforms": [
+ "linux",
+ "linux64",
+ "win32"
+ ],
+ "products": [
+ "firefox"
+ ],
+ "tags": [
]
},
"jenkins": {
@@ -37,8 +38,7 @@
"value": "candidate"
},
"LOCALE": {
- "key": "locale",
- "default": "en-US"
+ "key": "locale"
},
"PLATFORM": {
"key": "platform",
View
25 config/staging/daily.json
@@ -1,24 +1,27 @@
{
"pulse": {
"applabel": "qa-auto@mozilla.com|mozmill_daily",
- "routing_key_regex": "build\\..+(-l10n)?-nightly\\.\\d+\\.finished",
+ "branches": [
+ "mozilla-central",
+ "mozilla-aurora",
+ "mozilla-beta",
+ "mozilla-esr17"
+ ],
+ "locales": [
+ "en-US"
+ ],
"platforms": [
"linux",
"linux64",
"macosx",
"macosx64",
"win32"
],
- "branches": [
- "mozilla-central",
- "mozilla-aurora",
- "mozilla-esr17"
- ],
"products": [
"firefox"
],
- "locales": [
- "en-US"
+ "tags": [
+ "nightly"
]
},
"jenkins": {
@@ -36,8 +39,7 @@
"value": "daily"
},
"LOCALE": {
- "key": "locale",
- "default": "en-US"
+ "key": "locale"
},
"PLATFORM": {
"key": "platform",
@@ -102,6 +104,9 @@
],
"linux": [
"linux && ubuntu && 32bit"
+ ],
+ "linux64": [
+ "linux && ubuntu && 12.04 && 64bit"
]
}
},
View
16 config/staging/release.json
@@ -1,24 +1,25 @@
{
"pulse": {
"applabel": "qa-auto@mozilla.com|mozmill_release",
- "routing_key_regex": "build\\..+_build\\.\\d+\\.finished",
+ "branches": [
+ "release-mozilla-beta",
+ "release-mozilla-release",
+ "release-mozilla-esr17"
+ ],
"platforms": [
"linux",
"linux64",
"macosx",
"macosx64",
"win32"
],
- "branches": [
- "release-mozilla-beta",
- "release-mozilla-release",
- "release-mozilla-esr17"
- ],
"products": [
"firefox"
],
"locales": [
"en-US"
+ ],
+ "tags": [
]
},
"jenkins": {
@@ -36,8 +37,7 @@
"value": "candidate"
},
"LOCALE": {
- "key": "locale",
- "default": "en-US"
+ "key": "locale"
},
"PLATFORM": {
"key": "platform",
View
254 pulse.py
@@ -9,19 +9,15 @@
import logging
import optparse
import os
-import re
import socket
-import sys
-from time import sleep
-import traceback
import jenkins
-from mozillapulse import consumers
+from pulsebuildmonitor import start_pulse_monitor
class NotFoundException(Exception):
-
"""Exception for a resource not being found (e.g. no logs)"""
+
def __init__(self, message, location):
self.location = location
Exception.__init__(self, ': '.join([message, location]))
@@ -32,18 +28,17 @@ class JSONFile:
def __init__(self, filename):
self.filename = os.path.abspath(filename)
-
def read(self):
if not os.path.isfile(self.filename):
- raise NotFoundException('Specified file cannot be found.', self.filename)
+ raise NotFoundException('Specified file cannot be found.',
+ self.filename)
try:
f = open(self.filename, 'r')
return json.loads(f.read())
finally:
f.close()
-
def write(self, data):
folder = os.path.dirname(self.filename)
if not os.path.exists(folder):
@@ -59,67 +54,65 @@ def write(self, data):
class Automation:
def __init__(self, config, debug, log_folder, logger,
- message=None, show_properties=False):
- self.timeout = 10
+ message=None, display_only=False):
self.config = config
self.debug = debug
self.log_folder = log_folder
self.logger = logger
- self.show_properties = show_properties
+ self.display_only = display_only
self.test_message = message
self.jenkins = jenkins.Jenkins(self.config['jenkins']['url'],
self.config['jenkins']['username'],
self.config['jenkins']['password'])
- # Make the consumer dependent to the host to prevent queue corruption by
- # other machines we are using the same queue name
- applabel = '%s|%s' % (self.config['pulse']['applabel'], socket.getfqdn())
-
# Whenever only a single message has to be sent we can return immediately
if self.test_message:
data = JSONFile(self.test_message).read()
- self.on_build(data, None)
+ self.on_build(data)
return
- while True:
- try:
- # Initialize Pulse consumer with a non-durable view because we do not want
- # to queue up notifications if the consumer is not connected.
- pulse = consumers.BuildConsumer(applabel=applabel, durable=False)
- pulse.configure(callback=self.on_build,
- topic=['build.*.*.finished', 'heartbeat'])
-
- self.logger.info('Connecting to Mozilla Pulse as "%s"...', applabel)
- pulse.listen()
- except (KeyboardInterrupt, SystemExit):
- raise
- except Exception, e:
- # For now only log traceback. Later we will send it via email
- self.logger.exception('Pulse listener disconnected. ' \
- 'Trying to reconnect in %s seconds...',
- self.timeout)
- sleep(self.timeout)
+ # Make the consumer dependent to the host to prevent queue corruption by
+ # other machines which are using the same queue name
+ label = '%s|%s' % (self.config['pulse']['applabel'], socket.getfqdn())
+
+ self.monitor = start_pulse_monitor(buildCallback=self.on_build,
+ testCallback=None,
+ pulseCallback=self.on_debug if self.debug else None,
+ label=label,
+ trees=self.config['pulse']['branches'],
+ platforms=self.config['pulse']['platforms'],
+ products=self.config['pulse']['products'],
+ locales=self.config['pulse']['locales'],
+ buildtypes=None,
+ tests=None,
+ buildtags=self.config['pulse']['tags'],
+ logger=self.logger)
+ try:
+ while self.monitor.is_alive():
+ self.monitor.join(1.0)
+ except (KeyboardInterrupt, SystemExit):
+ self.logger.info('Shutting down Pulse listener')
def generate_job_parameters(self, testrun, node, platform, build_properties):
# Create parameter map from Pulse to Jenkins properties
map = self.config['testrun']['jenkins_parameter_map']
- parameter_map = copy.deepcopy(map['default']);
+ parameter_map = copy.deepcopy(map['default'])
if testrun in map:
for key in map[testrun]:
parameter_map[key] = map[testrun][key]
# Create parameters and fill in values as given by the map
- parameters = { }
+ parameters = {}
for entry in parameter_map:
value = None
if 'key' in parameter_map[entry]:
# A key means we have to retrieve a value from a dict
value = build_properties.get(parameter_map[entry]['key'],
- parameter_map[entry].get('default'));
+ parameter_map[entry].get('default'))
elif 'value' in parameter_map[entry]:
# A value means we have an hard-coded value
value = parameter_map[entry]['value']
@@ -139,150 +132,106 @@ def generate_job_parameters(self, testrun, node, platform, build_properties):
return parameters
-
def get_mozmill_environment_platform(self, platform):
- # Map to translate the platform to the mozmill environment platform
-
- ENVIRONMENT_PLATFORM_MAP = {
- 'linux': 'linux',
- 'linux64': 'linux',
- 'mac': 'mac',
- 'win32': 'windows',
- 'win64': 'windows'
- }
- return ENVIRONMENT_PLATFORM_MAP[platform];
+ # Map to translate the platform to the Mozmill environment platform
+ ENVIRONMENT_PLATFORM_MAP = {'linux': 'linux',
+ 'linux64': 'linux',
+ 'mac': 'mac',
+ 'win32': 'windows',
+ 'win64': 'windows'}
+ return ENVIRONMENT_PLATFORM_MAP[platform]
def get_platform_identifier(self, platform):
# Map to translate platform ids from Pulse to Mozmill / Firefox
PLATFORM_MAP = {'linux': 'linux',
- 'linux-debug': 'linux',
'linux64': 'linux64',
- 'linux64-debug': 'linux64',
'macosx': 'mac',
- 'macosx-debug': 'mac',
'macosx64': 'mac',
- 'macosx64-debug': 'mac',
'win32': 'win32',
- 'win32-debug': 'win32',
- 'win64': 'win64',
- 'win64-debug': 'win64'}
- return PLATFORM_MAP[platform];
+ 'win64': 'win64'}
+ return PLATFORM_MAP[platform]
- def preprocess_message(self, data, message):
- # Ensure that the message gets removed from the queue
- if message is not None:
- message.ack()
+ def on_build(self, data):
+ # From: http://hg.mozilla.org/build/buildbot/file/08b7c51d2962/master/buildbot/status/builder.py#l25
+ results = ['success', 'warnings', 'failure', 'skipped', 'exception', 'retry']
- # Create dictionary with properties of the build
- if data.get('payload') and data['payload'].get('build'):
- build_properties = data['payload']['build'].get('properties')
- properties = dict((k, v) for (k, v, source) in build_properties)
- properties['build_failed'] = (data['payload']['build']['results'] != 0)
- else:
- properties = dict()
+ log_data = {'BRANCH': data['tree'],
+ 'BUILD_ID': data['buildid'],
+ 'KEY': data['key'],
+ 'LOCALE': data['locale'],
+ 'PRODUCT': data['product'],
+ 'PLATFORM': data['platform'],
+ 'STATUS': results[data['status']],
+ 'TIMESTAMP': data['timestamp'],
+ 'VERSION': data['version']}
- return (data['_meta']['routing_key'], properties)
+ # Output build information to the console
+ self.logger.info('%(TIMESTAMP)s - %(PRODUCT)s %(VERSION)s (%(BUILD_ID)s, %(LOCALE)s, %(PLATFORM)s) [%(BRANCH)s]' % log_data)
+ # ... and store to disk
+ basename = '%(BUILD_ID)s_%(PRODUCT)s_%(LOCALE)s_%(PLATFORM)s_%(KEY)s.log' % log_data
+ filename = os.path.join(self.log_folder, data['tree'], basename)
- def log_notification(self, data, props):
- """Store the Pulse notification as log file on disk."""
try:
- branch = props.get('branch', 'None')
- routing_key = data['_meta']['routing_key']
-
- basename = '%(BUILD_ID)s_%(PRODUCT)s_%(LOCALE)s_%(PLATFORM)s_%(KEY)s.log' % {
- 'BUILD_ID': props.get('buildid'),
- 'PRODUCT': props.get('product', 'None'),
- 'LOCALE': props.get('locale', 'en-US'),
- 'PLATFORM': props.get('platform'),
- 'KEY': routing_key
- }
- filename = os.path.join(self.log_folder, branch, basename)
JSONFile(filename).write(data)
- except:
- self.logger.warning("JSON log file could not be written for %s." % routing_key)
-
-
- def on_build(self, data, message):
- (routing_key, props) = self.preprocess_message(data, message)
-
- if routing_key == 'heartbeat':
- return
+ except Exception, e:
+ self.logger.warning("Log file could not be written: %s." % str(e))
- # Cache often used properties
- branch = props.get('branch', 'None')
- locale = props.get('locale', 'en-US')
- platform = props.get('platform', 'None')
- product = props.get('product', 'None')
-
- # Displaying the properties will only work in manual mode and return immediately
- if self.test_message and self.show_properties:
+ # if `--display-only` option has been specified only print build information and return
+ if self.display_only:
self.logger.info("Build properties:")
- for property in props:
- self.logger.info("%20s:\t%s" % (property, props[property]))
- return
-
- # In debug mode save off all the notification messages
- if self.debug:
- self.log_notification(data, props)
-
- # Check if the routing key matches the expected regex
- pattern = re.compile(self.config['pulse']['routing_key_regex'], re.IGNORECASE)
- if not pattern.match(routing_key):
+ for prop in data:
+ self.logger.info("%20s:\t%s" % (prop, data[prop]))
return
# If the build process was broken we don't have to test this build
- if props['build_failed']:
- self.logger.info("Invalid build: %(PRODUCT)s %(VERSION)s %(PLATFORM)s %(LOCALE)s %(BUILDID)s %(PREV_BUILDID)s" % {
- 'PRODUCT': product,
- 'VERSION': props.get('appVersion'),
- 'PLATFORM': self.get_platform_identifier(platform),
- 'LOCALE': locale,
- 'BUILDID': props.get('buildid'),
- 'PREV_BUILDID': props.get('previous_buildid')
- })
+ if data['status'] != 0:
+ self.logger.info('Cancel processing of broken build: status=%s' % results[data['status']])
return
- # Output logging information for received notification
- self.logger.info("%s - Product: %s, Branch: %s, Platform: %s, Locale: %s" %
- (data['_meta']['sent'], product, branch, platform, locale))
-
- # If one of the expected values do not match we are not interested in the build
- valid_branch = not self.config['pulse']['branches'] or branch in self.config['pulse']['branches']
- valid_locale = not self.config['pulse']['locales'] or locale in self.config['pulse']['locales']
- valid_platform = not self.config['pulse']['platforms'] or platform in self.config['pulse']['platforms']
- valid_product = not self.config['pulse']['products'] or product in self.config['pulse']['products']
-
- if not (valid_product and valid_branch and valid_platform and valid_locale):
- return
-
- self.log_notification(data, props)
- self.logger.info("Trigger tests for %(PRODUCT)s %(VERSION)s %(PLATFORM)s %(LOCALE)s %(BUILDID)s %(PREV_BUILDID)s" % {
- 'PRODUCT': product,
- 'VERSION': props.get('appVersion'),
- 'PLATFORM': self.get_platform_identifier(platform),
- 'LOCALE': locale,
- 'BUILDID': props.get('buildid'),
- 'PREV_BUILDID': props.get('previous_buildid')
- })
-
- # Queue up testruns for the branch as given by config settings
- target_branch = self.config['testrun']['by_branch'][branch]
- target_platform = self.get_platform_identifier(platform)
+ # Queue up jobs for the branch as given by config settings
+ target_branch = self.config['testrun']['by_branch'][data['tree']]
+ target_platform = self.get_platform_identifier(data['platform'])
for testrun in target_branch['testruns']:
# TODO: Pretty bad hack, so make it configurable in the json config (#209)
# Do not run endurance tests for localized versions of Firefox
- if testrun in ['endurance'] and locale != 'en-US':
+ if testrun in ['endurance'] and data['locale'] != 'en-US':
continue
# Fire off a build for each supported platform
for node in target_branch['platforms'][target_platform]:
+ job = '%s_%s' % (data['tree'], testrun)
parameters = self.generate_job_parameters(testrun, node,
- target_platform, props)
- self.jenkins.build_job('%s_%s' % (branch, testrun), parameters)
+ target_platform, data)
+
+ self.logger.info('Triggering job "%s" on "%s"' % (job, node))
+ try:
+ self.jenkins.build_job(job, parameters)
+ except Exception, e:
+ # For now simply discard and continue.
+ # Later we might want to implement a queuing mechanism.
+ self.logger.error('Jenkins instance at "%s" cannot be reached: %s' % (
+ self.config['jenkins']['url'],
+ str(e)))
+
+ def on_debug(self, data):
+ """In debug mode save off all raw notifications"""
+
+ basename = '%(BUILD_ID)s_%(PRODUCT)s_%(LOCALE)s_%(PLATFORM)s_%(KEY)s.log' % {
+ 'BUILD_ID': data['payload']['buildid'],
+ 'PRODUCT': data['payload']['product'],
+ 'LOCALE': data['payload']['locale'],
+ 'PLATFORM': data['payload']['platform'],
+ 'KEY': data['payload']['key']}
+ filename = os.path.join('debug', data['payload']['tree'], basename)
+
+ try:
+ JSONFile(filename).write(data)
+ except Exception, e:
+ self.logger.warning("Debug file could not be written: %s." % str(e))
class DailyAutomation(Automation):
@@ -304,11 +253,11 @@ def main():
parser.add_option('--push-message',
dest='message',
help='Log file of a Pulse message to process for Jenkins')
- parser.add_option('--show-properties',
- dest='show_properties',
+ parser.add_option('--display-only',
+ dest='display_only',
action='store_true',
default=False,
- help='Show the properties of a build in the console.')
+ help='Only display build properties and don\'t trigger jobs.')
options, args = parser.parse_args()
if not len(args):
@@ -322,8 +271,7 @@ def main():
log_folder=options.log_folder,
logger=logger,
message=options.message,
- show_properties=options.show_properties)
-
+ display_only=options.display_only)
if __name__ == "__main__":
main()
View
2 setup/requirements.txt → requirements.txt
@@ -1,3 +1,3 @@
mercurial==2.2.1
python-jenkins==0.2
-MozillaPulse==0.61
+pulsebuildmonitor==0.70
View
7 setup/configure.sh → setup.sh
@@ -2,11 +2,10 @@
PYTHON_VERSION=$(python -c "import sys;print sys.version[:3]")
-BASE_DIR=$(dirname $(cd $(dirname $BASH_SOURCE); pwd))
+BASE_DIR=$(cd $(dirname $BASH_SOURCE); pwd)
ENV_DIR=$BASE_DIR/jenkins-env
-SETUP_DIR=$BASE_DIR/setup
-TMP_DIR=$SETUP_DIR/tmp
+TMP_DIR=$BASE_DIR/tmp
URL_VIRTUALENV=https://bitbucket.org/ianb/virtualenv/raw/1.5.2/virtualenv.py
@@ -28,7 +27,7 @@ if [ ! -n "${VIRTUAL_ENV:+1}" ]; then
fi
echo "Installing required Python modules"
-pip install -r $SETUP_DIR/requirements.txt
+pip install -r requirements.txt
echo "Deactivating the environment"
deactivate

0 comments on commit b02a575

Please sign in to comment.