master.cfg

# -*- python -*-
# ex: set syntax=python:

# Shorthand
c = BuildmasterConfig = {}

# most of the config is in an external file
import config
reload(config)
from config import *

####### BUILDSLAVES

import BuildSlaves
reload(BuildSlaves)
from BuildSlaves import SlavePasswords

# 'slavePortnum' defines the TCP port to listen on. This must match the value
# configured into the buildslaves (with their --master option)

c['slavePortnum'] = 9010

from buildbot.buildslave import BuildSlave
c['slaves'] = []
for platform, names in SLAVES.items():
    for name in names:
        c['slaves'].append(BuildSlave(name, SlavePasswords[platform], max_builds=1))

import buildbot.status.tinderbox
reload(buildbot.status.tinderbox)

from buildbot.scheduler import Nightly
from buildbot.status.tinderbox import TinderboxMailNotifier
from buildbot.steps.shell import WithProperties

import buildbotcustom.misc
import buildbotcustom.changes.hgpoller
import buildbotcustom.process.factory
import buildbotcustom.scheduler
reload(buildbotcustom.misc)
reload(buildbotcustom.changes.hgpoller)
reload(buildbotcustom.process.factory)
reload(buildbotcustom.scheduler)
from buildbotcustom.misc import isHgPollerTriggered
from buildbotcustom.changes.hgpoller import HgPoller, HgAllLocalesPoller
from buildbotcustom.process.factory import CCNightlyBuildFactory, \
  CCNightlyRepackFactory, CCUnittestBuildFactory, CodeCoverageFactory
from buildbotcustom.scheduler import MozScheduler

# l10n parallelization logic
import buildbotcustom.log
import buildbotcustom.l10n
reload(buildbotcustom.log)
reload(buildbotcustom.l10n)
from buildbotcustom.l10n import NightlyL10n, Scheduler as SchedulerL10n

c['change_source'] = []
c['schedulers'] = []
c['builders'] = []
c['status'] = []

for name in sorted(BRANCHES.keys()):
    # shorthand
    branch = BRANCHES[name]
    builders = []
    unittestBuilders = []
    nightlyBuilders = []
    weeklyBuilders = []
    # This dict provides a mapping between en-US nightly scheduler names
    # and l10n nightly scheduler names. It's filled out just below here.
    l10nNightlyBuilders = {}
    # generate a list of builders, nightly builders (names must be different)
    # for easy access
    for platform in branch['platforms'].keys():
        base_name =  branch['platforms'][platform]['base_name']
        builders.append('%s build' % base_name)
        # Skip l10n, unit tests and nightlies for debug builds
        if platform.find('debug') >= 0:
            continue

        builder = '%s nightly' % base_name
        nightlyBuilders.append(builder)
        if branch['enable_l10n'] and platform in ('linux','win32','macosx'):
            l10nNightlyBuilders[builder] = {} 
            l10nNightlyBuilders[builder]['tree'] = branch['l10n_tree']
            l10nNightlyBuilders[builder]['l10n_builder'] = \
                  '%s %s %s l10n' % (branch['brand_name'], name, platform)
            l10nNightlyBuilders[builder]['platform'] = platform
        if branch['enable_unittests'] and platform in ('linux','win32','macosx'):
            unittestBuilders.append('%s unit test' % base_name)
        if branch['enable_codecoverage'] and platform in ('linux',):
            weeklyBuilders.append('%s code coverage' % branch['platforms'][platform]['base_name'])

    # Currently, each branch goes to a different tree
    # If this changes in the future this may have to be
    # moved out of the loop
    c['status'].append(TinderboxMailNotifier(
        fromaddr="seamonkey.buildbot@build.mozilla.org",
        tree=branch['tinderbox_tree'],
        extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
        relayhost="mail.build.mozilla.org",
        builders=builders,
        logCompression="bzip2"
    ))
    # Separate notifier for nightlies, since they show the binaryurl (D link)
    c['status'].append(TinderboxMailNotifier(
        fromaddr="seamonkey.buildbot@build.mozilla.org",
        tree=branch['tinderbox_tree'],
        extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
        relayhost="mail.build.mozilla.org",
        builders=nightlyBuilders,
        logCompression="bzip2",
        binaryURL="http://ftp.mozilla.org/pub/mozilla.org/%s/nightly/latest-%s/" % (branch['product_name'], name)
    ))
    # Separate notifier for unittests, since they need to be run through
    # the unittest errorparser
    c['status'].append(TinderboxMailNotifier(
        fromaddr="seamonkey.buildbot@build.mozilla.org",
        tree=branch['tinderbox_tree'],
        extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
        relayhost="mail.build.mozilla.org",
        builders=unittestBuilders,
        logCompression="bzip2",
        errorparser="unittest"
    ))

    # Weekly builds (currently only code coverage) go to the SeaMonkey-Ports tree
    c['status'].append(TinderboxMailNotifier(
        fromaddr="seamonkey.buildbot@build.mozilla.org",
        tree="SeaMonkey-Ports",
        extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
        relayhost="mail.build.mozilla.org",
        builders=weeklyBuilders,
        logCompression="bzip2",
        errorparser="unittest"
    ))

    if branch['enable_l10n']:
        l10n_builders = []
        for b in l10nNightlyBuilders:
            l10n_builders.append(l10nNightlyBuilders[b]['l10n_builder'])
        # This notifies all l10n related build objects to Mozilla-l10n
        c['status'].append(TinderboxMailNotifier(
            fromaddr="seamonkey.buildbot@build.mozilla.org",
            tree="Mozilla-l10n",
            extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
            relayhost="mail.build.mozilla.org",
            logCompression="bzip2",
            builders=l10n_builders,
            binaryURL="http://ftp.mozilla.org/pub/mozilla.org/%s/nightly/latest-%s-l10n/" % (branch['product_name'], name)
        ))

        # We only want the builds from the specified builders
        # since their builds have a build property called "locale"
        c['status'].append(TinderboxMailNotifier(
            fromaddr="seamonkey.buildbot@build.mozilla.org",
            tree=WithProperties("Mozilla-l10n-%(locale)s"),
            extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
            relayhost="mail.build.mozilla.org",
            logCompression="bzip2",
            builders=l10n_builders,
            binaryURL="http://ftp.mozilla.org/pub/mozilla.org/%s/nightly/latest-%s-l10n/" % (branch['product_name'], name)
        ))

    # change sources
    c['change_source'].append(HgPoller(
        hgURL=HGURL,
        branch=branch['repo_path'],
        pushlogUrlOverride='%s%s/pushlog' % (HGURL, branch['repo_path']),
        pollInterval=1*60
    ))
    c['change_source'].append(HgPoller(
        hgURL=HGURL,
        branch=branch['repo_path'],
        pushlogUrlOverride='%s%s/pushlog' % (HGURL, branch['mozilla_repo_path']),
        pollInterval=1*60
    ))

    if branch['enable_l10n']:
        hg_all_locales_poller = HgAllLocalesPoller(hgURL = HGURL,
                            repositoryIndex = branch['l10n_repo_path'],
                            pollInterval = 5*60)
        hg_all_locales_poller.parallelRequests = 1
        c['change_source'].append(hg_all_locales_poller)

    # schedulers
    # this one gets triggered by the HG Poller
    c['schedulers'].append(MozScheduler(
        name=name,
        branch=branch['repo_path'],
        treeStableTimer=3*60,
        idleTimeout=branch.get('idle_timeout', None),
        builderNames=builders + unittestBuilders,
        fileIsImportant=lambda c: isHgPollerTriggered(c, HGURL)
    ))

    # Now, setup the nightly en-US schedulers and maybe,
    # their downstream l10n ones
    for builder in nightlyBuilders:
        nightly_scheduler=Nightly(
            name=builder,
            branch=branch['repo_path'],
            hour=[0],
            builderNames=[builder]
        )
        c['schedulers'].append(nightly_scheduler)

        if branch['enable_l10n'] and builder in l10nNightlyBuilders:
            l10n_builder = l10nNightlyBuilders[builder]['l10n_builder']
            platform = l10nNightlyBuilders[builder]['platform']
            tree = l10nNightlyBuilders[builder]['tree']
            c['schedulers'].append(NightlyL10n(
                                   name=l10n_builder,
                                   platform=platform,
                                   tree=tree,
                                   hour=[4],
                                   builderNames=[l10n_builder],
                                   repoType='hg',
                                   branch=branch['repo_path'],
                                   baseTag='default',
                                   localesFile=branch['allLocalesFile']
                                  ))

    for builder in weeklyBuilders:
        weekly_scheduler=Nightly(
            name=builder,
            branch=branch['repo_path'],
            dayOfWeek=5, # Saturday
            hour=[0],
            builderNames=[builder],
        )
        c['schedulers'].append(weekly_scheduler)

    for platform in sorted(branch['platforms'].keys()):
        # shorthand
        pf = branch['platforms'][platform]

        leakTest = False
        codesighs = True
        uploadPackages = True
        uploadSymbols = False
        if platform.find('-debug') > -1:
            leakTest = True
            codesighs = False
            uploadPackages = False
        if platform.find('win32') > -1 or platform.find('64') > -1:
            codesighs = False
        if 'upload_symbols' in pf and pf['upload_symbols']:
            uploadSymbols = True

        buildSpace = pf.get('build_space', DEFAULT_BUILD_SPACE)
        clobberTime = pf.get('clobber_time', DEFAULT_CLOBBER_TIME)

        mozilla2_dep_factory = CCNightlyBuildFactory(
            env=pf['env'],
            objdir=pf['platform_objdir'],
            platform=platform,
            hgHost=HGHOST,
            repoPath=branch['repo_path'],
            mozRepoPath=branch['mozilla_repo_path'],
            buildToolsRepoPath=BUILD_TOOLS_REPO_PATH,
            configRepoPath=CONFIG_REPO_PATH,
            configSubDir=CONFIG_SUBDIR,
            profiledBuild=pf['profiled_build'],
            productName=branch['product_name'],
            mozconfig=pf['mozconfig_dep'],
            branchName=name,
            stageServer=STAGE_SERVER,
            stageUsername=STAGE_USERNAME,
            stageGroup=STAGE_GROUP,
            stageSshKey=STAGE_SSH_KEY,
            stageBasePath=STAGE_BASE_PATH,
            graphServer=GRAPH_SERVER,
            graphSelector=GRAPH_SELECTOR,
            graphBranch=branch['major_version'],
            baseName=pf['base_name'],
            leakTest=leakTest,
            codesighs=codesighs,
            uploadPackages=uploadPackages,
            uploadSymbols=False,
            buildSpace=buildSpace,
            clobberURL=BASE_CLOBBER_URL,
            clobberTime=clobberTime,
            buildsBeforeReboot=pf['builds_before_reboot'],
            talosMasters=TALOS_MASTERS,
        )
        mozilla2_dep_builder = {
            'name': '%s build' % pf['base_name'],
            'slavenames': pf['slaves'],
            'builddir': '%s-%s' % (name, platform),
            'factory': mozilla2_dep_factory,
            'category': name,
        }
        c['builders'].append(mozilla2_dep_builder)

        # skip nightlies for debug builds
        if platform.find('debug') > -1:
             continue

        nightly_builder = '%s nightly' % pf['base_name']

        mozilla2_nightly_factory = CCNightlyBuildFactory(
            env=pf['env'],
            objdir=pf['platform_objdir'],
            platform=platform,
            hgHost=HGHOST,
            repoPath=branch['repo_path'],
            mozRepoPath=branch['mozilla_repo_path'],
            buildToolsRepoPath=BUILD_TOOLS_REPO_PATH,
            configRepoPath=CONFIG_REPO_PATH,
            configSubDir=CONFIG_SUBDIR,
            profiledBuild=pf['profiled_build'],
            productName=branch['product_name'],
            mozconfig=pf['mozconfig'],
            branchName=name,
            stageServer=STAGE_SERVER,
            stageUsername=STAGE_USERNAME,
            stageGroup=STAGE_GROUP,
            stageSshKey=STAGE_SSH_KEY,
            stageBasePath=STAGE_BASE_PATH,
            codesighs=False,
            uploadPackages=uploadPackages,
            uploadSymbols=uploadSymbols,
            nightly=True,
            createSnippet=branch['create_snippet'],
            ausBaseUploadDir=branch['aus2_base_upload_dir'],
            updatePlatform=pf['update_platform'],
            downloadBaseURL=DOWNLOAD_BASE_URL,
            ausUser=AUS2_USER,
            ausHost=AUS2_HOST,
            buildSpace=buildSpace,
            clobberURL=BASE_CLOBBER_URL,
            clobberTime=clobberTime,
            buildsBeforeReboot=pf['builds_before_reboot'],
            talosMasters=TALOS_MASTERS,
        )

        mozilla2_nightly_builder = {
            'name': nightly_builder,
            'slavenames': pf['slaves'],
            'builddir': '%s-%s-nightly' % (name, platform),
            'factory': mozilla2_nightly_factory,
            'category': name,
        }
        c['builders'].append(mozilla2_nightly_builder)

        if branch['enable_l10n']:
            if platform in ('linux','win32','macosx'):
                mozilla2_l10n_nightly_factory = CCNightlyRepackFactory(
                    hgHost=HGHOST,
                    tree=branch['l10n_tree'],
                    project=branch['product_name'],
                    appName=branch['app_name'],
                    enUSBinaryURL=branch['enUS_binaryURL'],
                    stageServer=STAGE_SERVER,
                    stageUsername=STAGE_USERNAME,
                    stageSshKey=STAGE_SSH_KEY,
                    branchName=name,
                    repoPath=branch['repo_path'],
                    mozRepoPath=branch['mozilla_repo_path'],
                    l10nRepoPath=branch['l10n_repo_path'],
                    buildToolsRepoPath=BUILD_TOOLS_REPO_PATH,
                    compareLocalesRepoPath=COMPARE_LOCALES_REPO_PATH,
                    compareLocalesTag=COMPARE_LOCALES_TAG,
                    buildSpace=2,
                    clobberURL=BASE_CLOBBER_URL,
                    clobberTime=clobberTime,
                )
                mozilla2_l10n_nightly_builder = {
                    'name': l10nNightlyBuilders[nightly_builder]['l10n_builder'],
                    'slavenames': L10N_SLAVES[platform],
                    'builddir': '%s-%s-l10n-nightly' % (name, platform),
                    'factory': mozilla2_l10n_nightly_factory,
                    'category': name,
                }
                c['builders'].append(mozilla2_l10n_nightly_builder)

        if branch['enable_unittests']:
            if platform in ('linux','win32','macosx'):
                runA11y = True
                if platform == 'macosx':
                    runA11y = branch['enable_mac_a11y']
                unittest_factory = CCUnittestBuildFactory(
                    platform=platform,
                    productName=branch['product_name'],
                    config_repo_path=CONFIG_REPO_PATH,
                    config_dir=CONFIG_SUBDIR,
                    objdir=OBJDIR_UNITTESTS,
                    branchName=name,
                    brandName=branch['brand_name'],
                    mochitest_leak_threshold=pf['mochitest_leak_threshold'],
                    mochichrome_leak_threshold=pf['mochichrome_leak_threshold'],
                    mochibrowser_leak_threshold=pf['mochibrowser_leak_threshold'],
                    crashtest_leak_threshold=pf['crashtest_leak_threshold'],
                    hgHost=HGHOST,
                    repoPath=branch['repo_path'],
                    mozRepoPath=branch['mozilla_repo_path'],
                    buildToolsRepoPath=BUILD_TOOLS_REPO_PATH,
                    buildSpace=branch['unittest_build_space'],
                    clobberURL=BASE_CLOBBER_URL,
                    clobberTime=clobberTime,
                    buildsBeforeReboot=pf['builds_before_reboot'],
                    stageServer=STAGE_SERVER,
                    stageUsername=STAGE_USERNAME,
                    stageSshKey=STAGE_SSH_KEY,
                    uploadPackages=True,
                    run_a11y=runA11y
                )
                unittest_builder = {
                    'name': '%s unit test' % pf['base_name'],
                    'slavenames': pf['slaves'],
                    'builddir': '%s-%s-unittest' % (name, platform),
                    'factory': unittest_factory,
                    'category': name,
                }
                c['builders'].append(unittest_builder)

        if branch['enable_codecoverage']:
            # We only do code coverage builds on linux right now
            if platform == 'linux':
                codecoverage_factory = CodeCoverageFactory(
                    platform=platform,
                    productName=branch['product_name'],
                    config_repo_path=CONFIG_REPO_PATH,
                    config_dir=CONFIG_SUBDIR,
                    objdir=OBJDIR_UNITTESTS,
                    hgHost=HGHOST,
                    branchName=name,
                    repoPath=branch['repo_path'],
                    buildToolsRepoPath=BUILD_TOOLS_REPO_PATH,
                    buildSpace=5,
                    clobberURL=BASE_CLOBBER_URL,
                    clobberTime=clobberTime,
                    buildsBeforeReboot=pf['builds_before_reboot'],
                    stageServer=STAGE_SERVER,
                    stageUsername=STAGE_USERNAME,
                    stageSshKey=STAGE_SSH_KEY,
                )
                codecoverage_builder = {
                    'name': '%s code coverage' % pf['base_name'],
                    'slavenames': pf['slaves'],
                    'builddir': '%s-%s-codecoverage' % (name, platform),
                    'factory': codecoverage_factory,
                    'category': name,
                }
                c['builders'].append(codecoverage_builder)

####### STATUS TARGETS

from buildbot.status import html

c['status'].append(html.WebStatus(
    http_port=8010, allowForce=True
))

####### L10n Scheduler with multiple dispatchers

# for debugging purposes of the repack on locale change system
buildbotcustom.log.init(
    scheduler = buildbotcustom.log.DEBUG,
    dispatcher = buildbotcustom.log.DEBUG
)

s = SchedulerL10n("l10n", "l10nbuilds.ini")
c['schedulers'].append(s)

####### Release Automation

import release_master
reload(release_master)

c['builders'].extend(release_master.builders)
c['schedulers'].extend(release_master.schedulers)
c['change_source'].extend(release_master.change_source)
c['status'].extend(release_master.status)

####### PROJECT IDENTITY

# the 'projectName' string will be used to describe the project that this
# buildbot is working on. For example, it is used as the title of the
# waterfall HTML page. The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.

c['projectName'] = "SeaMonkey 2"
c['projectURL'] = "http://wiki.mozilla.org/SeaMonkey:hg-based_build"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.Waterfall page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://cb-sea-linux-tbox.sj.mozilla.com:8010/"