Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

merge to production

changeset:   2497:125fb8df2570
parent:      2495:27f1f4e637e7
user:        Rail Aliiev <rail@mozilla.com>
date:        Thu Aug 02 18:06:38 2012 -0400
summary:     Bug 779907 - xulrunner builds don't pass configure on mac. r=bhearsum

changeset:   2498:c36aa753df85
user:        Ben Hearsum <bhearsum@mozilla.com>
date:        Thu Aug 02 15:11:50 2012 -0700
summary:     bug 780004: remove CodeCoverage and CC stuff from buildbotcustom. r=catlee

changeset:   2499:4488e93a55c9
user:        Ben Hearsum <bhearsum@mozilla.com>
date:        Thu Aug 02 15:31:09 2012 -0700
summary:     Kill UnittestBuildFactory too. r=catlee

changeset:   2500:2a3ea74b1a7a
tag:         tip
user:        Ben Hearsum <bhearsum@mozilla.com>
date:        Thu Aug 02 16:27:14 2012 -0700
summary:     Kill unused code. r=catlee

--HG--
branch : production-0.8
  • Loading branch information...
commit 4b331d86b35ce161e2df28f4724351d6b7a84cb6 2 parents 2432c12 + 5b62abc
Justin Wood authored
46 l10n.py
View
@@ -193,49 +193,3 @@ def __init__(self, name, builderNames, **kwargs):
def trigger(self, ss, set_props=None):
reason = "This build was triggered by the successful completion of the en-US nightly."
self.createL10nBuilds(revision=ss.revision, reason=reason, set_props=set_props)
-
-
-class DependentL10n(Dependent, L10nMixin):
- """
- This scheduler runs some set of 'downstream' builds when the
- 'upstream' scheduler has completed successfully.
- """
-
- compare_attrs = Dependent.compare_attrs[:]
-
- def __init__(self, name, upstream, builderNames, **kwargs):
- Dependent.__init__(self, name, upstream, builderNames)
- L10nMixin.__init__(self, **kwargs)
-
- def run(self):
- d = self.parent.db.runInteraction(self._run)
-
- def _createBuilds(sses):
- # This gets called with a list of SourceStamps to do builds for
- l = []
- for ss in sses:
- d1 = self.createL10nBuilds(ss.revision)
- l.append(d1)
- return defer.gatherResults(l)
-
- d.addCallback(_createBuilds)
- # Return None here since we don't need to be woken up in a specific
- # amount of time.
- d.addCallback(lambda ign: None)
-
- return d
-
- def _run(self, t):
- db = self.parent.db
- res = db.scheduler_get_subscribed_buildsets(self.schedulerid, t)
- # this returns bsid,ssid,results for all of our active subscriptions.
- # We ignore the ones that aren't complete yet. This leaves the
- # subscription in place until the buildset is complete.
- sses = []
- for (bsid,ssid,complete,results) in res:
- if complete:
- if results in (SUCCESS, WARNINGS):
- ss = db.getSourceStampNumberedNow(ssid, t)
- sses.append(ss)
- db.scheduler_unsubscribe_buildset(self.schedulerid, bsid, t)
- return sses
1,289 misc.py
View
@@ -48,7 +48,6 @@
from buildbotcustom.changes.hgpoller import HgPoller, HgAllLocalesPoller
from buildbotcustom.process.factory import NightlyBuildFactory, \
NightlyRepackFactory, UnittestPackagedBuildFactory, TalosFactory, \
- CCNightlyBuildFactory, CCNightlyRepackFactory, CCUnittestBuildFactory, \
TryBuildFactory, ScriptFactory, rc_eval_func
from buildbotcustom.process.factory import RemoteUnittestFactory
from buildbotcustom.scheduler import MultiScheduler, BuilderChooserScheduler, \
@@ -1621,6 +1620,8 @@ def generateBranchObjects(config, name, secrets=None):
buildsBeforeReboot=pf['builds_before_reboot'],
packageSDK=True,
signingServers=secrets.get(pf.get('nightly_signing_servers')),
+ tooltool_manifest_src=pf.get('tooltool_manifest_src', None),
+ tooltool_url_list=config.get('tooltool_url_list', []),
)
mozilla2_xulrunner_builder = {
'name': '%s xulrunner' % pf['base_name'],
@@ -1675,1292 +1676,6 @@ def generateBranchObjects(config, name, secrets=None):
return branchObjects
-def generateCCBranchObjects(config, name, secrets=None):
- """name is the name of branch which is usually the last part of the path
- to the repository. For example, 'comm-central-trunk', or 'comm-aurora'
- or 'comm-beta'.
- config is a dictionary containing all of the necessary configuration
- information for a branch. The required keys depends greatly on what's
- enabled for a branch (unittests, l10n, etc). The best way to figure out
- what you need to pass is by looking at existing configs and using
- 'buildbot checkconfig' to verify.
- """
- # We return this at the end
- branchObjects = {
- 'builders': [],
- 'change_source': [],
- 'schedulers': [],
- 'status': []
- }
- if secrets is None:
- secrets = {}
- builders = []
- unittestBuilders = []
- triggeredUnittestBuilders = []
- nightlyBuilders = []
- xulrunnerNightlyBuilders = []
- periodicPgoBuilders = [] # Only used for the 'periodic' strategy.
- debugBuilders = []
- weeklyBuilders = []
- coverageBuilders = []
- # prettyNames is a mapping to pass to the try_parser for validation
- PRETTY_NAME = '%s build'
- prettyNames = {}
- unittestPrettyNames = {}
- unittestSuites = []
- # These dicts provides mapping between en-US dep and nightly scheduler names
- # to l10n dep and l10n nightly scheduler names. It's filled out just below here.
- l10nBuilders = {}
- l10nNightlyBuilders = {}
- pollInterval = config.get('pollInterval', 60)
- l10nPollInterval = config.get('l10nPollInterval', 5*60)
-
- # We only understand a couple PGO strategies
- assert config['pgo_strategy'] in ('per-checkin', 'periodic', None), \
- "%s is not an understood PGO strategy" % config['pgo_strategy']
-
- # This section is to make it easier to disable certain products.
- # Ideally we could specify a shorter platforms key on the branch,
- # but that doesn't work
- enabled_platforms = []
- for platform in sorted(config['platforms'].keys()):
- pf = config['platforms'][platform]
- if pf['stage_product'] in config['enabled_products']:
- enabled_platforms.append(platform)
-
- # generate a list of builders, nightly builders (names must be different)
- # for easy access
- for platform in enabled_platforms:
- pf = config['platforms'][platform]
- base_name = pf['base_name']
- pretty_name = PRETTY_NAME % base_name
- if platform.endswith("-debug"):
- debugBuilders.append(pretty_name)
- prettyNames[platform] = pretty_name
- # Debug unittests
- if pf.get('enable_unittests'):
- test_builders = []
- if 'opt_base_name' in config['platforms'][platform]:
- base_name = config['platforms'][platform]['opt_base_name']
- else:
- base_name = config['platforms'][platform.replace("-debug", "")]['base_name']
- for suites_name, suites in config['unittest_suites']:
- unittestPrettyNames[platform] = '%s debug test' % base_name
- test_builders.extend(generateTestBuilderNames('%s debug test' % base_name, suites_name, suites))
- triggeredUnittestBuilders.append(('%s-%s-unittest' % (name, platform), test_builders, config.get('enable_merging', True)))
- # Skip l10n, unit tests
- # Skip nightlies for debug builds unless requested
- if not pf.has_key('enable_nightly'):
- continue
- elif pf.get('enable_dep', True):
- builders.append(pretty_name)
- prettyNames[platform] = pretty_name
-
- # Fill the l10n dep dict
- if config['enable_l10n'] and platform in config['l10n_platforms'] and \
- config['enable_l10n_onchange']:
- l10nBuilders[base_name] = {}
- l10nBuilders[base_name]['tree'] = config['l10n_tree']
- l10nBuilders[base_name]['l10n_builder'] = \
- '%s %s %s l10n dep' % (pf['product_name'].capitalize(),
- name, platform)
- l10nBuilders[base_name]['platform'] = platform
- # Check if branch wants nightly builds
- if config['enable_nightly']:
- if pf.has_key('enable_nightly'):
- do_nightly = pf['enable_nightly']
- else:
- do_nightly = True
- else:
- do_nightly = False
-
- # Check if platform as a PGO builder
- if config['pgo_strategy'] == 'periodic' and platform in config['pgo_platforms']:
- periodicPgoBuilders.append('%s pgo-build' % pf['base_name'])
-
- if do_nightly:
- builder = '%s nightly' % base_name
- nightlyBuilders.append(builder)
- # Fill the l10nNightly dict
- if config['enable_l10n'] and platform in config['l10n_platforms']:
- l10nNightlyBuilders[builder] = {}
- l10nNightlyBuilders[builder]['tree'] = config['l10n_tree']
- l10nNightlyBuilders[builder]['l10n_builder'] = \
- '%s %s %s l10n nightly' % (pf['product_name'].capitalize(),
- name, platform)
- l10nNightlyBuilders[builder]['platform'] = platform
- if config['enable_shark'] and pf.get('enable_shark'):
- nightlyBuilders.append('%s shark' % base_name)
- if config['enable_valgrind'] and \
- platform in config['valgrind_platforms']:
- nightlyBuilders.append('%s valgrind' % base_name)
- # Regular unittest builds
- if pf.get('enable_unittests'):
- unittestBuilders.append('%s unit test' % base_name)
- test_builders = []
- for suites_name, suites in config['unittest_suites']:
- test_builders.extend(generateTestBuilderNames('%s test' % base_name, suites_name, suites))
- unittestPrettyNames[platform] = '%s test' % base_name
- triggeredUnittestBuilders.append(('%s-%s-unittest' % (name, platform), test_builders, config.get('enable_merging', True)))
- # Optimized unittest builds
- if pf.get('enable_opt_unittests'):
- test_builders = []
- for suites_name, suites in config['unittest_suites']:
- unittestPrettyNames[platform] = '%s opt test' % base_name
- test_builders.extend(generateTestBuilderNames('%s opt test' % base_name, suites_name, suites))
- triggeredUnittestBuilders.append(('%s-%s-opt-unittest' % (name, platform), test_builders, config.get('enable_merging', True)))
- if config['enable_codecoverage'] and platform in ('linux',):
- coverageBuilders.append('%s code coverage' % base_name)
- if config.get('enable_blocklist_update', False) and platform in ('linux',):
- weeklyBuilders.append('%s blocklist update' % base_name)
- if pf.get('enable_xulrunner', config['enable_xulrunner']):
- xulrunnerNightlyBuilders.append('%s xulrunner' % base_name)
- if config['enable_weekly_bundle']:
- weeklyBuilders.append('%s hg bundle' % name)
-
- logUploadCmd = makeLogUploadCommand(name, config, is_try=config.get('enable_try'),
- is_shadow=bool(name=='shadow-central'), platform_prop='stage_platform',product_prop='product')
-
- # this comment is for grepping! SubprocessLogHandler
- branchObjects['status'].append(QueuedCommandHandler(
- logUploadCmd,
- QueueDir.getQueue('commands'),
- builders=builders + unittestBuilders + debugBuilders + periodicPgoBuilders,
- ))
-
- if nightlyBuilders:
- branchObjects['status'].append(QueuedCommandHandler(
- logUploadCmd + ['--nightly'],
- QueueDir.getQueue('commands'),
- builders=nightlyBuilders,
- ))
-
- # Currently, each branch goes to a different tree
- # If this changes in the future this may have to be
- # moved out of the loop
- if not config.get('disable_tinderbox_mail'):
- branchObjects['status'].append(TinderboxMailNotifier(
- fromaddr="comm.buildbot@build.mozilla.org",
- tree=config['tinderbox_tree'],
- extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
- relayhost="mail.build.mozilla.org",
- builders=builders + nightlyBuilders + unittestBuilders + debugBuilders,
- logCompression="gzip",
- errorparser="unittest"
- ))
- if config.get('enable_xulrunner'):
- # XULRunner builds
- branchObjects['status'].append(TinderboxMailNotifier(
- fromaddr="comm.buildbot@build.mozilla.org",
- tree=config['xulrunner_tinderbox_tree'],
- extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
- relayhost="mail.build.mozilla.org",
- builders=xulrunnerNightlyBuilders,
- logCompression="gzip"
- ))
- # Code coverage builds go to a different tree
- branchObjects['status'].append(TinderboxMailNotifier(
- fromaddr="comm.buildbot@build.mozilla.org",
- tree=config['weekly_tinderbox_tree'],
- extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
- relayhost="mail.build.mozilla.org",
- builders=coverageBuilders,
- logCompression="gzip",
- errorparser="unittest"
- ))
-
- # Try Server notifier
- if config.get('enable_mail_notifier'):
- packageUrl = config['package_url']
- packageDir = config['package_dir']
-
- if config.get('notify_real_author'):
- extraRecipients = []
- sendToInterestedUsers = True
- else:
- extraRecipients = config['email_override']
- sendToInterestedUsers = False
-
- # This notifies users as soon as we receive their push, and will let them
- # know where to find builds/logs
- branchObjects['status'].append(ChangeNotifier(
- fromaddr="tryserver@build.mozilla.org",
- lookup=MercurialEmailLookup(),
- relayhost="mail.build.mozilla.org",
- sendToInterestedUsers=sendToInterestedUsers,
- extraRecipients=extraRecipients,
- branches=[config['repo_path']],
- messageFormatter=lambda c: buildTryChangeMessage(c,
- '/'.join([packageUrl, packageDir])),
- ))
-
- if config['enable_l10n']:
- l10n_builders = []
- for b in l10nBuilders:
- if config['enable_l10n_onchange']:
- l10n_builders.append(l10nBuilders[b]['l10n_builder'])
- l10n_builders.append(l10nNightlyBuilders['%s nightly' % b]['l10n_builder'])
- l10n_binaryURL = config['enUS_binaryURL']
- if l10n_binaryURL.endswith('/'):
- l10n_binaryURL = l10n_binaryURL[:-1]
- l10n_binaryURL += "-l10n"
- nomergeBuilders.extend(l10n_builders)
-
- branchObjects['status'].append(TinderboxMailNotifier(
- fromaddr="comm.buildbot@build.mozilla.org",
- tree=config['l10n_tinderbox_tree'],
- extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
- relayhost="mail.build.mozilla.org",
- logCompression="gzip",
- builders=l10n_builders,
- binaryURL=l10n_binaryURL
- ))
-
- # We only want the builds from the specified builders
- # since their builds have a build property called "locale"
- branchObjects['status'].append(TinderboxMailNotifier(
- fromaddr="comm.buildbot@build.mozilla.org",
- tree=WithProperties(config['l10n_tinderbox_tree'] + "-%(locale)s"),
- extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
- relayhost="mail.build.mozilla.org",
- logCompression="gzip",
- builders=l10n_builders,
- binaryURL=l10n_binaryURL
- ))
-
- # Log uploads for dep l10n repacks
- branchObjects['status'].append(QueuedCommandHandler(
- logUploadCmd + ['--l10n'],
- QueueDir.getQueue('commands'),
- builders=[l10nBuilders[b]['l10n_builder'] for b in l10nBuilders],
- ))
- # and for nightly repacks
- branchObjects['status'].append(QueuedCommandHandler(
- logUploadCmd + ['--l10n', '--nightly'],
- QueueDir.getQueue('commands'),
- builders=[l10nNightlyBuilders['%s nightly' % b]['l10n_builder'] for b in l10nBuilders]
- ))
-
- # Skip https repos until bug 592060 is fixed and we have a https-capable HgPoller
- if config['hgurl'].startswith('https:'):
- pass
- else:
- if config.get('enable_try', False):
- tipsOnly = True
- # Pay attention to all branches for pushes to try
- repo_branch = None
- else:
- tipsOnly = True
- # Other branches should only pay attention to the default branch
- repo_branch = "default"
-
- branchObjects['change_source'].append(HgPoller(
- hgURL=config['hgurl'],
- branch=config['repo_path'],
- tipsOnly=tipsOnly,
- repo_branch=repo_branch,
- pollInterval=pollInterval,
- storeRev="polled_comm_revision",
- ))
- # for Mozilla tree, need valid branch, so override pushlog URL
- branchObjects['change_source'].append(HgPoller(
- hgURL=config['hgurl'],
- branch=config['repo_path'],
- pushlogUrlOverride='%s/%s/json-pushes?full=1' % (config['hgurl'],
- config['mozilla_repo_path']),
- tipsOnly=tipsOnly,
- repo_branch=repo_branch,
- pollInterval=pollInterval,
- storeRev="polled_moz_revision",
- ))
-
- if config['enable_l10n'] and config['enable_l10n_onchange']:
- hg_all_locales_poller = HgAllLocalesPoller(hgURL = config['hgurl'],
- repositoryIndex = config['l10n_repo_path'],
- pollInterval=l10nPollInterval,
- branch=name)
- hg_all_locales_poller.parallelRequests = 1
- branchObjects['change_source'].append(hg_all_locales_poller)
-
- # schedulers
- # this one gets triggered by the HG Poller
- # for Try we have a custom scheduler that can accept a function to read commit comments
- # in order to know what to schedule
- extra_args = {}
- if config.get('enable_try'):
- scheduler_class = makePropertiesScheduler(BuilderChooserScheduler, [buildUIDSchedFunc])
- extra_args['chooserFunc'] = tryChooser
- extra_args['numberOfBuildsToTrigger'] = 1
- extra_args['prettyNames'] = prettyNames
- else:
- scheduler_class = makePropertiesScheduler(Scheduler, [buildIDSchedFunc, buildUIDSchedFunc])
-
- if not config.get('enable_merging', True):
- nomergeBuilders.extend(builders + unittestBuilders + debugBuilders)
- nomergeBuilders.extend(periodicPgoBuilders) # these should never, ever merge
- extra_args['treeStableTimer'] = None
-
- branchObjects['schedulers'].append(scheduler_class(
- name=name,
- branch=config['repo_path'],
- builderNames=builders + unittestBuilders + debugBuilders,
- fileIsImportant=lambda c: isHgPollerTriggered(c, config['hgurl']) and shouldBuild(c),
- **extra_args
- ))
-
- if config['enable_l10n']:
- l10n_builders = []
- for b in l10nBuilders:
- l10n_builders.append(l10nBuilders[b]['l10n_builder'])
- # This L10n scheduler triggers only the builders of its own branch
- branchObjects['schedulers'].append(Scheduler(
- name="%s l10n" % name,
- branch=config['l10n_repo_path'],
- treeStableTimer=None,
- builderNames=l10n_builders,
- fileIsImportant=lambda c: isImportantL10nFile(c, config['l10n_modules']),
- properties={
- 'app': pf['app_name'],
- 'en_revision': 'default',
- 'l10n_revision': 'default',
- }
- ))
-
- for scheduler_branch, test_builders, merge in triggeredUnittestBuilders:
- scheduler_name = scheduler_branch
- for test in test_builders:
- unittestSuites.append(test.split(' ')[-1])
- if not merge:
- nomergeBuilders.extend(test_builders)
- extra_args = {}
- if config.get('enable_try'):
- scheduler_class = BuilderChooserScheduler
- extra_args['chooserFunc'] = tryChooser
- extra_args['numberOfBuildsToTrigger'] = 1
- extra_args['prettyNames'] = prettyNames
- extra_args['unittestSuites'] = unittestSuites
- extra_args['unittestPrettyNames'] = unittestPrettyNames
- else:
- scheduler_class = Scheduler
- branchObjects['schedulers'].append(scheduler_class(
- name=scheduler_name,
- branch=scheduler_branch,
- builderNames=test_builders,
- treeStableTimer=None,
- **extra_args
- ))
-
- if not config.get('disable_tinderbox_mail'):
- branchObjects['status'].append(TinderboxMailNotifier(
- fromaddr="comm.buildbot@build.mozilla.org",
- tree=config['packaged_unittest_tinderbox_tree'],
- extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
- relayhost="mail.build.mozilla.org",
- builders=test_builders,
- logCompression="gzip",
- errorparser="unittest"
- ))
-
- branchObjects['status'].append(QueuedCommandHandler(
- logUploadCmd,
- QueueDir.getQueue('commands'),
- builders=test_builders,
- ))
-
- # Now, setup the nightly en-US schedulers and maybe,
- # their downstream l10n ones
- if nightlyBuilders or xulrunnerNightlyBuilders:
- goodFunc = lastGoodFunc(
- branch=config['repo_path'],
- builderNames=builders,
- triggerBuildIfNoChanges=False,
- l10nBranch=config.get('l10n_repo_path')
- )
-
- nightly_scheduler = makePropertiesScheduler(
- SpecificNightly,
- [buildIDSchedFunc, buildUIDSchedFunc])(
- ssFunc=goodFunc,
- name="%s nightly" % name,
- branch=config['repo_path'],
- # bug 482123 - keep the minute to avoid problems with DST
- # changes
- hour=config['start_hour'], minute=config['start_minute'],
- builderNames=nightlyBuilders + xulrunnerNightlyBuilders,
- )
- branchObjects['schedulers'].append(nightly_scheduler)
-
- if len(periodicPgoBuilders) > 0:
- pgo_scheduler = makePropertiesScheduler(
- Nightly,
- [buildIDSchedFunc, buildUIDSchedFunc])(
- name="%s pgo" % name,
- branch=config['repo_path'],
- builderNames=periodicPgoBuilders,
- hour=range(0,24,config['periodic_pgo_interval']),
- )
- branchObjects['schedulers'].append(pgo_scheduler)
-
- for builder in nightlyBuilders + xulrunnerNightlyBuilders:
- if config['enable_l10n'] and \
- config['enable_nightly'] and builder in l10nNightlyBuilders:
- l10n_builder = l10nNightlyBuilders[builder]['l10n_builder']
- platform = l10nNightlyBuilders[builder]['platform']
- branchObjects['schedulers'].append(TriggerableL10n(
- name=l10n_builder,
- platform=platform,
- builderNames=[l10n_builder],
- branch=config['repo_path'],
- baseTag='default',
- localesURL=config.get('localesURL', None)
- ))
-
- weekly_scheduler = Nightly(
- name='weekly-%s' % name,
- branch=config['repo_path'],
- dayOfWeek=5, # Saturday
- hour=[3], minute=[02],
- builderNames=coverageBuilders + weeklyBuilders,
- )
- branchObjects['schedulers'].append(weekly_scheduler)
-
- # We iterate throught the platforms a second time, so we need
- # to ensure that disabled platforms aren't configured the second time
- enabled_platforms = []
- for platform in sorted(config['platforms'].keys()):
- pf = config['platforms'][platform]
- if pf['stage_product'] in config['enabled_products']:
- enabled_platforms.append(platform)
-
- for platform in enabled_platforms:
- # shorthand
- pf = config['platforms'][platform]
-
- # The stage platform needs to be used by the factory __init__ methods
- # as well as the log handler status target. Instead of repurposing the
- # platform property on each builder, we will create a new property
- # on the needed builders
- stage_platform = pf.get('stage_platform', platform)
-
- uploadPackages = True
- uploadSymbols = False
- packageTests = False
- talosMasters = pf.get('talos_masters', [])
- unittestBranch = "%s-%s-opt-unittest" % (name, platform)
- # Generate the PGO branch even if it isn't on for dep builds
- # because we will still use it for nightlies... maybe
- pgoUnittestBranch = "%s-%s-pgo-unittest" % (name, platform)
- tinderboxBuildsDir = None
- if platform.find('-debug') > -1:
- # Some platforms can't run on the build host
- leakTest = pf.get('enable_leaktests', True)
- codesighs = False
- if not pf.get('enable_unittests'):
- uploadPackages = pf.get('packageTests', False)
- else:
- packageTests = True
- talosMasters = None
- # Platform already has the -debug suffix
- unittestBranch = "%s-%s-unittest" % (name, platform)
- tinderboxBuildsDir = "%s-%s" % (name, platform)
- else:
- if pf.get('enable_opt_unittests'):
- packageTests=True
- codesighs = pf.get('enable_codesighs', True)
- leakTest = False
-
- # Allow for test packages on platforms that can't be tested
- # on the same master.
- packageTests = pf.get('packageTests', packageTests)
-
- if platform.find('win') > -1:
- codesighs = False
-
- doBuildAnalysis = pf.get('enable_build_analysis', False)
-
- buildSpace = pf.get('build_space', config['default_build_space'])
- l10nSpace = config['default_l10n_space']
- clobberTime = pf.get('clobber_time', config['default_clobber_time'])
- mochitestLeakThreshold = pf.get('mochitest_leak_threshold', None)
- # -chrome- and -browser- are only used by CCUnittestBuildFactory
- mochichromeLeakThreshold = pf.get('mochichrome_leak_threshold', None)
- mochibrowserLeakThreshold = pf.get('mochibrowser_leak_threshold', None)
- crashtestLeakThreshold = pf.get('crashtest_leak_threshold', None)
- checkTest = pf.get('enable_checktests', False)
- valgrindCheck = pf.get('enable_valgrind_checktests', False)
-
- extra_args = {}
- if config.get('enable_try'):
- factory_class = TryBuildFactory
- extra_args['packageUrl'] = config['package_url']
- extra_args['packageDir'] = config['package_dir']
- extra_args['branchName'] = name
- uploadSymbols = pf.get('upload_symbols', False)
- else:
- factory_class = CCNightlyBuildFactory
- uploadSymbols = False
-
- stageBasePath = '%s/%s' % (config['stage_base_path'],
- pf['stage_product'])
-
- # For the 'per-checkin' pgo strategy, we want PGO
- # enabled on what would be 'opt' builds.
- if platform in config['pgo_platforms']:
- if config['pgo_strategy'] == 'periodic' or config['pgo_strategy'] == None:
- per_checkin_build_uses_pgo = False
- elif config['pgo_strategy'] == 'per-checkin':
- per_checkin_build_uses_pgo = True
- else:
- # All platforms that can't do PGO... shouldn't do PGO.
- per_checkin_build_uses_pgo = False
-
- if per_checkin_build_uses_pgo:
- per_checkin_unittest_branch = pgoUnittestBranch
- else:
- per_checkin_unittest_branch = unittestBranch
-
- # Some platforms shouldn't do dep builds (i.e. RPM)
- if pf.get('enable_dep', True):
- factory_kwargs = {
- 'env': pf['env'],
- 'objdir': pf['platform_objdir'],
- 'platform': platform,
- 'hgHost': config['hghost'],
- 'repoPath': config['repo_path'],
- 'mozRepoPath': config['mozilla_repo_path'],
- 'buildToolsRepoPath': config['build_tools_repo_path'],
- 'configRepoPath': config['config_repo_path'],
- 'configSubDir': config['config_subdir'],
- 'profiledBuild': per_checkin_build_uses_pgo,
- 'productName': pf['product_name'],
- 'mozconfig': pf['mozconfig'],
- 'srcMozconfig': pf.get('src_mozconfig'),
- 'use_mock': pf.get('use_mock'),
- 'stageServer': config['stage_server'],
- 'stageUsername': config['stage_username'],
- 'stageGroup': config['stage_group'],
- 'stageSshKey': config['stage_ssh_key'],
- 'stageBasePath': stageBasePath,
- 'stageLogBaseUrl': config.get('stage_log_base_url', None),
- 'stagePlatform': pf['stage_platform'],
- 'stageProduct': pf['stage_product'],
- 'graphServer': config['graph_server'],
- 'graphSelector': config['graph_selector'],
- 'graphBranch': config.get('graph_branch', config['tinderbox_tree']),
- 'doBuildAnalysis': doBuildAnalysis,
- 'baseName': pf['base_name'],
- 'leakTest': leakTest,
- 'checkTest': checkTest,
- 'valgrindCheck': valgrindCheck,
- 'codesighs': codesighs,
- 'uploadPackages': uploadPackages,
- 'uploadSymbols': uploadSymbols,
- 'buildSpace': buildSpace,
- 'clobberURL': config['base_clobber_url'],
- 'clobberTime': clobberTime,
- 'buildsBeforeReboot': pf['builds_before_reboot'],
- 'talosMasters': talosMasters,
- 'packageTests': packageTests,
- 'unittestMasters': pf.get('unittest_masters', config['unittest_masters']),
- 'unittestBranch': per_checkin_unittest_branch,
- 'tinderboxBuildsDir': tinderboxBuildsDir,
- 'enable_ccache': pf.get('enable_ccache', False),
- 'useSharedCheckouts': pf.get('enable_shared_checkouts', False),
- 'testPrettyNames': pf.get('test_pretty_names', False),
- 'l10nCheckTest': pf.get('l10n_check_test', False),
- 'android_signing': pf.get('android_signing', False),
- 'post_upload_include_platform': pf.get('post_upload_include_platform', False),
- 'signingServers': secrets.get(pf.get('dep_signing_servers')),
- 'baseMirrorUrls': config.get('base_mirror_urls'),
- 'baseBundleUrls': config.get('base_bundle_urls'),
- 'branchName': name,
- }
- factory_kwargs.update(extra_args)
-
- if name in ('comm-1.9.1', 'comm-1.9.2', 'comm-2.0'):
- # We force profiledBuild off here because its meaning has changed
- # We deal with turning on PGO for these old branches in the actual factory
- factory_kwargs['profiledBuild'] = False
-
- mozilla2_dep_factory = factory_class(**factory_kwargs)
- mozilla2_dep_builder = {
- 'name': '%s build' % pf['base_name'],
- 'slavenames': pf['slaves'],
- 'builddir': '%s-%s' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s' % (name, platform), pf['stage_product']),
- 'factory': mozilla2_dep_factory,
- 'category': name,
- 'nextSlave': _nextFastSlave,
- # Uncomment to enable only fast slaves for dep builds.
- #'nextSlave': lambda b, sl: _nextFastSlave(b, sl, only_fast=True),
- 'properties': {'branch': name,
- 'platform': platform,
- 'stage_platform': stage_platform,
- 'product': pf['stage_product'],
- 'slavebuilddir' : reallyShort('%s-%s' % (name, platform), pf['stage_product'])},
- }
- branchObjects['builders'].append(mozilla2_dep_builder)
-
- # We have some platforms which need to be built every X hours with PGO.
- # These builds are as close to regular dep builds as we can make them,
- # other than PGO
- if config['pgo_strategy'] == 'periodic' and platform in config['pgo_platforms']:
- pgo_kwargs = factory_kwargs.copy()
- pgo_kwargs['profiledBuild'] = True
- pgo_kwargs['stagePlatform'] += '-pgo'
- pgo_kwargs['unittestBranch'] = pgoUnittestBranch
- pgo_factory = factory_class(**pgo_kwargs)
- pgo_builder = {
- 'name': '%s pgo-build' % pf['base_name'],
- 'slavenames': pf['slaves'],
- 'builddir': '%s-%s-pgo' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-pgo' % (name, platform), pf['stage_product']),
- 'factory': pgo_factory,
- 'category': name,
- 'nextSlave': _nextFastSlave,
- 'properties': {'branch': name,
- 'platform': platform,
- 'stage_platform': stage_platform + '-pgo',
- 'product': pf['stage_product'],
- 'slavebuilddir' : reallyShort('%s-%s-pgo' % (name, platform), pf['stage_product'])},
- }
- branchObjects['builders'].append(pgo_builder)
-
- # skip nightlies for debug builds unless requested at platform level
- if platform.find('debug') > -1:
- if pf.get('enable_unittests'):
- for suites_name, suites in config['unittest_suites']:
- if "macosx" in platform and 'mochitest-a11y' in suites:
- suites = suites[:]
- suites.remove('mochitest-a11y')
-
- if 'opt_base_name' in config['platforms'][platform]:
- base_name = config['platforms'][platform]['opt_base_name']
- else:
- base_name = config['platforms'][platform.replace("-debug", "")]['base_name']
-
- branchObjects['builders'].extend(generateTestBuilder(
- config, name, platform, "%s debug test" % base_name,
- "%s-%s-unittest" % (name, platform),
- suites_name, suites, mochitestLeakThreshold,
- crashtestLeakThreshold, stagePlatform=stage_platform,
- stageProduct=pf['stage_product']))
- if not pf.has_key('enable_nightly'):
- continue
-
- if config['enable_nightly']:
- if pf.has_key('enable_nightly'):
- do_nightly = pf['enable_nightly']
- else:
- do_nightly = True
- else:
- do_nightly = False
-
- if do_nightly:
- nightly_builder = '%s nightly' % pf['base_name']
-
- platform_env = pf['env'].copy()
- if 'update_channel' in config and config.get('create_snippet'):
- platform_env['MOZ_UPDATE_CHANNEL'] = config['update_channel']
-
- triggeredSchedulers=None
- if config['enable_l10n'] and pf.get('is_mobile_l10n') and pf.get('l10n_chunks'):
- mobile_l10n_scheduler_name = '%s-%s-l10n' % (name, platform)
- mobile_l10n_builders = []
- builder_env = platform_env.copy()
- for n in range(1, int(pf['l10n_chunks']) + 1):
- builddir='%s-%s-l10n_%s' % (name, platform, str(n))
- builderName = "%s l10n nightly %s/%s" % \
- (pf['base_name'], n, pf['l10n_chunks'])
- mobile_l10n_builders.append(builderName)
- factory = ScriptFactory(
- scriptRepo='%s%s' % (config['hgurl'],
- config['mozharness_repo_path']),
- scriptName='scripts/mobile_l10n.py',
- extra_args=['--cfg',
- 'single_locale/%s_%s.py' % (name, platform),
- '--total-chunks', str(pf['l10n_chunks']),
- '--this-chunk', str(n)]
- )
- slavebuilddir = reallyShort(builddir, pf['stage_product'])
- branchObjects['builders'].append({
- 'name': builderName,
- 'slavenames': pf.get('slaves'),
- 'builddir': builddir,
- 'slavebuilddir': slavebuilddir,
- 'factory': factory,
- 'category': name,
- 'nextSlave': _nextL10nSlave(),
- 'properties': {'branch': '%s' % config['repo_path'],
- 'builddir': '%s-l10n_%s' % (builddir, str(n)),
- 'stage_platform': stage_platform,
- 'product': pf['stage_product'],
- 'slavebuilddir': slavebuilddir},
- 'env': builder_env
- })
-
- branchObjects["schedulers"].append(Triggerable(
- name=mobile_l10n_scheduler_name,
- builderNames=mobile_l10n_builders
- ))
- triggeredSchedulers=[mobile_l10n_scheduler_name]
-
- else: # Non-mobile l10n is done differently at this time
- if config['enable_l10n'] and platform in config['l10n_platforms'] and \
- nightly_builder in l10nNightlyBuilders:
- triggeredSchedulers=[l10nNightlyBuilders[nightly_builder]['l10n_builder']]
-
-
- multiargs = {}
- if config.get('enable_l10n') and config.get('enable_multi_locale') and pf.get('multi_locale'):
- multiargs['multiLocale'] = True
- multiargs['multiLocaleMerge'] = config['multi_locale_merge']
- multiargs['compareLocalesRepoPath'] = config['compare_locales_repo_path']
- multiargs['compareLocalesTag'] = config['compare_locales_tag']
- multiargs['mozharnessRepoPath'] = config['mozharness_repo_path']
- multiargs['mozharnessTag'] = config['mozharness_tag']
- multi_config_name = 'multi_locale/%s_%s.json' % (name, platform)
- if 'android' in platform:
- multiargs['multiLocaleScript'] = 'scripts/multil10n.py'
- elif 'maemo' in platform:
- multiargs['multiLocaleScript'] = 'scripts/maemo_multi_locale_build.py'
- multiargs['multiLocaleConfig'] = multi_config_name
-
- create_snippet = config['create_snippet']
- if pf.has_key('create_snippet') and config['create_snippet']:
- create_snippet = pf.get('create_snippet')
- if create_snippet and 'android' in platform:
- # Ideally, this woud use some combination of product name and
- # stage_platform, but that can be done in a follow up.
- # Android doesn't create updates for all the branches that
- # Firefox desktop does.
- if config.get('create_mobile_snippet'):
- ausargs = {
- 'downloadBaseURL': config['mobile_download_base_url'],
- 'downloadSubdir': '%s-%s' % (name, pf.get('stage_platform', platform)),
- 'ausBaseUploadDir': config['aus2_mobile_base_upload_dir'],
- }
- else:
- create_snippet = False
- ausargs = {}
- else:
- ausargs = {
- 'downloadBaseURL': config['download_base_url'],
- 'downloadSubdir': '%s-%s' % (name, pf.get('stage_platform', platform)),
- 'ausBaseUploadDir': config['aus2_base_upload_dir'],
- }
-
-
- nightly_kwargs = {}
- nightly_kwargs.update(multiargs)
- nightly_kwargs.update(ausargs)
-
- # We make the assumption that *all* nightly builds
- # are to be done with PGO. This is to ensure that all
- # branches get some PGO coverage
- # We do not stick '-pgo' in the stage_platform for
- # nightlies because it'd be ugly and break stuff
- if platform in config['pgo_platforms']:
- nightly_pgo = True
- nightlyUnittestBranch = pgoUnittestBranch
- else:
- nightlyUnittestBranch = unittestBranch
- nightly_pgo = False
-
- # More 191,192,20 special casing
- if name in ('comm-1.9.1', 'comm-1.9.2', 'comm-2.0'):
- nightlyUnittestBranch = unittestBranch
- nightly_pgo = False
-
- mozilla2_nightly_factory = CCNightlyBuildFactory(
- env=platform_env,
- objdir=pf['platform_objdir'],
- platform=platform,
- hgHost=config['hghost'],
- repoPath=config['repo_path'],
- mozRepoPath=config['mozilla_repo_path'],
- buildToolsRepoPath=config['build_tools_repo_path'],
- configRepoPath=config['config_repo_path'],
- configSubDir=config['config_subdir'],
- profiledBuild=nightly_pgo,
- productName=pf['product_name'],
- mozconfig=pf['mozconfig'],
- srcMozconfig=pf.get('src_mozconfig'),
- use_mock=pf.get('use_mock'),
- stageServer=config['stage_server'],
- stageUsername=config['stage_username'],
- stageGroup=config['stage_group'],
- stageSshKey=config['stage_ssh_key'],
- stageBasePath=stageBasePath,
- stageLogBaseUrl=config.get('stage_log_base_url', None),
- stagePlatform=pf['stage_platform'],
- stageProduct=pf['stage_product'],
- codesighs=False,
- doBuildAnalysis=doBuildAnalysis,
- uploadPackages=uploadPackages,
- uploadSymbols=pf.get('upload_symbols', False),
- nightly=True,
- createSnippet=create_snippet,
- createPartial=pf.get('create_partial', config['create_partial']),
- updatePlatform=pf['update_platform'],
- ausUser=config['aus2_user'],
- ausSshKey=config['aus2_ssh_key'],
- ausHost=config['aus2_host'],
- hashType=config['hash_type'],
- buildSpace=buildSpace,
- clobberURL=config['base_clobber_url'],
- clobberTime=clobberTime,
- buildsBeforeReboot=pf['builds_before_reboot'],
- talosMasters=talosMasters,
- packageTests=packageTests,
- unittestMasters=pf.get('unittest_masters', config['unittest_masters']),
- unittestBranch=nightlyUnittestBranch,
- triggerBuilds=config['enable_l10n'],
- triggeredSchedulers=triggeredSchedulers,
- tinderboxBuildsDir=tinderboxBuildsDir,
- enable_ccache=pf.get('enable_ccache', False),
- useSharedCheckouts=pf.get('enable_shared_checkouts', False),
- testPrettyNames=pf.get('test_pretty_names', False),
- l10nCheckTest=pf.get('l10n_check_test', False),
- android_signing=pf.get('android_signing', False),
- post_upload_include_platform=pf.get('post_upload_include_platform', False),
- signingServers=secrets.get(pf.get('nightly_signing_servers')),
- baseMirrorUrls=config.get('base_mirror_urls'),
- baseBundleUrls=config.get('base_bundle_urls'),
- branchName=name,
- **nightly_kwargs
- )
-
- mozilla2_nightly_builder = {
- 'name': nightly_builder,
- 'slavenames': pf['slaves'],
- 'builddir': '%s-%s-nightly' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-nightly' % (name, platform), pf['stage_product']),
- 'factory': mozilla2_nightly_factory,
- 'category': name,
- 'nextSlave': lambda b, sl: _nextFastSlave(b, sl, only_fast=True),
- 'properties': {'branch': name,
- 'platform': platform,
- 'stage_platform': stage_platform,
- 'product': pf['stage_product'],
- 'nightly_build': True,
- 'slavebuilddir': reallyShort('%s-%s-nightly' % (name, platform), pf['stage_product'])},
- }
- branchObjects['builders'].append(mozilla2_nightly_builder)
-
- if config['enable_l10n']:
- if platform in config['l10n_platforms']:
- # TODO Linux and mac are not working with mozconfig at this point
- # and this will disable it for now. We will fix this in bug 518359.
- objdir = ''
- mozconfig = None
-
- mozilla2_l10n_nightly_factory = CCNightlyRepackFactory(
- env=platform_env,
- objdir=objdir,
- platform=platform,
- hgHost=config['hghost'],
- tree=config['l10n_tree'],
- project=pf['product_name'],
- appName=pf['app_name'],
- enUSBinaryURL=config['enUS_binaryURL'],
- nightly=True,
- configRepoPath=config['config_repo_path'],
- configSubDir=config['config_subdir'],
- mozconfig=mozconfig,
- l10nNightlyUpdate=config['l10nNightlyUpdate'],
- l10nDatedDirs=config['l10nDatedDirs'],
- createPartial=config['create_partial_l10n'],
- ausBaseUploadDir=config['aus2_base_upload_dir_l10n'],
- updatePlatform=pf['update_platform'],
- downloadBaseURL=config['download_base_url'],
- ausUser=config['aus2_user'],
- ausSshKey=config['aus2_ssh_key'],
- ausHost=config['aus2_host'],
- hashType=config['hash_type'],
- stageServer=config['stage_server'],
- stageUsername=config['stage_username'],
- stageSshKey=config['stage_ssh_key'],
- repoPath=config['repo_path'],
- mozRepoPath=config['mozilla_repo_path'],
- l10nRepoPath=config['l10n_repo_path'],
- buildToolsRepoPath=config['build_tools_repo_path'],
- compareLocalesRepoPath=config['compare_locales_repo_path'],
- compareLocalesTag=config['compare_locales_tag'],
- buildSpace=l10nSpace,
- clobberURL=config['base_clobber_url'],
- clobberTime=clobberTime,
- signingServers=secrets.get(pf.get('nightly_signing_servers')),
- baseMirrorUrls=config.get('base_mirror_urls'),
- extraConfigureArgs=config.get('l10n_extra_configure_args', []),
- branchName=name,
- )
- mozilla2_l10n_nightly_builder = {
- 'name': l10nNightlyBuilders[nightly_builder]['l10n_builder'],
- 'slavenames': config['l10n_slaves'][platform],
- 'builddir': '%s-%s-l10n-nightly' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-l10n-nightly' % (name, platform), pf['stage_product']),
- 'factory': mozilla2_l10n_nightly_factory,
- 'category': name,
- 'nextSlave': _nextL10nSlave(),
- 'properties': {'branch': name,
- 'platform': platform,
- 'product': pf['stage_product'],
- 'stage_platform': stage_platform,
- 'slavebuilddir': reallyShort('%s-%s-l10n-nightly' % (name, platform), pf['stage_product']),},
- }
- branchObjects['builders'].append(mozilla2_l10n_nightly_builder)
-
- if config['enable_shark'] and pf.get('enable_shark'):
- if name in ('comm-1.9.1','comm-1.9.2'):
- shark_objdir = config['objdir']
- else:
- shark_objdir = pf['platform_objdir']
- mozilla2_shark_factory = CCNightlyBuildFactory(
- env=platform_env,
- objdir=shark_objdir,
- platform=platform,
- stagePlatform=stage_platform,
- hgHost=config['hghost'],
- repoPath=config['repo_path'],
- mozRepoPath=config['mozilla_repo_path'],
- buildToolsRepoPath=config['build_tools_repo_path'],
- configRepoPath=config['config_repo_path'],
- configSubDir=config['config_subdir'],
- profiledBuild=False,
- productName=pf['product_name'],
- mozconfig='%s/%s/shark' % (platform, name),
- srcMozconfig=pf.get('src_shark_mozconfig'),
- stageServer=config['stage_server'],
- stageUsername=config['stage_username'],
- stageGroup=config['stage_group'],
- stageSshKey=config['stage_ssh_key'],
- stageBasePath=stageBasePath,
- stageLogBaseUrl=config.get('stage_log_base_url', None),
- stageProduct=pf.get('stage_product'),
- codesighs=False,
- uploadPackages=uploadPackages,
- uploadSymbols=False,
- nightly=True,
- createSnippet=False,
- buildSpace=buildSpace,
- clobberURL=config['base_clobber_url'],
- clobberTime=clobberTime,
- buildsBeforeReboot=pf['builds_before_reboot'],
- post_upload_include_platform=pf.get('post_upload_include_platform', False),
- branchName=name,
- )
- mozilla2_shark_builder = {
- 'name': '%s shark' % pf['base_name'],
- 'slavenames': pf['slaves'],
- 'builddir': '%s-%s-shark' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-shark' % (name, platform), pf['stage_product']),
- 'factory': mozilla2_shark_factory,
- 'category': name,
- 'nextSlave': _nextSlowSlave,
- 'properties': {'branch': name,
- 'platform': platform,
- 'stage_platform': stage_platform,
- 'product': pf['stage_product'],
- 'slavebuilddir': reallyShort('%s-%s-shark' % (name, platform), pf['stage_product'])},
- }
- branchObjects['builders'].append(mozilla2_shark_builder)
- if config['enable_valgrind'] and \
- platform in config['valgrind_platforms']:
- valgrind_env=platform_env.copy()
- valgrind_env['REVISION'] = WithProperties("%(revision)s")
- mozilla2_valgrind_factory = ScriptFactory(
- "%s%s" % (config['hgurl'],config['build_tools_repo_path']),
- 'scripts/valgrind/valgrind.sh',
- )
- mozilla2_valgrind_builder = {
- 'name': '%s valgrind' % pf['base_name'],
- 'slavenames': pf['slaves'],
- 'builddir': '%s-%s-valgrind' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-valgrind' % (name, platform), pf['stage_product']),
- 'factory': mozilla2_valgrind_factory,
- 'category': name,
- 'env': valgrind_env,
- 'nextSlave': _nextSlowSlave,
- 'properties': {'branch': name,
- 'platform': platform,
- 'stage_platform': stage_platform,
- 'product': pf['stage_product'],
- 'slavebuilddir': reallyShort('%s-%s-valgrind' % (name, platform), pf['stage_product'])},
- }
- branchObjects['builders'].append(mozilla2_valgrind_builder)
-
- # We still want l10n_dep builds if nightlies are off
- if config['enable_l10n'] and platform in config['l10n_platforms'] and \
- config['enable_l10n_onchange']:
- mozilla2_l10n_dep_factory = CCNightlyRepackFactory(
- env=platform_env,
- platform=platform,
- hgHost=config['hghost'],
- tree=config['l10n_tree'],
- project=pf['product_name'],
- appName=pf['app_name'],
- enUSBinaryURL=config['enUS_binaryURL'],
- nightly=False,
- l10nDatedDirs=config['l10nDatedDirs'],
- stageServer=config['stage_server'],
- stageUsername=config['stage_username'],
- stageSshKey=config['stage_ssh_key'],
- repoPath=config['repo_path'],
- mozRepoPath=config['mozilla_repo_path'],
- l10nRepoPath=config['l10n_repo_path'],
- buildToolsRepoPath=config['build_tools_repo_path'],
- compareLocalesRepoPath=config['compare_locales_repo_path'],
- compareLocalesTag=config['compare_locales_tag'],
- buildSpace=l10nSpace,
- clobberURL=config['base_clobber_url'],
- clobberTime=clobberTime,
- signingServers=secrets.get(pf.get('dep_signing_servers')),
- baseMirrorUrls=config.get('base_mirror_urls'),
- extraConfigureArgs=config.get('l10n_extra_configure_args', []),
- branchName=name,
- )
- mozilla2_l10n_dep_builder = {
- 'name': l10nBuilders[pf['base_name']]['l10n_builder'],
- 'slavenames': config['l10n_slaves'][platform],
- 'builddir': '%s-%s-l10n-dep' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-l10n-dep' % (name, platform), pf['stage_product']),
- 'factory': mozilla2_l10n_dep_factory,
- 'category': name,
- 'nextSlave': _nextL10nSlave(),
- 'properties': {'branch': name,
- 'platform': platform,
- 'stage_platform': stage_platform,
- 'product': pf['stage_product'],
- 'slavebuilddir': reallyShort('%s-%s-l10n-dep' % (name, platform), pf['stage_product'])},
- }
- branchObjects['builders'].append(mozilla2_l10n_dep_builder)
-
- if pf.get('enable_unittests'):
- runA11y = True
- if platform.startswith('macosx'):
- runA11y = config['enable_mac_a11y']
-
- extra_args = {}
- if config.get('enable_try'):
- factory_class = TryUnittestBuildFactory
- extra_args['branchName'] = name
- else:
- factory_class = CCUnittestBuildFactory
-
- unittest_factory = factory_class(
- env=pf.get('unittest-env', {}),
- platform=platform,
- productName=pf['product_name'],
- config_repo_path=config['config_repo_path'],
- config_dir=config['config_subdir'],
- objdir=config['objdir_unittests'],
- hgHost=config['hghost'],
- repoPath=config['repo_path'],
- mozRepoPath=config['mozilla_repo_path'],
- buildToolsRepoPath=config['build_tools_repo_path'],
- buildSpace=config['unittest_build_space'],
- clobberURL=config['base_clobber_url'],
- clobberTime=clobberTime,
- buildsBeforeReboot=pf['builds_before_reboot'],
- run_a11y=runA11y,
- mochitest_leak_threshold=mochitestLeakThreshold,
- crashtest_leak_threshold=crashtestLeakThreshold,
- stageServer=config['stage_server'],
- stageUsername=config['stage_username'],
- stageSshKey=config['stage_ssh_key'],
- unittestMasters=config['unittest_masters'],
- unittestBranch="%s-%s-unittest" % (name, platform),
- uploadPackages=True,
- exec_xpcshell_suites = config['unittest_exec_xpcshell_suites'],
- exec_reftest_suites = config['unittest_exec_reftest_suites'],
- exec_mochi_suites = config['unittest_exec_mochi_suites'],
- exec_mozmill_suites = config['unittest_exec_mozmill_suites'],
- mochichrome_leak_threshold=mochichromeLeakThreshold,
- mochibrowser_leak_threshold=mochibrowserLeakThreshold,
- branchName=name,
- **extra_args
- )
- unittest_builder = {
- 'name': '%s unit test' % pf['base_name'],
- 'slavenames': pf['slaves'],
- 'builddir': '%s-%s-unittest' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-unittest' % (name, platform), pf['stage_product']),
- 'factory': unittest_factory,
- 'category': name,
- 'nextSlave': _nextFastSlave,
- 'properties': {'branch': name,
- 'platform': platform,
- 'stage_platform': stage_platform,
- 'product': pf['stage_product'],
- 'slavebuilddir': reallyShort('%s-%s-unittest' % (name, platform), pf['stage_product'])},
- }
- branchObjects['builders'].append(unittest_builder)
-
- for suites_name, suites in config['unittest_suites']:
- runA11y = True
- if platform.startswith('macosx'):
- runA11y = config['enable_mac_a11y']
-
- # For the regular unittest build, run the a11y suite if
- # enable_mac_a11y is set on mac
- if not runA11y and 'mochitest-a11y' in suites:
- suites = suites[:]
- suites.remove('mochitest-a11y')
-
- if pf.get('enable_unittests'):
- branchObjects['builders'].extend(generateTestBuilder(
- config, name, platform, "%s test" % pf['base_name'],
- "%s-%s-unittest" % (name, platform),
- suites_name, suites, mochitestLeakThreshold,
- crashtestLeakThreshold, stagePlatform=stage_platform,
- stageProduct=pf['stage_product']))
-
- # Remove mochitest-a11y from other types of builds, since they're not
- # built with a11y enabled
- if platform.startswith("macosx") and 'mochitest-a11y' in suites:
- # Create a new factory that doesn't have mochitest-a11y
- suites = suites[:]
- suites.remove('mochitest-a11y')
-
- if pf.get('enable_opt_unittests'):
- branchObjects['builders'].extend(generateTestBuilder(
- config, name, platform, "%s opt test" % pf['base_name'],
- "%s-%s-opt-unittest" % (name, platform),
- suites_name, suites, mochitestLeakThreshold,
- crashtestLeakThreshold, stagePlatform=stage_platform,
- stageProduct=pf['stage_product']))
-
- if config['enable_codecoverage']:
- # We only do code coverage builds on linux right now
- if platform == 'linux':
- codecoverage_factory = CodeCoverageFactory(
- platform=platform,
- productName=pf['product_name'],
- config_repo_path=config['config_repo_path'],
- config_dir=config['config_subdir'],
- objdir=config['objdir_unittests'],
- hgHost=config['hghost'],
- repoPath=config['repo_path'],
- buildToolsRepoPath=config['build_tools_repo_path'],
- buildSpace=7,
- clobberURL=config['base_clobber_url'],
- clobberTime=clobberTime,
- buildsBeforeReboot=pf['builds_before_reboot'],
- mochitest_leak_threshold=mochitestLeakThreshold,
- crashtest_leak_threshold=crashtestLeakThreshold,
- stageServer=config['stage_server'],
- stageUsername=config['stage_username'],
- stageSshKey=config['stage_ssh_key'],
- )
- codecoverage_builder = {
- 'name': '%s code coverage' % pf['base_name'],
- 'slavenames': pf['slaves'],
- 'builddir': '%s-%s-codecoverage' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-codecoverage' % (name, platform), pf['stage_product']),
- 'factory': codecoverage_factory,
- 'category': name,
- 'nextSlave': _nextSlowSlave,
- 'properties': {'branch': name, 'platform': platform, 'slavebuilddir': reallyShort('%s-%s-codecoverage' % (name, platform), pf['stage_product'])},
- }
- branchObjects['builders'].append(codecoverage_builder)
-
- if config.get('enable_blocklist_update', False):
- if platform == 'linux':
- blocklistBuilder = generateBlocklistBuilder(config, name, platform, pf['base_name'], pf['slaves'])
- branchObjects['builders'].append(blocklistBuilder)
-
- if pf.get('enable_xulrunner', config['enable_xulrunner']):
- xr_env = pf['env'].copy()
- xr_env['SYMBOL_SERVER_USER'] = config['stage_username_xulrunner']
- xr_env['SYMBOL_SERVER_PATH'] = config['symbol_server_xulrunner_path']
- xr_env['SYMBOL_SERVER_SSH_KEY'] = \
- xr_env['SYMBOL_SERVER_SSH_KEY'].replace(config['stage_ssh_key'], config['stage_ssh_xulrunner_key'])
- if pf.has_key('xr_mozconfig'):
- mozconfig = pf['xr_mozconfig']
- else:
- mozconfig = '%s/%s/xulrunner' % (platform, name)
- xulrunnerStageBasePath = '%s/xulrunner' % config['stage_base_path']
- mozilla2_xulrunner_factory = NightlyBuildFactory(
- env=xr_env,
- objdir=pf['platform_objdir'],
- platform=platform,
- hgHost=config['hghost'],
- repoPath=config['repo_path'],
- buildToolsRepoPath=config['build_tools_repo_path'],
- configRepoPath=config['config_repo_path'],
- configSubDir=config['config_subdir'],
- profiledBuild=False,
- productName='xulrunner',
- mozconfig=mozconfig,
- srcMozconfig=pf.get('src_xulrunner_mozconfig'),
- stageServer=config['stage_server'],
- stageUsername=config['stage_username_xulrunner'],
- stageGroup=config['stage_group'],
- stageSshKey=config['stage_ssh_xulrunner_key'],
- stageBasePath=xulrunnerStageBasePath,
- codesighs=False,
- uploadPackages=uploadPackages,
- uploadSymbols=True,
- nightly=True,
- createSnippet=False,
- buildSpace=buildSpace,
- clobberURL=config['base_clobber_url'],
- clobberTime=clobberTime,
- buildsBeforeReboot=pf['builds_before_reboot'],
- packageSDK=True,
- signingServers=secrets.get(pf.get('nightly_signing_servers')),
- )
- mozilla2_xulrunner_builder = {
- 'name': '%s xulrunner' % pf['base_name'],
- 'slavenames': pf['slaves'],
- 'builddir': '%s-%s-xulrunner' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-xulrunner' % (name, platform), pf['stage_product']),
- 'factory': mozilla2_xulrunner_factory,
- 'category': name,
- 'nextSlave': _nextSlowSlave,
- 'properties': {'branch': name, 'platform': platform, 'slavebuilddir': reallyShort('%s-%s-xulrunner' % (name, platform), pf['stage_product'])},
- }
- branchObjects['builders'].append(mozilla2_xulrunner_builder)
-
- # -- end of per-platform loop --
-
- if config['enable_weekly_bundle']:
- stageBasePath = '%s/%s' % (config['stage_base_path'],
- pf['stage_product'])
- bundle_factory = ScriptFactory(
- config['hgurl'] + config['build_tools_repo_path'],
- 'scripts/bundle/hg-bundle.sh',
- interpreter='bash',
- script_timeout=3600,
- script_maxtime=3600,
- extra_args=[
- name,
- config['repo_path'],
- config['stage_server'],
- config['stage_username'],
- stageBasePath,
- config['stage_ssh_key'],
- ],
- )
- slaves = set()
- for p in sorted(config['platforms'].keys()):
- slaves.update(set(config['platforms'][p]['slaves']))
- bundle_builder = {
- 'name': '%s hg bundle' % name,
- 'slavenames': list(slaves),
- 'builddir': '%s-bundle' % (name,),
- 'slavebuilddir': reallyShort('%s-bundle' % (name,)),
- 'factory': bundle_factory,
- 'category': name,
- 'nextSlave': _nextSlowSlave,
- 'properties': {'slavebuilddir': reallyShort('%s-bundle' % (name,))}
- }
- branchObjects['builders'].append(bundle_builder)
-
- return branchObjects
-
-
def generateTalosBranchObjects(branch, branch_config, PLATFORMS, SUITES,
ACTIVE_UNITTEST_PLATFORMS, factory_class=TalosFactory):
branchObjects = {'schedulers': [], 'builders': [], 'status': [], 'change_source': []}
1,562 process/factory.py
View
@@ -22,7 +22,6 @@
import buildbotcustom.steps.release
import buildbotcustom.steps.source
import buildbotcustom.steps.test
-import buildbotcustom.steps.transfer
import buildbotcustom.steps.updates
import buildbotcustom.steps.talos
import buildbotcustom.steps.unittest
@@ -39,7 +38,6 @@
reload(buildbotcustom.steps.release)
reload(buildbotcustom.steps.source)
reload(buildbotcustom.steps.test)
-reload(buildbotcustom.steps.transfer)
reload(buildbotcustom.steps.updates)
reload(buildbotcustom.steps.talos)
reload(buildbotcustom.steps.unittest)
@@ -54,11 +52,11 @@
from buildbotcustom.steps.base import ShellCommand, SetProperty, Mercurial, \
Trigger, RetryingShellCommand, RetryingSetProperty
from buildbotcustom.steps.misc import TinderboxShellCommand, SendChangeStep, \
- GetBuildID, MozillaClobberer, FindFile, DownloadFile, UnpackFile, \
+ MozillaClobberer, FindFile, DownloadFile, UnpackFile, \
SetBuildProperty, DisconnectStep, OutputStep, MockCommand, \
RepackPartners, UnpackTest, FunctionalStep, setBuildIDProps, \
RetryingMockProperty, MockInit, MockInstall
-from buildbotcustom.steps.release import UpdateVerify, SnippetComparison
+from buildbotcustom.steps.release import SnippetComparison
from buildbotcustom.steps.source import MercurialCloneCommand
from buildbotcustom.steps.test import AliveTest, \
CompareLeakLogs, Codesighs, GraphServerPost
@@ -2164,145 +2162,6 @@ def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
sendchange_props=sendchange_props,
))
-class CCMercurialBuildFactory(MercurialBuildFactory):
- def __init__(self, skipBlankRepos=False, mozRepoPath='',
- inspectorRepoPath='', venkmanRepoPath='',
- chatzillaRepoPath='', cvsroot='', **kwargs):
- self.skipBlankRepos = skipBlankRepos
- self.mozRepoPath = mozRepoPath
- self.inspectorRepoPath = inspectorRepoPath
- self.venkmanRepoPath = venkmanRepoPath
- self.chatzillaRepoPath = chatzillaRepoPath
- self.cvsroot = cvsroot
- MercurialBuildFactory.__init__(self, mozillaDir='mozilla',
- mozconfigBranch='default', **kwargs)
-
- def addSourceSteps(self):
- # First set our revisions, if no property by the name, use 'default'
- comm_rev = WithProperties("%(polled_comm_revision:-default)s")
- moz_rev = WithProperties("%(polled_moz_revision:-default)s")
-
- if self.useSharedCheckouts:
- self.addStep(JSONPropertiesDownload(
- name="download_props",
- slavedest="buildprops.json",
- workdir='.'
- ))
-
- stepCC = self.makeHgtoolStep(
- wc='build',
- workdir='.',
- rev=comm_rev,
- )
- self.addStep(stepCC)
-
- stepMC = self.makeHgtoolStep(
- name="moz_hg_update",
- wc='build%s' % self.mozillaDir,
- workdir='.',
- rev=moz_rev,
- repo_url=self.getRepository(self.mozRepoPath),
- )
- self.addStep(stepMC)
- else:
- self.addStep(Mercurial(
- name='hg_update',
- mode='update',
- baseURL='http://%s/' % self.hgHost,
- defaultBranch=self.repoPath,
- alwaysUseLatest=True,
- timeout=60*60 # 1 hour
- ))
-
- if self.buildRevision:
- self.addStep(ShellCommand(
- name='hg_update',
- command=['hg', 'up', '-C', '-r', self.buildRevision],
- haltOnFailure=True
- ))
- self.addStep(SetProperty(
- name='set_got_revision',
- command=['hg', 'identify', '-i'],
- property='got_revision'
- ))
- #Fix for bug 612319 to correct http://ssh:// changeset links
- if self.hgHost[0:5] == "ssh://":
- changesetLink = '<a href=https://%s/%s/rev' % (self.hgHost[6:],
- self.repoPath)
- else:
- changesetLink = '<a href=http://%s/%s/rev' % (self.hgHost,
- self.repoPath)
- changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
- self.addStep(OutputStep(
- name='tinderboxprint_changeset',
- data=['TinderboxPrint:', WithProperties(changesetLink)]
- ))
- # build up the checkout command with all options
- co_command = ['python', 'client.py', 'checkout']
- # comm-* is handled by code above, no need to do network churn here
- co_command.append("--skip-comm")
- if (not self.useSharedCheckouts) and self.mozRepoPath:
- co_command.append('--mozilla-repo=%s' % self.getRepository(self.mozRepoPath))
- if self.inspectorRepoPath:
- co_command.append('--inspector-repo=%s' % self.getRepository(self.inspectorRepoPath))
- elif self.skipBlankRepos:
- co_command.append('--skip-inspector')
- if self.venkmanRepoPath:
- co_command.append('--venkman-repo=%s' % self.getRepository(self.venkmanRepoPath))
- elif self.skipBlankRepos:
- co_command.append('--skip-venkman')
- if self.chatzillaRepoPath:
- co_command.append('--chatzilla-repo=%s' % self.getRepository(self.chatzillaRepoPath))
- elif self.skipBlankRepos:
- co_command.append('--skip-chatzilla')
- if self.cvsroot:
- co_command.append('--cvsroot=%s' % self.cvsroot)
- if self.buildRevision:
- co_command.append('--mozilla-rev=%s' % self.buildRevision)
- co_command.append('--inspector-rev=%s' % self.buildRevision)
- co_command.append('--venkman-rev=%s' % self.buildRevision)
- co_command.append('--chatzilla-rev=%s' % self.buildRevision)
- # execute the checkout
- self.addStep(ShellCommand(
- command=co_command,
- description=['running', 'client.py', 'checkout'],
- descriptionDone=['client.py', 'checkout'],
- haltOnFailure=True,
- timeout=60*60 # 1 hour
- ))
-
- self.addStep(SetProperty(
- name='set_hg_revision',
- command=['hg', 'identify', '-i'],
- workdir='build%s' % self.mozillaDir,
- property='hg_revision'
- ))
- #Fix for bug 612319 to correct http://ssh:// changeset links
- if self.hgHost[0:5] == "ssh://":
- changesetLink = '<a href=https://%s/%s/rev' % (self.hgHost[6:],
- self.mozRepoPath)
- else:
- changesetLink = '<a href=http://%s/%s/rev' % (self.hgHost,
- self.mozRepoPath)
- changesetLink += '/%(hg_revision)s title="Built from Mozilla revision %(hg_revision)s">moz:%(hg_revision)s</a>'
- self.addStep(OutputStep(
- name='tinderboxprint_changeset',
- data=['TinderboxPrint:', WithProperties(changesetLink)]
- ))
- self.addStep(SetBuildProperty(
- name='set_comments',
- property_name="comments",
- value=lambda build:build.source.changes[-1].comments if len(build.source.changes) > 0 else "",
- ))
-
- def addUploadSteps(self, pkgArgs=None):
- MercurialBuildFactory.addUploadSteps(self, pkgArgs)
- self.addStep(ShellCommand(
- command=['make', 'package-compare'],
- workdir='build/%s' % self.objdir,
- haltOnFailure=False
- ))
-
def marFilenameToProperty(prop_name=None):
'''Parse a file listing and return the first mar filename found as
@@ -2757,32 +2616,6 @@ def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
))
-class CCNightlyBuildFactory(CCMercurialBuildFactory, NightlyBuildFactory):
- def __init__(self, skipBlankRepos=False, mozRepoPath='',
- inspectorRepoPath='', venkmanRepoPath='',
- chatzillaRepoPath='', cvsroot='', **kwargs):
- self.skipBlankRepos = skipBlankRepos
- self.mozRepoPath = mozRepoPath
- self.inspectorRepoPath = inspectorRepoPath
- self.venkmanRepoPath = venkmanRepoPath
- self.chatzillaRepoPath = chatzillaRepoPath
- self.cvsroot = cvsroot
- NightlyBuildFactory.__init__(self, mozillaDir='mozilla',
- mozconfigBranch='default', **kwargs)
-
- # MercurialBuildFactory defines those, and our inheritance chain makes us
- # look there before NightlyBuildFactory, so we need to define them here and
- # call the actually wanted implementation.
- def addCreateUpdateSteps(self):
- NightlyBuildFactory.addCreateUpdateSteps(self)
-
- def addCreateSnippetsSteps(self, milestone_extra=''):
- NightlyBuildFactory.addCreateSnippetsSteps(self, milestone_extra)
-
- def addUploadSnippetsSteps(self):
- NightlyBuildFactory.addUploadSnippetsSteps(self)
-
-
class ReleaseBuildFactory(MercurialBuildFactory):
def __init__(self, env, version, buildNumber, brandName=None,
unittestMasters=None, unittestBranch=None, talosMasters=None,
@@ -3080,24 +2913,6 @@ def get_url(rc, stdout, stderr):
log_eval_func=lambda c,s: regex_log_evaluator(c, s, upload_errors),
))
-class CCReleaseBuildFactory(CCMercurialBuildFactory, ReleaseBuildFactory):
- def __init__(self, mozRepoPath='', inspectorRepoPath='',
- venkmanRepoPath='', chatzillaRepoPath='', cvsroot='',
- **kwargs):
- self.skipBlankRepos = True
- self.mozRepoPath = mozRepoPath
- self.inspectorRepoPath = inspectorRepoPath
- self.venkmanRepoPath = venkmanRepoPath
- self.chatzillaRepoPath = chatzillaRepoPath
- self.cvsroot = cvsroot
- ReleaseBuildFactory.__init__(self, mozillaDir='mozilla',
- mozconfigBranch='default', **kwargs)
-
- def addFilePropertiesSteps(self, filename=None, directory=None,
- fileType=None, maxDepth=1, haltOnFailure=False):
- # We don't need to do this for release builds.
- pass
-
def identToProperties(default_prop=None):
'''Create a method that is used in a SetProperty step to map the
@@ -3598,62 +3413,6 @@ def doTestPrettyNames(self):
workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
))
-class CCBaseRepackFactory(BaseRepackFactory):
- # Override ignore_dirs so that we don't delete l10n nightly builds
- # before running a l10n nightly build
- ignore_dirs = MozillaBuildFactory.ignore_dirs + ['*-nightly']
-
- def __init__(self, skipBlankRepos=False, mozRepoPath='',
- inspectorRepoPath='', venkmanRepoPath='',
- chatzillaRepoPath='', cvsroot='', buildRevision='',
- **kwargs):
- self.skipBlankRepos = skipBlankRepos
- self.mozRepoPath = mozRepoPath
- self.inspectorRepoPath = inspectorRepoPath
- self.venkmanRepoPath = venkmanRepoPath
- self.chatzillaRepoPath = chatzillaRepoPath
- self.cvsroot = cvsroot
- self.buildRevision = buildRevision
- BaseRepackFactory.__init__(self, mozillaDir='mozilla',
- mozconfigBranch='default', **kwargs)
-
- def getSources(self):
- BaseRepackFactory.getSources(self)
- # build up the checkout command with all options
- co_command = ['python', 'client.py', 'checkout',
- WithProperties('--comm-rev=%(en_revision)s')]
- if self.mozRepoPath:
- co_command.append('--mozilla-repo=%s' % self.getRepository(self.mozRepoPath))
- if self.inspectorRepoPath:
- co_command.append('--inspector-repo=%s' % self.getRepository(self.inspectorRepoPath))
- elif self.skipBlankRepos:
- co_command.append('--skip-inspector')
- if self.venkmanRepoPath:
- co_command.append('--venkman-repo=%s' % self.getRepository(self.venkmanRepoPath))
- elif self.skipBlankRepos:
- co_command.append('--skip-venkman')
- if self.chatzillaRepoPath:
- co_command.append('--chatzilla-repo=%s' % self.getRepository(self.chatzillaRepoPath))
- elif self.skipBlankRepos:
- co_command.append('--skip-chatzilla')
- if self.cvsroot:
- co_command.append('--cvsroot=%s' % self.cvsroot)
- if self.buildRevision:
- co_command.append('--comm-rev=%s' % self.buildRevision)
- co_command.append('--mozilla-rev=%s' % self.buildRevision)
- co_command.append('--inspector-rev=%s' % self.buildRevision)
- co_command.append('--venkman-rev=%s' % self.buildRevision)
- co_command.append('--chatzilla-rev=%s' % self.buildRevision)
- # execute the checkout
- self.addStep(ShellCommand(
- command=co_command,
- description=['running', 'client.py', 'checkout'],
- descriptionDone=['client.py', 'checkout'],
- haltOnFailure=True,
- workdir='%s/%s' % (self.baseWorkDir, self.origSrcDir),
- timeout=60*60*3 # 3 hours (crazy, but necessary for now)
- ))
-
class NightlyRepackFactory(BaseRepackFactory, NightlyBuildFactory):
extraConfigureArgs = []
@@ -3940,68 +3699,6 @@ def doRepack(self):
self.addCreatePartialUpdateSteps(extraArgs=[WithProperties('AB_CD=%(locale)s')])
-class CCNightlyRepackFactory(CCBaseRepackFactory, NightlyRepackFactory):
- def __init__(self, skipBlankRepos=False, mozRepoPath='',
- inspectorRepoPath='', venkmanRepoPath='',
- chatzillaRepoPath='', cvsroot='', buildRevision='',
- **kwargs):
- self.skipBlankRepos = skipBlankRepos
- self.mozRepoPath = mozRepoPath
- self.inspectorRepoPath = inspectorRepoPath
- self.venkmanRepoPath = venkmanRepoPath
- self.chatzillaRepoPath = chatzillaRepoPath
- self.cvsroot = cvsroot
- self.buildRevision = buildRevision
- NightlyRepackFactory.__init__(self, mozillaDir='mozilla',
- mozconfigBranch='default', **kwargs)
-
- # it sucks to override all of updateEnUS but we need to do it that way
- # this is basically mirroring what mobile does
- def updateEnUS(self):
- '''Update en-US to the source stamp we get from make ident.
-
- Requires that we run make unpack first.
- '''
- self.addStep(ShellCommand(
- name='make_unpack',
- command=['make', 'unpack'],
- descriptionDone='unpacked en-US',
- haltOnFailure=True,
- env=self.env,
- workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
- ))
-
- self.addStep(SetProperty(
- command=['make', 'ident'],
- haltOnFailure=True,
- workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
- extract_fn=identToProperties()
- ))
- self.addStep(ShellCommand(
- name='update_comm_enUS_revision',
- command=['hg', 'update', '-C', '-r',
- WithProperties('%(comm_revision)s')],
- haltOnFailure=True,
- workdir='%s/%s' % (self.baseWorkDir, self.origSrcDir)))
- self.addStep(ShellCommand(
- name='update_mozilla_enUS_revision',
- command=['hg', 'update', '-C', '-r',
- WithProperties('%(moz_revision)s')],
- haltOnFailure=True,
- workdir='%s/%s' % (self.baseWorkDir, self.mozillaSrcDir)))
-
- def tinderboxPrintRevisions(self):
- self.tinderboxPrint('comm_revision',WithProperties('%(comm_revision)s'))
- self.tinderboxPrint('moz_revision',WithProperties('%(moz_revision)s'))
- self.tinderboxPrint('l10n_revision',WithProperties('%(l10n_revision)s'))
-
- # BaseRepackFactory defines that, and our inheritance chain makes us look
- # there before NightlyRepackFactory, so we need to define it here and call
- # the actually wanted implementation.
- def doRepack(self):
- NightlyRepackFactory.doRepack(self)
-
-
class ReleaseFactory(MozillaBuildFactory):
def getCandidatesDir(self, product, version, buildNumber,
nightlyDir="nightly"):
@@ -4025,249 +3722,6 @@ def makeLongVersion(self, version):
version = re.sub('rc([0-9]+)$', ' RC \\1', version)
return version
-class CCReleaseRepackFactory(CCBaseRepackFactory, ReleaseFactory):
- def __init__(self, platform, buildRevision, version, buildNumber,
- env={}, brandName=None, mergeLocales=False,
- mozRepoPath='', inspectorRepoPath='', venkmanRepoPath='',
- chatzillaRepoPath='', cvsroot='', **kwargs):
- self.skipBlankRepos = True
- self.mozRepoPath = mozRepoPath
- self.inspectorRepoPath = inspectorRepoPath
- self.venkmanRepoPath = venkmanRepoPath
- self.chatzillaRepoPath = chatzillaRepoPath
- self.cvsroot = cvsroot
- self.buildRevision = buildRevision
- self.version = version
- self.buildNumber = buildNumber
- if brandName:
- self.brandName = brandName
- else:
- self.brandName = kwargs['project'].capitalize()
- # more vars are added in downloadBuilds
- env.update({
- 'MOZ_PKG_PRETTYNAMES': '1',
- 'MOZ_PKG_VERSION': self.version,
- 'MOZ_MAKE_COMPLETE_MAR': '1'
- })
-
- assert 'project' in kwargs
- # TODO: better place to put this/call this
- self.postUploadCmd = 'post_upload.py ' + \
- '-p %s ' % kwargs['project'] + \
- '-v %s ' % self.version + \
- '-n %s ' % self.buildNumber + \
- '--release-to-candidates-dir'
- BaseRepackFactory.__init__(self, env=env, platform=platform,
- mergeLocales=mergeLocales, mozillaDir='mozilla',
- mozconfigBranch='default', **kwargs)
-
- # Repeated here since the Parent classes fail hgtool/checkouts due to
- # relbranch issues, and not actually having the tag after clone
- def getSources(self):
- self.addStep(MercurialCloneCommand(
- name='get_enUS_src',
- command=['sh', '-c',
- WithProperties('if [ -d '+self.origSrcDir+'/.hg ]; then ' +
- 'hg -R '+self.origSrcDir+' pull && '+
- 'hg -R '+self.origSrcDir+' up -C ;'+
- 'else ' +
- 'hg clone ' +
- 'http://'+self.hgHost+'/'+self.repoPath+' ' +
- self.origSrcDir+' ; ' +
- 'fi ' +
- '&& hg -R '+self.origSrcDir+' update -C -r %(en_revision)s')],
- descriptionDone="en-US source",
- workdir=self.baseWorkDir,
- haltOnFailure=True,
- timeout=30*60 # 30 minutes
- ))
- self.addStep(MercurialCloneCommand(
- name='get_locale_src',
- command=['sh', '-c',
- WithProperties('if [ -d %(locale)s/.hg ]; then ' +
- 'hg -R %(locale)s pull -r default ; ' +
- 'else ' +
- 'hg clone ' +
- 'http://'+self.hgHost+'/'+self.l10nRepoPath+
- '/%(locale)s/ ; ' +
- 'fi ' +
- '&& hg -R %(locale)s update -C -r %(l10n_revision)s')],
- descriptionDone="locale source",
- timeout=10*60, # 10 minutes
- haltOnFailure=True,
- workdir='%s/%s' % (self.baseWorkDir, self.l10nRepoPath)
- ))
- # build up the checkout command with all options
- co_command = ['python', 'client.py', 'checkout',
- WithProperties('--comm-rev=%(en_revision)s')]
- if self.mozRepoPath:
- co_command.append('--mozilla-repo=%s' % self.getRepository(self.mozRepoPath))
- if self.inspectorRepoPath:
- co_command.append('--inspector-repo=%s' % self.getRepository(self.inspectorRepoPath))
- elif self.skipBlankRepos:
- co_command.append('--skip-inspector')
- if self.venkmanRepoPath:
- co_command.append('--venkman-repo=%s' % self.getRepository(self.venkmanRepoPath))
- elif self.skipBlankRepos:
- co_command.append('--skip-venkman')
- if self.chatzillaRepoPath:
- co_command.append('--chatzilla-repo=%s' % self.getRepository(self.chatzillaRepoPath))
- elif self.skipBlankRepos:
- co_command.append('--skip-chatzilla')
- if self.cvsroot:
- co_command.append('--cvsroot=%s' % self.cvsroot)
- if self.buildRevision:
- co_command.append('--comm-rev=%s' % self.buildRevision)
- co_command.append('--mozilla-rev=%s' % self.buildRevision)
- co_command.append('--inspector-rev=%s' % self.buildRevision)
- co_command.append('--venkman-rev=%s' % self.buildRevision)
- co_command.append('--chatzilla-rev=%s' % self.buildRevision)
- # execute the checkout
- self.addStep(ShellCommand(
- command=co_command,
- description=['running', 'client.py', 'checkout'],
- descriptionDone=['client.py', 'checkout'],
- haltOnFailure=True,
- workdir='%s/%s' % (self.baseWorkDir, self.origSrcDir),
- timeout=60*60*3 # 3 hours (crazy, but necessary for now)
- ))
-
- def updateSources(self):
- self.addStep(ShellCommand(
- name='update_sources',
- command=['hg', 'up', '-C', '-r', self.buildRevision],
- workdir='build/'+self.origSrcDir,
- description=['update %s' % self.branchName,
- 'to %s' % self.buildRevision],
- haltOnFailure=True
- ))
- self.addStep(ShellCommand(
- name='update_locale_sources',
- command=['hg', 'up', '-C', '-r', self.buildRevision],
- workdir=WithProperties('build/' + self.l10nRepoPath + '/%(locale)s'),
- description=['update to', self.buildRevision]
- ))
- self.addStep(SetProperty(
- command=['hg', 'ident', '-i'],
- haltOnFailure=True,
- property='l10n_revision',
- workdir=WithProperties('build/' + self.l10nRepoPath +
- '/%(locale)s')
- ))
- self.addStep(ShellCommand(
- command=['hg', 'up', '-C', '-r', self.buildRevision],
- workdir='build/'+self.mozillaSrcDir,
- description=['update mozilla',
- 'to %s' % self.buildRevision],
- haltOnFailure=True
- ))
- if self.venkmanRepoPath:
- self.addStep(ShellCommand(
- command=['hg', 'up', '-C', '-r', self.buildRevision],
- workdir='build/'+self.mozillaSrcDir+'/extensions/venkman',
- description=['update venkman',
- 'to %s' % self.buildRevision],
- haltOnFailure=True
- ))
- if self.inspectorRepoPath:
- self.addStep(ShellCommand(
- command=['hg', 'up', '-C', '-r', self.buildRevision],
- workdir='build/'+self.mozillaSrcDir+'/extensions/inspector',
- description=['update inspector',
- 'to %s' % self.buildRevision],
- haltOnFailure=True
- ))
- if self.chatzillaRepoPath:
- self.addStep(ShellCommand(
- command=['hg', 'up', '-C', '-r', self.buildRevision],
- workdir='build/'+self.mozillaSrcDir+'/extensions/irc',
- description=['update chatzilla',
- 'to %s' % self.buildRevision],
- haltOnFailure=True
- ))
-
- def preClean(self):
- # We need to know the absolute path to the input builds when we repack,
- # so we need retrieve at run-time as a build property
- self.addStep(SetProperty(
- command=['bash', '-c', 'pwd'],
- property='srcdir',
- workdir='build/'+self.origSrcDir
- ))
-
- def downloadBuilds(self):
- candidatesDir = 'http://%s' % self.stageServer + \
- '/pub/mozilla.org/%s/nightly' % self.project + \
- '/%s-candidates/build%s' % (self.version,
- self.buildNumber)
-
- # This block sets platform specific data that our wget command needs.
- # build is mapping between the local and remote filenames
- # platformDir is the platform specific directory builds are contained
- # in on the stagingServer.
- # This block also sets the necessary environment variables that the
- # doRepack() steps rely on to locate their source build.
- builds = {}
- platformDir = getPlatformFtpDir(self.platform.split("-")[0])
- if self.platform.startswith('linux'):
- filename = '%s.tar.bz2' % self.project
- builds[filename] = '%s-%s.tar.bz2' % (self.project, self.version)
- self.env['ZIP_IN'] = WithProperties('%(srcdir)s/' + filename)
- elif self.platform.startswith('macosx'):
- filename = '%s.dmg' % self.project
- builds[filename] = '%s %s.dmg' % (self.brandName,
- self.version)
- self.env['ZIP_IN'] = WithProperties('%(srcdir)s/' + filename)
- elif self.platform.startswith('win32'):
- platformDir = 'unsigned/' + platformDir
- filename = '%s.zip' % self.project
- instname = '%s.exe' % self.project
- builds[filename] = '%s-%s.zip' % (self.project, self.version)
- builds[instname] = '%s Setup %s.exe' % (self.brandName,
- self.version)
- self.env['ZIP_IN'] = WithProperties('%(srcdir)s/' + filename)
- self.env['WIN32_INSTALLER_IN'] = \
- WithProperties('%(srcdir)s/' + instname)
- else:
- raise "Unsupported platform"
-
- for name in builds:
- self.addStep(ShellCommand(
- name='get_candidates_%s' % name,
- command=['wget', '-O', name, '--no-check-certificate',
- '%s/%s/en-US/%s' % (candidatesDir, platformDir,
- builds[name])],
- workdir='build/'+self.origSrcDir,
- haltOnFailure=True
- ))
-
- def doRepack(self):
- # Because we're generating updates we need to build the libmar tools
- self.addStep(ShellCommand(
- name='make_tier_nspr',
- command=['make','tier_nspr'],
- workdir='build/'+self.mozillaObjdir,
- description=['make tier_nspr'],
- haltOnFailure=True
- ))
- self.addStep(ShellCommand(
- name='make_libmar',
- command=['make'],
- workdir='build/'+self.mozillaObjdir+'/modules/libmar',
- description=['make libmar'],
- haltOnFailure=True
- ))
-
- self.addStep(ShellCommand(
- name='repack_installers',
- description=['repack', 'installers'],
- command=['sh','-c',
- WithProperties('make installers-%(locale)s LOCALE_MERGEDIR=$PWD/merged')],
- env=self.env,
- haltOnFailure=True,
- workdir='build/'+self.objdir+'/'+self.appName+'/locales'
- ))
-
class StagingRepositorySetupFactory(ReleaseFactory):
"""This Factory should be run at the start of a staging release run. It
deletes and reclones all of the repositories in 'repositories'. Note that
@@ -4486,98 +3940,6 @@ def addConfigSteps(self, workdir='build'):
))
-class CCSourceFactory(ReleaseFactory):
- def __init__(self, productName, version, baseTag, stagingServer,
- stageUsername, stageSshKey, buildNumber, mozRepoPath,
- inspectorRepoPath='', venkmanRepoPath='',
- chatzillaRepoPath='', cvsroot='', autoconfDirs=['.'],
- buildSpace=1, **kwargs):
- ReleaseFactory.__init__(self, buildSpace=buildSpace, **kwargs)
- releaseTag = '%s_RELEASE' % (baseTag)
- sourceTarball = 'source/%s-%s.source.tar.bz2' % (productName,
- version)
- # '-c' is for "release to candidates dir"
- postUploadCmd = 'post_upload.py -p %s -v %s -n %s -c' % \
- (productName, version, buildNumber)
- uploadEnv = {'UPLOAD_HOST': stagingServer,
- 'UPLOAD_USER': stageUsername,
- 'UPLOAD_SSH_KEY': '~/.ssh/%s' % stageSshKey,
- 'UPLOAD_TO_TEMP': '1',
- 'POST_UPLOAD_CMD': postUploadCmd}
-
- self.addStep(ShellCommand(
- command=['rm', '-rf', 'source'],
- workdir='.',
- haltOnFailure=True
- ))
- self.addStep(ShellCommand(
- command=['mkdir', 'source'],
- workdir='.',
- haltOnFailure=True
- ))
- self.addStep(MercurialCloneCommand(
- command=['hg', 'clone', self.repository, self.branchName],
- workdir='.',
- description=['clone %s' % self.branchName],
- haltOnFailure=True,
- timeout=30*60 # 30 minutes
- ))
- # build up the checkout command that will bring us up to the release version
- co_command = ['python', 'client.py', 'checkout',
- '--comm-rev=%s' % releaseTag,
- '--mozilla-repo=%s' % self.getRepository(mozRepoPath),
- '--mozilla-rev=%s' % releaseTag]
- if inspectorRepoPath:
- co_command.append('--inspector-repo=%s' % self.getRepository(inspectorRepoPath))
- co_command.append('--inspector-rev=%s' % releaseTag)
- else:
- co_command.append('--skip-inspector')
- if venkmanRepoPath:
- co_command.append('--venkman-repo=%s' % self.getRepository(venkmanRepoPath))
- co_command.append('--venkman-rev=%s' % releaseTag)
- else:
- co_command.append('--skip-venkman')
- if chatzillaRepoPath:
- co_command.append('--chatzilla-repo=%s' % self.getRepository(chatzillaRepoPath))
- co_command.append('--chatzilla-rev=%s' % releaseTag)
- else:
- co_command.append('--skip-chatzilla')
- if cvsroot:
- co_command.append('--cvsroot=%s' % cvsroot)
- # execute the checkout
- self.addStep(ShellCommand(
- command=co_command,
- workdir=self.branchName,
- description=['update to', releaseTag],
- haltOnFailure=True,
- timeout=60*60 # 1 hour
- ))
- # the autoconf and actual tarring steps
- # should be replaced by calling the build target
- for dir in autoconfDirs:
- self.addStep(ShellCommand(
- command=['autoconf-2.13'],
- workdir='%s/%s' % (self.branchName, dir),
- haltOnFailure=True
- ))
- self.addStep(ShellCommand(
- command=['tar', '-cj', '--owner=0', '--group=0', '--numeric-owner',
- '--mode=go-w', '--exclude=.hg*', '--exclude=CVS',
- '--exclude=.cvs*', '-f', sourceTarball, self.branchName],
- workdir='.',
- description=['create tarball'],
- haltOnFailure=True
- ))
- self.addStep(ShellCommand(
- command=['python', '%s/mozilla/build/upload.py' % self.branchName,
- '--base-path', '.', sourceTarball],
- workdir='.',
- env=uploadEnv,
- description=['upload files'],
- ))
-
-
-
class ReleaseUpdatesFactory(ReleaseFactory):
snippetStagingDir = '/opt/aus2/snippets/staging'
def __init__(self, cvsroot, patcherToolsTag, patcherConfig, verifyConfigs,
@@ -5192,21 +4554,6 @@ def getSnippetDir(self):
self.buildNumber)
-class UpdateVerifyFactory(ReleaseFactory):
- def __init__(self, verifyConfig, buildSpace=.3, useOldUpdater=False,
- **kwargs):
- ReleaseFactory.__init__(self, repoPath='nothing',
- buildSpace=buildSpace, **kwargs)
- command=['bash', 'verify.sh', '-c', verifyConfig]
- if useOldUpdater:
- command.append('--old-updater')
- self.addStep(UpdateVerify(
- command=command,
- workdir='tools/release/updates',
- description=['./verify.sh', verifyConfig]
- ))
-
-
class ReleaseFinalVerification(ReleaseFactory):
def __init__(self, verifyConfigs, platforms=None, **kwargs):
# MozillaBuildFactory needs the 'repoPath' argument, but we don't
@@ -5290,851 +4637,6 @@ def __init__(self, baseTag, appName, config, productName, version,
workdir='tools/release',
))
-class UnittestBuildFactory(MozillaBuildFactory):
- def __init__(self, platform, productName, config_repo_path, config_dir,
- objdir, mochitest_leak_threshold=None,
- crashtest_leak_threshold=None, uploadPackages=False,
- unittestMasters=None, unittestBranch=None, stageUsername=None,
- stageServer=None, stageSshKey=None, run_a11y=True,
- env={}, **kwargs):
- self.env = {}
-
- MozillaBuildFactory.__init__(self, env=env, **kwargs)
-
- self.productName = productName
- self.stageServer = stageServer
- self.stageUsername = stageUsername
- self.stageSshKey = stageSshKey
- self.uploadPackages = uploadPackages
- self.config_repo_path = config_repo_path
- self.config_dir = config_dir
- self.objdir = objdir
- self.run_a11y = run_a11y
- self.crashtest_leak_threshold = crashtest_leak_threshold
- self.mochitest_leak_threshold = mochitest_leak_threshold
-
- self.unittestMasters = unittestMasters or []
- self.unittestBranch = unittestBranch
- if self.unittestMasters:
- assert self.unittestBranch
-
- self.config_repo_url = self.getRepository(self.config_repo_path)
-
- env_map = {
- 'linux': 'linux-unittest',
- 'linux64': 'linux64-unittest',
- 'macosx': 'macosx-unittest',
- 'macosx64': 'macosx64-unittest',
- 'win32': 'win32-unittest',
- }
-
- self.platform = platform.split('-')[0]
- assert self.platform in getSupportedPlatforms()
-
- self.env = MozillaEnvironments[env_map[self.platform]].copy()
- self.env['MOZ_OBJDIR'] = self.objdir
- self.env.update(env)
-
- if self.platform == 'win32':
- self.addStep(SetProperty(
- command=['bash', '-c', 'pwd -W'],
- property='toolsdir',
- workdir='tools'
- ))
-
- self.addStep(Mercurial(
- name='hg_update',
- mode='update',
- baseURL='http://%s/' % self.hgHost,
- defaultBranch=self.repoPath,
- timeout=60*60 # 1 hour
- ))
-
- self.addPrintChangesetStep()
-
- self.addStep(ShellCommand(
- name='rm_configs',
- command=['rm', '-rf', 'mozconfigs'],
- workdir='.'
- ))
-
- self.addStep(MercurialCloneCommand(
- name='buildbot_configs',
- command=['hg', 'clone', self.config_repo_url, 'mozconfigs'],
- workdir='.'
- ))
-
- self.addCopyMozconfigStep()
-
- # TODO: Do we need this special windows rule?
- if self.platform == 'win32':
- self.addStep(ShellCommand(
- name='mozconfig_contents',
- command=["type", ".mozconfig"]
- ))
- else:
- self.addStep(ShellCommand(
- name='mozconfig_contents',
- command=['cat', '.mozconfig']
- ))
-
- self.addStep(ShellCommand(
- name='compile',
- command=["make", "-f", "client.mk", "build"],
- description=['compile'],
- timeout=60*60, # 1 hour
- haltOnFailure=1,
- env=self.env,
- ))
-
- self.addStep(ShellCommand(
- name='make_buildsymbols',
- command=['make', 'buildsymbols'],
- workdir='build/%s' % self.objdir,
- timeout=60*60,
- env=self.env,
- ))
-
- # Need to override toolsdir as set by MozillaBuildFactory because
- # we need Windows-style paths.
- if self.platform.startswith('win'):
- self.addStep(SetProperty(
- command=['bash', '-c', 'pwd -W'],
- property='toolsdir',
- workdir='tools'
- ))
-
- self.doUpload()
-
- self.env['MINIDUMP_STACKWALK'] = getPlatformMinidumpPath(self.platform)
- self.env['MINIDUMP_SAVE_PATH'] = WithProperties('%(basedir:-)s/minidumps')