Permalink
Browse files

Merge branch 'uploading' of git://github.com/catlee/buildbotcustom in…

…to uploading
  • Loading branch information...
2 parents a0146aa + fbc3acc commit e2fdf9ae34fdf3dec9558ca761a8605362155a3f @catlee committed May 9, 2012
Showing with 357 additions and 144 deletions.
  1. +46 −0 .hgtags
  2. +5 −0 bin/postrun.py
  3. +4 −1 changes/hgpoller.py
  4. +28 −21 misc.py
  5. +182 −88 process/factory.py
  6. +31 −8 process/release.py
  7. +4 −3 status/errors.py
  8. +12 −7 steps/misc.py
  9. +5 −4 steps/unittest.py
  10. +1 −1 test/test_hgpoller.py
  11. +39 −11 test/test_misc_nextslaves.py
View
46 .hgtags
@@ -439,3 +439,49 @@ f7df8a191e4550b6257308013d7fa512a4caaf94 FIREFOX_11_0b1_RELEASE
f7df8a191e4550b6257308013d7fa512a4caaf94 FIREFOX_11_0b1_BUILD1
f7df8a191e4550b6257308013d7fa512a4caaf94 FENNEC_11_0b1_RELEASE
f7df8a191e4550b6257308013d7fa512a4caaf94 FENNEC_11_0b1_BUILD1
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FENNEC_11_0b2_BUILD1
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FENNEC_11_0b2_RELEASE
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FIREFOX_11_0b2_BUILD1
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FIREFOX_11_0b2_RELEASE
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FIREFOX_10_0_1_BUILD1
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FIREFOX_10_0_1_RELEASE
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FIREFOX_10_0_1esr_BUILD1
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FIREFOX_10_0_1esr_RELEASE
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FENNEC_10_0_1_RELEASE
+9fecda5f14926c70fe3e03cd5e8c7eb925bc103c FENNEC_10_0_1_BUILD1
+c98d749d2f6fde005b532b62a3dbb1d7d470a8b6 FIREFOX_11_0b3_BUILD1
+c98d749d2f6fde005b532b62a3dbb1d7d470a8b6 FIREFOX_11_0b3_RELEASE
+c98d749d2f6fde005b532b62a3dbb1d7d470a8b6 FENNEC_11_0b3_BUILD1
+c98d749d2f6fde005b532b62a3dbb1d7d470a8b6 FENNEC_11_0b3_RELEASE
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FIREFOX_11_0b3_BUILD2
+c98d749d2f6fde005b532b62a3dbb1d7d470a8b6 FIREFOX_11_0b3_RELEASE
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FIREFOX_11_0b3_RELEASE
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FENNEC_11_0b3_BUILD2
+c98d749d2f6fde005b532b62a3dbb1d7d470a8b6 FENNEC_11_0b3_RELEASE
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FENNEC_11_0b3_RELEASE
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FIREFOX_10_0_2_BUILD1
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FIREFOX_10_0_2_RELEASE
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FENNEC_10_0_2_BUILD1
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FENNEC_10_0_2_RELEASE
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FIREFOX_10_0_2esr_RELEASE
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FIREFOX_10_0_2esr_BUILD1
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FIREFOX_3_6_27_RELEASE
+1f0195d7a54d1908ff6799c6024ce49bbb687d3f FIREFOX_3_6_27_BUILD1
+d959791e6098d2354bfd55c5743184e36ff41169 FIREFOX_11_0b4_BUILD1
+d959791e6098d2354bfd55c5743184e36ff41169 FIREFOX_11_0b4_RELEASE
+d959791e6098d2354bfd55c5743184e36ff41169 FENNEC_11_0b4_BUILD1
+d959791e6098d2354bfd55c5743184e36ff41169 FENNEC_11_0b4_RELEASE
+9c3ee89b839c25f196f793345da997d940a0b71f FIREFOX_11_0b5_BUILD1
+9c3ee89b839c25f196f793345da997d940a0b71f FIREFOX_11_0b5_RELEASE
+9c3ee89b839c25f196f793345da997d940a0b71f FENNEC_11_0b5_BUILD1
+9c3ee89b839c25f196f793345da997d940a0b71f FENNEC_11_0b5_RELEASE
+5719f96e1ebd89e7aa19d5bdb9563509fb444ba8 FIREFOX_10_0_3esr_RELEASE
+5719f96e1ebd89e7aa19d5bdb9563509fb444ba8 FIREFOX_10_0_3esr_BUILD1
+5719f96e1ebd89e7aa19d5bdb9563509fb444ba8 FENNEC_10_0_3esr_RELEASE
+5719f96e1ebd89e7aa19d5bdb9563509fb444ba8 FENNEC_10_0_3esr_BUILD1
+9214dd6ba5c6122cd6e47234a32014e207ce52f0 FIREFOX_11_0b6_RELEASE
+9214dd6ba5c6122cd6e47234a32014e207ce52f0 FIREFOX_11_0b6_BUILD1
+9214dd6ba5c6122cd6e47234a32014e207ce52f0 FENNEC_11_0b6_RELEASE
+9214dd6ba5c6122cd6e47234a32014e207ce52f0 FENNEC_11_0b6_BUILD1
+23e40889eeb6021e509dcbde272595987dfcebed FIREFOX_3_6_28_BUILD1
+23e40889eeb6021e509dcbde272595987dfcebed FIREFOX_3_6_28_RELEASE
View
@@ -167,6 +167,11 @@ def getBuildInfo(self, build):
else:
retval['product'] = None
+ # fennec builds are called 'fennec' by buildbot, but are 'mobile' as
+ # far as post_upload.py is concerned.
+ if retval['product'] == 'fennec':
+ retval['product'] = 'mobile'
+
if props.getProperty('branch') is not None:
retval['branch'] = props['branch']
else:
View
@@ -425,13 +425,15 @@ class HgAllLocalesPoller(base.ChangeSource, BasePoller):
parallelRequests = 2
verboseChilds = False
- def __init__(self, hgURL, repositoryIndex, pollInterval=120):
+ def __init__(self, hgURL, repositoryIndex, pollInterval=120, branch=None):
"""
@type repositoryIndex: string
@param repositoryIndex: The URL listing all locale repos
@type pollInterval int
@param pollInterval The time (in seconds) between queries for
changes
+ @type branch: string
+ @param branch: Used by caller to uniquely identify this object
"""
BasePoller.__init__(self)
@@ -444,6 +446,7 @@ def __init__(self, hgURL, repositoryIndex, pollInterval=120):
self.locales = []
self.pendingLocales = []
self.activeRequests = 0
+ self.branch = branch
def startService(self):
self.loop = LoopingCall(self.poll)
View
49 misc.py
@@ -159,8 +159,7 @@ def generateTestBuilderNames(name_prefix, suites_name, suites):
return test_builders
fastRegexes = []
-nReservedFastSlaves = 0
-nReservedSlowSlaves = 0
+nReservedSlaves = 0
def _partitionSlaves(slaves):
"""Partitions the list of slaves into 'fast' and 'slow' slaves, according
@@ -180,9 +179,18 @@ def _partitionSlaves(slaves):
def _partitionUnreservedSlaves(slaves):
fast, slow = _partitionSlaves(slaves)
- return fast[nReservedFastSlaves:], slow[nReservedSlowSlaves:]
+ fast = fast[nReservedSlaves:]
+ reserved_fast = len(fast)
+ # If we don't have enough fast slaves to satisfy us, set aside some slow slaves too
+ # If nReservedSlaves > len(fast), then we need to reserve some slow slaves,
+ # and reserved_slow will be > 0
+ # If nReservedSlaves <= len(fast), then we have enough fast slaves, and
+ # reserved_slow will be 0
+ reserved_slow = max(0, nReservedSlaves-reserved_fast)
+ slow = slow[reserved_slow:]
+ return fast, slow
-def _readReservedFile(filename, fast=True):
+def _readReservedFile(filename):
if not filename or not os.path.exists(filename):
n = 0
else:
@@ -201,15 +209,10 @@ def _readReservedFile(filename, fast=True):
log.err()
return
- global nReservedSlowSlaves, nReservedFastSlaves
- if fast:
- if n != nReservedFastSlaves:
- log.msg("Setting nReservedFastSlaves to %i (was %i)" % (n, nReservedFastSlaves))
- nReservedFastSlaves = n
- else:
- if n != nReservedSlowSlaves:
- log.msg("Setting nReservedSlowSlaves to %i (was %i)" % (n, nReservedSlowSlaves))
- nReservedSlowSlaves = n
+ global nReservedSlaves
+ if n != nReservedSlaves:
+ log.msg("Setting nReservedFastSlaves to %i (was %i)" % (n, nReservedSlaves))
+ nReservedSlaves = n
def _getLastTimeOnBuilder(builder, slavename):
# New builds are at the end of the buildCache, so
@@ -391,7 +394,8 @@ def generateTestBuilder(config, branch_name, platform, name_prefix,
buildToolsRepoPath=config['build_tools_repo_path'],
branchName=branch_name,
remoteExtras=pf.get('remote_extras'),
- downloadSymbols=pf.get('download_symbols', True),
+ downloadSymbols=pf.get('download_symbols', False),
+ downloadSymbolsOnDemand=pf.get('download_symbols_ondemand', True),
)
builder = {
'name': '%s %s' % (name_prefix, suites_name),
@@ -450,9 +454,9 @@ def generateTestBuilder(config, branch_name, platform, name_prefix,
thisChunk=i+1,
chunkByDir=suites.get('chunkByDir'),
env=pf.get('unittest-env', {}),
- downloadSymbols=pf.get('download_symbols', True),
+ downloadSymbols=pf.get('download_symbols', False),
+ downloadSymbolsOnDemand=pf.get('download_symbols_ondemand', True),
resetHwClock=resetHwClock,
- stackwalk_cgi=config.get('stackwalk_cgi'),
)
builder = {
'name': '%s %s-%i/%i' % (name_prefix, suites_name, i+1, totalChunks),
@@ -479,10 +483,10 @@ def generateTestBuilder(config, branch_name, platform, name_prefix,
buildToolsRepoPath=config['build_tools_repo_path'],
buildSpace=1.0,
buildsBeforeReboot=config['platforms'][platform]['builds_before_reboot'],
- downloadSymbols=pf.get('download_symbols', True),
+ downloadSymbols=pf.get('download_symbols', False),
+ downloadSymbolsOnDemand=pf.get('download_symbols_ondemand', True),
env=pf.get('unittest-env', {}),
resetHwClock=resetHwClock,
- stackwalk_cgi=config.get('stackwalk_cgi'),
)
builder = {
'name': '%s %s' % (name_prefix, suites_name),
@@ -756,7 +760,8 @@ def generateBranchObjects(config, name, secrets=None):
if config['enable_l10n'] and config['enable_l10n_onchange']:
hg_all_locales_poller = HgAllLocalesPoller(hgURL = config['hgurl'],
repositoryIndex = config['l10n_repo_path'],
- pollInterval=l10nPollInterval)
+ pollInterval=l10nPollInterval,
+ branch=name)
hg_all_locales_poller.parallelRequests = 1
branchObjects['change_source'].append(hg_all_locales_poller)
@@ -2020,7 +2025,8 @@ def generateCCBranchObjects(config, name, secrets=None):
if config['enable_l10n'] and config['enable_l10n_onchange']:
hg_all_locales_poller = HgAllLocalesPoller(hgURL = config['hgurl'],
repositoryIndex = config['l10n_repo_path'],
- pollInterval=l10nPollInterval)
+ pollInterval=l10nPollInterval,
+ branch=name)
hg_all_locales_poller.parallelRequests = 1
branchObjects['change_source'].append(hg_all_locales_poller)
@@ -2275,7 +2281,7 @@ def generateCCBranchObjects(config, name, secrets=None):
'configSubDir': config['config_subdir'],
'profiledBuild': per_checkin_build_uses_pgo,
'productName': pf['product_name'],
- 'mozconfig': pf['mozconfig_dep'],
+ 'mozconfig': pf['mozconfig'],
'srcMozconfig': pf.get('src_mozconfig'),
'use_scratchbox': pf.get('use_scratchbox'),
'stageServer': config['stage_server'],
@@ -3664,6 +3670,7 @@ def generateJetpackObjects(config, SLAVES):
relayhost="mail.build.mozilla.org",
builders=[b['name'] for b in builders],
logCompression="gzip",
+ errorparser="unittest",
)
status = [tbox_mailer]
Oops, something went wrong.

0 comments on commit e2fdf9a

Please sign in to comment.