Permalink
Browse files

Merging from default

changeset:   2967:e7e563b5c708
parent:      2964:0f3e640208d0
user:        Ben Hearsum <bhearsum@mozilla.com>
date:        Thu Jan 31 14:06:22 2013 -0500
summary:     bug 831989: use pymake by default on windows. r=coop

changeset:   2968:81a7d0437342
tag:         tip
user:        Ben Hearsum <bhearsum@mozilla.com>
date:        Thu Jan 31 14:11:33 2013 -0500
summary:     bug 827306: ensure that non-release build dirs are the same length as release build dirs - add tests for normalizeName;refactor. r=catlee

--HG--
branch : production-0.8
  • Loading branch information...
2 parents e1a2fe7 + 5682f87 commit cea9a01bb9998adfaf14ca0dce352ca8e62e134a @ccooper ccooper committed Jan 31, 2013
Showing with 121 additions and 103 deletions.
  1. +19 −22 common.py
  2. +32 −31 misc.py
  3. +9 −4 process/factory.py
  4. +61 −46 process/release.py
View
@@ -1,3 +1,4 @@
+import re
import time
import uuid
@@ -45,13 +46,7 @@ def incrementBuildID(buildID):
return genBuildID(epoch + 1)
-def reallyShort(name, product=None):
- # FIXME: hacky workaround to fix thunderbird windows builds
- if name in ('release-comm-esr17-win32_build',
- 'release-comm-beta-win32_build',
- 'release-comm-release-win32_build'):
- return 'zzz-%s' % name.split('-')[2]
-
+def normalizeName(name, product=None):
prefix = ''
if product != None and 'thunderbird' in product:
prefix = 'tb-'
@@ -67,7 +62,7 @@ def reallyShort(name, product=None):
'jaegermonkey': 'jm',
'shadow': 'sh',
'mobile': 'mb',
- 'desktop': None,
+ 'desktop': '',
'debug': 'dbg',
'xulrunner': 'xr',
'build': 'bld',
@@ -106,17 +101,19 @@ def reallyShort(name, product=None):
'gecko': 'g',
'localizer': 'lz',
}
- hyphen_seperated_words = name.split('-')
- words = []
- for word in hyphen_seperated_words:
- space_seperated_words = word.split('_')
- for word in space_seperated_words:
- words.extend(word.split(' '))
- new_words = []
- for word in words:
- if word in mappings.keys():
- if mappings[word]:
- new_words.append(mappings[word])
- else:
- new_words.append(word)
- return prefix + '-'.join(new_words)
+ for word, replacement in mappings.iteritems():
+ # Regexes are slow, so make sure the word is there at all before
+ # trying to do a substitution.
+ if word in name:
+ # Match strings that...
+ regex = re.compile(
+ r'(-|_|\A)' + # Are the start of the string or after a separator
+ ('%s' % word) + # Contain the word we're looking
+ r'(-|_|\Z)' # And don't have anything other the end of the string
+ # or a separator atfer them
+ )
+ name = regex.sub(r'\1%s\2' % replacement, name)
+ # Not sure why we do this, but apparently we replace all the underscores
+ # with hyphens...
+ name = name.replace('_', '-')
+ return prefix + name
View
@@ -45,7 +45,7 @@
reload(build.paths)
reload(mozilla_buildtools.queuedir)
-from buildbotcustom.common import reallyShort
+from buildbotcustom.common import normalizeName
from buildbotcustom.changes.hgpoller import HgPoller, HgAllLocalesPoller
from buildbotcustom.process.factory import NightlyBuildFactory, \
NightlyRepackFactory, UnittestPackagedBuildFactory, TalosFactory, \
@@ -460,11 +460,11 @@ def makeBundleBuilder(config, name):
'name': '%s hg bundle' % name,
'slavenames': list(slaves),
'builddir': '%s-bundle' % (name,),
- 'slavebuilddir': reallyShort('%s-bundle' % (name,)),
+ 'slavebuilddir': normalizeName('%s-bundle' % (name,)),
'factory': bundle_factory,
'category': name,
'nextSlave': _nextSlowSlave,
- 'properties': {'slavebuilddir': reallyShort('%s-bundle' % (name,)),
+ 'properties': {'slavebuilddir': normalizeName('%s-bundle' % (name,)),
'branch': name,
'platform': None,
'product': 'firefox',
@@ -1033,7 +1033,7 @@ def generateBranchObjects(config, name, secrets=None):
'name': '%s_dep' % pf['base_name'],
'slavenames': pf['slaves'],
'builddir': '%s_dep' % pf['base_name'],
- 'slavebuilddir': reallyShort('%s_dep' % pf['base_name']),
+ 'slavebuilddir': normalizeName('%s_dep' % pf['base_name']),
'factory': factory,
'category': name,
'properties': {
@@ -1057,7 +1057,7 @@ def generateBranchObjects(config, name, secrets=None):
'name': '%s_nightly' % pf['base_name'],
'slavenames': pf['slaves'],
'builddir': '%s_nightly' % pf['base_name'],
- 'slavebuilddir': reallyShort('%s_nightly' % pf['base_name']),
+ 'slavebuilddir': normalizeName('%s_nightly' % pf['base_name']),
'factory': factory,
'category': name,
'properties': {
@@ -1104,6 +1104,12 @@ def generateBranchObjects(config, name, secrets=None):
clobberTime = pf.get('clobber_time', config['default_clobber_time'])
checkTest = pf.get('enable_checktests', False)
valgrindCheck = pf.get('enable_valgrind_checktests', False)
+ # Turn pymake on by default for Windows, and off by default for
+ # other platforms.
+ if 'win' in platform:
+ enable_pymake = pf.get('enable_pymake', True)
+ else:
+ enable_pymake = pf.get('enable_pymake', False)
extra_args = {}
if config.get('enable_try'):
@@ -1116,10 +1122,6 @@ def generateBranchObjects(config, name, secrets=None):
factory_class = NightlyBuildFactory
uploadSymbols = False
- if pf.get('enable_pymake'):
- extra_args['makeCmd'] = ['python',
- WithProperties("%(basedir)s/build/build/pymake/make.py")]
-
stageBasePath = '%s/%s' % (config['stage_base_path'],
pf['stage_product'])
@@ -1236,6 +1238,7 @@ def generateBranchObjects(config, name, secrets=None):
'mozharnessTag': config.get('mozharness_tag'),
'geckoL10nRoot': config.get('gecko_l10n_root'),
'geckoLanguagesFile': pf.get('gecko_languages_file'),
+ 'enable_pymake': enable_pymake,
}
factory_kwargs.update(extra_args)
if pf.get('product_name') == 'b2g':
@@ -1248,7 +1251,7 @@ def generateBranchObjects(config, name, secrets=None):
'name': '%s build' % pf['base_name'],
'slavenames': pf['slaves'],
'builddir': '%s-%s' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s' % (name, platform), pf['stage_product']),
+ 'slavebuilddir': normalizeName('%s-%s' % (name, platform), pf['stage_product']),
'factory': mozilla2_dep_factory,
'category': name,
'nextSlave': _nextFastSlave,
@@ -1258,7 +1261,7 @@ def generateBranchObjects(config, name, secrets=None):
'platform': platform,
'stage_platform': stage_platform,
'product': pf['stage_product'],
- 'slavebuilddir': reallyShort('%s-%s' % (name, platform), pf['stage_product'])},
+ 'slavebuilddir': normalizeName('%s-%s' % (name, platform), pf['stage_product'])},
}
branchObjects['builders'].append(mozilla2_dep_builder)
@@ -1276,15 +1279,15 @@ def generateBranchObjects(config, name, secrets=None):
'name': '%s pgo-build' % pf['base_name'],
'slavenames': pf['slaves'],
'builddir': '%s-%s-pgo' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-pgo' % (name, platform), pf['stage_product']),
+ 'slavebuilddir': normalizeName('%s-%s-pgo' % (name, platform), pf['stage_product']),
'factory': pgo_factory,
'category': name,
'nextSlave': _nextFastSlave,
'properties': {'branch': name,
'platform': platform,
'stage_platform': stage_platform + '-pgo',
'product': pf['stage_product'],
- 'slavebuilddir': reallyShort('%s-%s-pgo' % (name, platform), pf['stage_product'])},
+ 'slavebuilddir': normalizeName('%s-%s-pgo' % (name, platform), pf['stage_product'])},
}
branchObjects['builders'].append(pgo_builder)
@@ -1330,7 +1333,7 @@ def generateBranchObjects(config, name, secrets=None):
scriptName='scripts/mobile_l10n.py',
extra_args=extra_args
)
- slavebuilddir = reallyShort(builddir, pf['stage_product'])
+ slavebuilddir = normalizeName(builddir, pf['stage_product'])
branchObjects['builders'].append({
'name': builderName,
'slavenames': pf.get('slaves'),
@@ -1390,10 +1393,6 @@ def generateBranchObjects(config, name, secrets=None):
nightly_kwargs.update(multiargs)
nightly_kwargs.update(ausargs)
- if pf.get('enable_pymake'):
- nightly_kwargs['makeCmd'] = ['python',
- WithProperties("%(basedir)s/build/build/pymake/make.py")]
-
# We make the assumption that *all* nightly builds
# are to be done with PGO. This is to ensure that all
# branches get some PGO coverage
@@ -1485,6 +1484,7 @@ def generateBranchObjects(config, name, secrets=None):
mozharnessTag=config.get('mozharness_tag'),
geckoL10nRoot=config.get('gecko_l10n_root'),
geckoLanguagesFile=pf.get('gecko_languages_file'),
+ enable_pymake=enable_pymake,
**nightly_kwargs
)
@@ -1493,7 +1493,7 @@ def generateBranchObjects(config, name, secrets=None):
'name': nightly_builder,
'slavenames': pf['slaves'],
'builddir': '%s-%s-nightly' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-nightly' % (name, platform), pf['stage_product']),
+ 'slavebuilddir': normalizeName('%s-%s-nightly' % (name, platform), pf['stage_product']),
'factory': mozilla2_nightly_factory,
'category': name,
'nextSlave': lambda b, sl: _nextFastSlave(b, sl, only_fast=True),
@@ -1502,7 +1502,7 @@ def generateBranchObjects(config, name, secrets=None):
'stage_platform': stage_platform,
'product': pf['stage_product'],
'nightly_build': True,
- 'slavebuilddir': reallyShort('%s-%s-nightly' % (name, platform), pf['stage_product'])},
+ 'slavebuilddir': normalizeName('%s-%s-nightly' % (name, platform), pf['stage_product'])},
}
branchObjects['builders'].append(mozilla2_nightly_builder)
@@ -1577,15 +1577,15 @@ def generateBranchObjects(config, name, secrets=None):
'name': l10nNightlyBuilders[nightly_builder]['l10n_builder'],
'slavenames': pf.get('l10n_slaves', pf['slaves']),
'builddir': '%s-%s-l10n-nightly' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-l10n-nightly' % (name, platform), pf['stage_product']),
+ 'slavebuilddir': normalizeName('%s-%s-l10n-nightly' % (name, platform), pf['stage_product']),
'factory': mozilla2_l10n_nightly_factory,
'category': name,
'nextSlave': _nextL10nSlave(),
'properties': {'branch': name,
'platform': platform,
'product': pf['stage_product'],
'stage_platform': stage_platform,
- 'slavebuilddir': reallyShort('%s-%s-l10n-nightly' % (name, platform), pf['stage_product']), },
+ 'slavebuilddir': normalizeName('%s-%s-l10n-nightly' % (name, platform), pf['stage_product']), },
}
branchObjects['builders'].append(
mozilla2_l10n_nightly_builder)
@@ -1608,7 +1608,7 @@ def generateBranchObjects(config, name, secrets=None):
'name': '%s valgrind' % pf['base_name'],
'slavenames': pf['slaves'],
'builddir': '%s-%s-valgrind' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-valgrind' % (name, platform), pf['stage_product']),
+ 'slavebuilddir': normalizeName('%s-%s-valgrind' % (name, platform), pf['stage_product']),
'factory': mozilla2_valgrind_factory,
'category': name,
'env': valgrind_env,
@@ -1617,7 +1617,7 @@ def generateBranchObjects(config, name, secrets=None):
'platform': platform,
'stage_platform': stage_platform,
'product': pf['stage_product'],
- 'slavebuilddir': reallyShort('%s-%s-valgrind' % (name, platform), pf['stage_product'])},
+ 'slavebuilddir': normalizeName('%s-%s-valgrind' % (name, platform), pf['stage_product'])},
}
branchObjects['builders'].append(mozilla2_valgrind_builder)
@@ -1675,15 +1675,15 @@ def generateBranchObjects(config, name, secrets=None):
'name': l10nBuilders[pf['base_name']]['l10n_builder'],
'slavenames': pf.get('l10n_slaves', pf['slaves']),
'builddir': '%s-%s-l10n-dep' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-l10n-dep' % (name, platform), pf['stage_product']),
+ 'slavebuilddir': normalizeName('%s-%s-l10n-dep' % (name, platform), pf['stage_product']),
'factory': mozilla2_l10n_dep_factory,
'category': name,
'nextSlave': _nextL10nSlave(),
'properties': {'branch': name,
'platform': platform,
'stage_platform': stage_platform,
'product': pf['stage_product'],
- 'slavebuilddir': reallyShort('%s-%s-l10n-dep' % (name, platform), pf['stage_product'])},
+ 'slavebuilddir': normalizeName('%s-%s-l10n-dep' % (name, platform), pf['stage_product'])},
}
branchObjects['builders'].append(mozilla2_l10n_dep_builder)
@@ -1739,16 +1739,17 @@ def generateBranchObjects(config, name, secrets=None):
mock_target=pf.get('mock_target'),
mock_packages=pf.get('mock_packages'),
mock_copyin_files=pf.get('mock_copyin_files'),
+ enable_pymake=enable_pymake,
)
mozilla2_xulrunner_builder = {
'name': '%s xulrunner' % pf['base_name'],
'slavenames': pf['slaves'],
'builddir': '%s-%s-xulrunner' % (name, platform),
- 'slavebuilddir': reallyShort('%s-%s-xulrunner' % (name, platform), pf['stage_product']),
+ 'slavebuilddir': normalizeName('%s-%s-xulrunner' % (name, platform), pf['stage_product']),
'factory': mozilla2_xulrunner_factory,
'category': name,
'nextSlave': _nextSlowSlave,
- 'properties': {'branch': name, 'platform': platform, 'slavebuilddir': reallyShort('%s-%s-xulrunner' % (name, platform)), 'product': 'xulrunner'},
+ 'properties': {'branch': name, 'platform': platform, 'slavebuilddir': normalizeName('%s-%s-xulrunner' % (name, platform)), 'product': 'xulrunner'},
}
branchObjects['builders'].append(mozilla2_xulrunner_builder)
@@ -2276,10 +2277,10 @@ def generateBlocklistBuilder(config, branch_name, platform, base_name, slaves):
'name': '%s blocklist update' % base_name,
'slavenames': slaves,
'builddir': '%s-%s-blocklistupdate' % (branch_name, platform),
- 'slavebuilddir': reallyShort('%s-%s-blocklistupdate' % (branch_name, platform)),
+ 'slavebuilddir': normalizeName('%s-%s-blocklistupdate' % (branch_name, platform)),
'factory': blocklistupdate_factory,
'category': branch_name,
- 'properties': {'branch': branch_name, 'platform': platform, 'slavebuilddir': reallyShort('%s-%s-blocklistupdate' % (branch_name, platform)), 'product': 'firefox'},
+ 'properties': {'branch': branch_name, 'platform': platform, 'slavebuilddir': normalizeName('%s-%s-blocklistupdate' % (branch_name, platform)), 'product': 'firefox'},
}
return blocklistupdate_builder
@@ -2457,7 +2458,7 @@ def generateSpiderMonkeyObjects(project, config, SLAVES):
builder = {'name': name,
'builddir': '%s_%s_spidermonkey-%s' % (branch, platform, variant),
- 'slavebuilddir': reallyShort('%s_%s_spidermonkey-%s' % (branch, platform, variant)),
+ 'slavebuilddir': normalizeName('%s_%s_spidermonkey-%s' % (branch, platform, variant)),
'slavenames': pf['slaves'],
'nextSlave': _nextSlowIdleSlave(config['idle_slaves']),
'factory': f,
View
@@ -66,7 +66,7 @@
from buildbotcustom.steps.signing import SigningServerAuthenication
from buildbotcustom.env import MozillaEnvironments
from buildbotcustom.common import getSupportedPlatforms, getPlatformFtpDir, \
- genBuildID, reallyShort
+ genBuildID, normalizeName
from buildbotcustom.steps.mock import MockReset, MockInit, MockCommand, MockInstall, \
MockMozillaCheck, MockProperty, RetryingMockProperty, RetryingMockCommand, \
MockAliveTest, MockCompareLeakLogs
@@ -388,7 +388,7 @@ def __init__(self, hgHost, repoPath, buildToolsRepoPath, buildSpace=0,
balrog_api_root=None, balrog_credentials_file=None,
use_mock=False, mock_target=None,
mock_packages=None, mock_copyin_files=None,
- makeCmd=['make'], **kwargs):
+ enable_pymake=False, **kwargs):
BuildFactory.__init__(self, **kwargs)
if hgHost.endswith('/'):
@@ -414,7 +414,7 @@ def __init__(self, hgHost, repoPath, buildToolsRepoPath, buildSpace=0,
self.mock_target = mock_target
self.mock_packages = mock_packages
self.mock_copyin_files = mock_copyin_files
- self.makeCmd = makeCmd
+ self.enable_pymake = enable_pymake
self.repository = self.getRepository(repoPath)
if branchName:
@@ -426,6 +426,11 @@ def __init__(self, hgHost, repoPath, buildToolsRepoPath, buildSpace=0,
else:
self.clobberBranch = clobberBranch
+ if self.enable_pymake:
+ self.makeCmd = ['python', WithProperties("%(basedir)s/build/build/pymake/make.py")]
+ else:
+ self.makeCmd = ['make']
+
if self.signingServers and self.enableSigning:
self.signing_command = get_signing_cmd(
self.signingServers, self.env.get('PYTHON26'))
@@ -3073,7 +3078,7 @@ def list2dict(rv, stdout, stderr):
class BaseRepackFactory(MozillaBuildFactory):
# Override ignore_dirs so that we don't delete l10n nightly builds
# before running a l10n nightly build
- ignore_dirs = MozillaBuildFactory.ignore_dirs + [reallyShort('*-nightly')]
+ ignore_dirs = MozillaBuildFactory.ignore_dirs + [normalizeName('*-nightly')]
extraConfigureArgs = []
Oops, something went wrong.

0 comments on commit cea9a01

Please sign in to comment.