Permalink
Browse files

changeset: 1902:e2ac4dbaa83f

tag:         tip
parent:      1894:86bfd36c4b57
user:        Chris AtLee <catlee@mozilla.com>
date:        Tue Nov 08 14:07:59 2011 -0500
summary:     Bug 699407: Use hg mirrors whereever possible. r=bhearsum

--HG--
branch : production-0.8
  • Loading branch information...
2 parents a81be76 + e483430 commit 8fb53f9ceb88705d5c54cacf358bd87f31a34777 @armenzg armenzg committed Nov 9, 2011
Showing with 104 additions and 71 deletions.
  1. +12 −0 misc.py
  2. +92 −71 process/factory.py
View
@@ -1123,6 +1123,8 @@ def generateBranchObjects(config, name):
'l10nCheckTest': pf.get('l10n_check_test', False),
'android_signing': pf.get('android_signing', False),
'post_upload_include_platform': pf.get('post_upload_include_platform', False),
+ 'baseMirrorUrls': config.get('base_mirror_urls'),
+ 'baseBundleUrls': config.get('base_bundle_urls'),
}
factory_kwargs.update(extra_args)
@@ -1377,6 +1379,8 @@ def generateBranchObjects(config, name):
l10nCheckTest=pf.get('l10n_check_test', False),
android_signing=pf.get('android_signing', False),
post_upload_include_platform=pf.get('post_upload_include_platform', False),
+ baseMirrorUrls=config.get('base_mirror_urls'),
+ baseBundleUrls=config.get('base_bundle_urls'),
**nightly_kwargs
)
@@ -1402,6 +1406,8 @@ def generateBranchObjects(config, name):
# TODO Linux and mac are not working with mozconfig at this point
# and this will disable it for now. We will fix this in bug 518359.
env = {}
+ if 'HG_SHARE_BASE_DIR' in platform_env:
+ env['HG_SHARE_BASE_DIR'] = platform_env['HG_SHARE_BASE_DIR']
objdir = ''
mozconfig = None
@@ -1439,6 +1445,7 @@ def generateBranchObjects(config, name):
buildSpace=l10nSpace,
clobberURL=config['base_clobber_url'],
clobberTime=clobberTime,
+ baseMirrorUrls=config.get('base_mirror_urls'),
)
mozilla2_l10n_nightly_builder = {
'name': l10nNightlyBuilders[nightly_builder]['l10n_builder'],
@@ -1534,7 +1541,11 @@ def generateBranchObjects(config, name):
# We still want l10n_dep builds if nightlies are off
if config['enable_l10n'] and platform in config['l10n_platforms'] and \
config['enable_l10n_onchange']:
+ env = {}
+ if 'HG_SHARE_BASE_DIR' in platform_env:
+ env['HG_SHARE_BASE_DIR'] = platform_env['HG_SHARE_BASE_DIR']
mozilla2_l10n_dep_factory = NightlyRepackFactory(
+ env=env,
platform=platform,
hgHost=config['hghost'],
tree=config['l10n_tree'],
@@ -1554,6 +1565,7 @@ def generateBranchObjects(config, name):
buildSpace=l10nSpace,
clobberURL=config['base_clobber_url'],
clobberTime=clobberTime,
+ baseMirrorUrls=config.get('base_mirror_urls'),
)
mozilla2_l10n_dep_builder = {
'name': l10nBuilders[pf['base_name']]['l10n_builder'],
View
@@ -2,7 +2,6 @@
from datetime import datetime
import os.path, re
-from time import strftime
import urllib
import random
@@ -347,7 +346,8 @@ class MozillaBuildFactory(RequestSortingBuildFactory):
def __init__(self, hgHost, repoPath, buildToolsRepoPath, buildSpace=0,
clobberURL=None, clobberTime=None, buildsBeforeReboot=None,
- branchName=None, baseWorkDir='build', hashType='sha512', **kwargs):
+ branchName=None, baseWorkDir='build', hashType='sha512',
+ baseMirrorUrls=None, baseBundleUrls=None, **kwargs):
BuildFactory.__init__(self, **kwargs)
if hgHost.endswith('/'):
@@ -362,6 +362,8 @@ def __init__(self, hgHost, repoPath, buildToolsRepoPath, buildSpace=0,
self.buildsBeforeReboot = buildsBeforeReboot
self.baseWorkDir = baseWorkDir
self.hashType = hashType
+ self.baseMirrorUrls = baseMirrorUrls
+ self.baseBundleUrls = baseBundleUrls
self.repository = self.getRepository(repoPath)
if branchName:
@@ -597,6 +599,62 @@ def addFilePropertiesSteps(self, filename, directory, fileType,
extract_fn = self.unsetFilepath,
))
+ def makeHgtoolStep(self, name='hg_update', repo_url=None, wc=None,
+ mirrors=None, bundles=None, env=None,
+ clone_by_revision=False, rev=None, workdir='build',
+ use_properties=True, locks=None):
+
+ if not env:
+ env = self.env
+
+ if use_properties:
+ env = env.copy()
+ env['PROPERTIES_FILE'] = 'buildprops.json'
+
+ cmd = ['python', WithProperties("%(toolsdir)s/buildfarm/utils/hgtool.py")]
+
+ if clone_by_revision:
+ cmd.append('--clone-by-revision')
+
+ if mirrors is None and self.baseMirrorUrls:
+ mirrors = ["%s/%s" % (url, self.repoPath) for url in self.baseMirrorUrls]
+
+ if mirrors:
+ for mirror in mirrors:
+ cmd.extend(["--mirror", mirror])
+
+ if bundles is None and self.baseBundleUrls:
+ bundles = ["%s/%s.hg" % (url, self.getRepoName(self.repository)) for url in self.baseBundleUrls]
+
+ if bundles:
+ for bundle in bundles:
+ cmd.extend(["--bundle", bundle])
+
+ if not repo_url:
+ repo_url = self.repository
+
+ if rev:
+ cmd.extend(["--rev", rev])
+
+ cmd.append(repo_url)
+
+ if wc:
+ cmd.append(wc)
+
+ if locks is None:
+ locks = []
+
+ return RetryingShellCommand(
+ name=name,
+ command=cmd,
+ timeout=60*60,
+ env=env,
+ workdir=workdir,
+ haltOnFailure=True,
+ flunkOnFailure=True,
+ locks=locks,
+ )
+
class MercurialBuildFactory(MozillaBuildFactory):
def __init__(self, env, objdir, platform, configRepoPath, configSubDir,
@@ -958,23 +1016,8 @@ def addSourceSteps(self):
workdir='.'
))
- env = self.env.copy()
- env['PROPERTIES_FILE'] = 'buildprops.json'
- cmd = [
- 'python',
- WithProperties("%(toolsdir)s/buildfarm/utils/hgtool.py"),
- 'http://%s/%s' % (self.hgHost, self.repoPath),
- 'build',
- ]
- self.addStep(ShellCommand(
- name='hg_update',
- command=cmd,
- timeout=60*60,
- env=env,
- workdir='.',
- haltOnFailure=True,
- flunkOnFailure=True,
- ))
+ self.addStep(self.makeHgtoolStep(wc='build', workdir='.'))
+
self.addStep(SetProperty(
name = 'set_got_revision',
command=['hg', 'parent', '--template={node}'],
@@ -1717,24 +1760,15 @@ def addSourceSteps(self):
workdir='.'
))
- env = self.env.copy()
- env['PROPERTIES_FILE'] = 'buildprops.json'
- cmd = [
- 'python',
- WithProperties("%(toolsdir)s/buildfarm/utils/hgtool.py"),
- 'http://%s/%s' % (self.hgHost, self.repoPath),
- 'build',
- ]
- self.addStep(ShellCommand(
- name='hg_update',
- command=cmd,
- timeout=60*60,
- locks=[hg_try_lock.access('counting')],
- env=env,
- workdir='.',
- haltOnFailure=True,
- flunkOnFailure=True,
- ))
+ step = self.makeHgtoolStep(
+ clone_by_revision=True,
+ bundles=[],
+ wc='build',
+ workdir='.',
+ locks=[hg_try_lock.access('counting')],
+ )
+ self.addStep(step)
+
else:
self.addStep(Mercurial(
name='hg_update',
@@ -2996,40 +3030,27 @@ def doUpload(self):
))
def getSources(self):
- self.addStep(ShellCommand(
- name='get_enUS_src',
- command=[
- 'python',
- WithProperties("%(toolsdir)s/buildfarm/utils/hgtool.py"),
- WithProperties("--rev=%(en_revision)s"),
- 'http://%s/%s' % (self.hgHost, self.repoPath),
- self.origSrcDir,
- ],
- env=self.env,
- descriptionDone="en-US source",
- workdir=self.baseWorkDir,
- locks=[hg_l10n_lock.access('counting')],
- haltOnFailure=True,
- flunkOnFailure=True,
- timeout=30*60 # 30 minutes
- ))
- self.addStep(ShellCommand(
- name='get_locale_src',
- command=[
- 'python',
- WithProperties("%(toolsdir)s/buildfarm/utils/hgtool.py"),
- WithProperties("--rev=%(l10n_revision)s"),
- WithProperties("http://" + self.hgHost + "/" + \
- self.l10nRepoPath + "/%(locale)s")
- ],
- env=self.env,
- descriptionDone="locale source",
- workdir='%s/%s' % (self.baseWorkDir, self.l10nRepoPath),
- locks=[hg_l10n_lock.access('counting')],
- haltOnFailure=True,
- flunkOnFailure=True,
- timeout=5*60, # 5 minutes
- ))
+ step = self.makeHgtoolStep(
+ name='get_enUS_src',
+ wc=self.origSrcDir,
+ rev=WithProperties("%(en_revision)s"),
+ locks=[hg_l10n_lock.access('counting')],
+ workdir=self.baseWorkDir,
+ use_properties=False,
+ )
+ self.addStep(step)
+
+ step = self.makeHgtoolStep(
+ name='get_locale_src',
+ rev=WithProperties("%(l10n_revision)s"),
+ repo_url=WithProperties("http://" + self.hgHost + "/" + \
+ self.l10nRepoPath + "/%(locale)s"),
+ workdir='%s/%s' % (self.baseWorkDir, self.l10nRepoPath),
+ locks=[hg_l10n_lock.access('counting')],
+ use_properties=False,
+ mirrors=[],
+ )
+ self.addStep(step)
def updateEnUS(self):
'''Update the en-US source files to the revision used by

0 comments on commit 8fb53f9

Please sign in to comment.