Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

merged from default

changeset:   2318:5f526ab13f38
parent:      2316:a719acf41723
user:        Chris AtLee <catlee@mozilla.com>
date:        Fri Dec 02 16:03:39 2011 -0500
summary:     Bug 509158: Signing support in build factories. r=bhearsum

changeset:   2319:202dfd424ea1
user:        Rail Aliiev <rail@mozilla.com>
date:        Fri Dec 02 16:07:07 2011 -0500
summary:     Bug 706832 - Implement master side token generation for signing on demand. r=catlee,bhearsum

changeset:   2320:c6f4f75908cb
user:        Chris Cooper <ccooper@deadsquid.com>
date:        Fri Dec 02 16:14:16 2011 -0500
summary:     Bug 671450 - Try different sources for revision in log_uploader - r=nthomas

changeset:   2321:a92845bec18e
user:        Ben Hearsum <bhearsum@mozilla.com>
date:        Fri Dec 02 17:01:44 2011 -0500
summary:     bug 509158: Sign builds as part of the build process: enable signing server for debug builds; disable pre-signed updater on elm. r=rail/catlee

changeset:   2322:d61c24d02d97
tag:         tip
user:        Chris AtLee <catlee@mozilla.com>
date:        Sun Dec 04 14:22:24 2011 -0500
summary:     Bug 509158: Don't enable signing for l10n check steps. r=rail

--HG--
branch : production-0.8
  • Loading branch information...
commit 2fbdfdd4275f6bcf6c61b0f6de60171860f92137 2 parents 9518530 + 3a1f902
@catlee catlee authored
Showing with 222 additions and 22 deletions.
  1. +8 −1 bin/log_uploader.py
  2. +11 −10 misc.py
  3. +82 −11 process/factory.py
  4. +121 −0 steps/signing.py
View
9 bin/log_uploader.py
@@ -294,10 +294,17 @@ def formatLog(tmpdir, build, builder_suffix=''):
uploadArgs['to_shadow'] = False
uploadArgs['to_tinderbox_dated'] = True
+ props = build.getProperties()
+ if props.getProperty('got_revision') is not None:
+ revision=props['got_revision']
+ elif props.getProperty('revision') is not None:
+ revision=props['revision']
+ else:
+ revision=None
uploadArgs.update(dict(
to_try=False,
who=None,
- revision=None,
+ revision=revision,
buildid=buildid,
))
post_upload_cmd = postUploadCmdPrefix(**uploadArgs)
View
21 misc.py
@@ -550,7 +550,7 @@ def generateCCTestBuilder(config, branch_name, platform, name_prefix,
return builders
-def generateBranchObjects(config, name):
+def generateBranchObjects(config, name, secrets=None):
"""name is the name of branch which is usually the last part of the path
to the repository. For example, 'mozilla-central', 'mozilla-aurora', or
'mozilla-1.9.1'.
@@ -567,6 +567,8 @@ def generateBranchObjects(config, name):
'schedulers': [],
'status': []
}
+ if secrets is None:
+ secrets = {}
builders = []
unittestBuilders = []
triggeredUnittestBuilders = []
@@ -1124,6 +1126,7 @@ def generateBranchObjects(config, name):
'l10nCheckTest': pf.get('l10n_check_test', False),
'android_signing': pf.get('android_signing', False),
'post_upload_include_platform': pf.get('post_upload_include_platform', False),
+ 'signingServers': secrets.get(pf.get('dep_signing_servers')),
'baseMirrorUrls': config.get('base_mirror_urls'),
'baseBundleUrls': config.get('base_bundle_urls'),
'mozillaDir': config.get('mozilla_dir', None),
@@ -1381,6 +1384,7 @@ def generateBranchObjects(config, name):
l10nCheckTest=pf.get('l10n_check_test', False),
android_signing=pf.get('android_signing', False),
post_upload_include_platform=pf.get('post_upload_include_platform', False),
+ signingServers=secrets.get(pf.get('nightly_signing_servers')),
baseMirrorUrls=config.get('base_mirror_urls'),
baseBundleUrls=config.get('base_bundle_urls'),
**nightly_kwargs
@@ -1407,14 +1411,11 @@ def generateBranchObjects(config, name):
if platform in config['l10n_platforms']:
# TODO Linux and mac are not working with mozconfig at this point
# and this will disable it for now. We will fix this in bug 518359.
- env = {}
- if 'HG_SHARE_BASE_DIR' in platform_env:
- env['HG_SHARE_BASE_DIR'] = platform_env['HG_SHARE_BASE_DIR']
objdir = ''
mozconfig = None
mozilla2_l10n_nightly_factory = NightlyRepackFactory(
- env=env,
+ env=platform_env,
objdir=objdir,
platform=platform,
hgHost=config['hghost'],
@@ -1447,6 +1448,7 @@ def generateBranchObjects(config, name):
buildSpace=l10nSpace,
clobberURL=config['base_clobber_url'],
clobberTime=clobberTime,
+ signingServers=secrets.get(pf.get('nightly_signing_servers')),
baseMirrorUrls=config.get('base_mirror_urls'),
)
mozilla2_l10n_nightly_builder = {
@@ -1498,7 +1500,7 @@ def generateBranchObjects(config, name):
buildSpace=buildSpace,
clobberURL=config['base_clobber_url'],
clobberTime=clobberTime,
- buildsBeforeReboot=pf['builds_before_reboot']
+ buildsBeforeReboot=pf['builds_before_reboot'],
)
mozilla2_shark_builder = {
'name': '%s shark' % pf['base_name'],
@@ -1543,11 +1545,8 @@ def generateBranchObjects(config, name):
# We still want l10n_dep builds if nightlies are off
if config['enable_l10n'] and platform in config['l10n_platforms'] and \
config['enable_l10n_onchange']:
- env = {}
- if 'HG_SHARE_BASE_DIR' in platform_env:
- env['HG_SHARE_BASE_DIR'] = platform_env['HG_SHARE_BASE_DIR']
mozilla2_l10n_dep_factory = NightlyRepackFactory(
- env=env,
+ env=platform_env,
platform=platform,
hgHost=config['hghost'],
tree=config['l10n_tree'],
@@ -1567,6 +1566,7 @@ def generateBranchObjects(config, name):
buildSpace=l10nSpace,
clobberURL=config['base_clobber_url'],
clobberTime=clobberTime,
+ signingServers=secrets.get(pf.get('dep_signing_servers')),
baseMirrorUrls=config.get('base_mirror_urls'),
)
mozilla2_l10n_dep_builder = {
@@ -1750,6 +1750,7 @@ def generateBranchObjects(config, name):
clobberTime=clobberTime,
buildsBeforeReboot=pf['builds_before_reboot'],
packageSDK=True,
+ signingServers=secrets.get(pf.get('nightly_signing_servers')),
)
mozilla2_xulrunner_builder = {
'name': '%s xulrunner' % pf['base_name'],
View
93 process/factory.py
@@ -26,6 +26,7 @@
import buildbotcustom.steps.updates
import buildbotcustom.steps.talos
import buildbotcustom.steps.unittest
+import buildbotcustom.steps.signing
import buildbotcustom.env
import buildbotcustom.misc_scheduler
import build.paths
@@ -42,6 +43,7 @@
reload(buildbotcustom.steps.updates)
reload(buildbotcustom.steps.talos)
reload(buildbotcustom.steps.unittest)
+reload(buildbotcustom.steps.signing)
reload(buildbotcustom.env)
reload(build.paths)
reload(release.info)
@@ -64,6 +66,7 @@
from buildbotcustom.steps.transfer import MozillaStageUpload
from buildbotcustom.steps.updates import CreateCompleteUpdateSnippet, \
CreatePartialUpdateSnippet
+from buildbotcustom.steps.signing import SigningServerAuthenication
from buildbotcustom.env import MozillaEnvironments
from buildbotcustom.common import getSupportedPlatforms, getPlatformFtpDir, \
genBuildID, reallyShort
@@ -82,14 +85,19 @@
hg_l10n_lock = locks.MasterLock("hg_l10n_lock", maxCount=20)
class DummyFactory(BuildFactory):
- def __init__(self):
+ def __init__(self, delay=5, triggers=None):
BuildFactory.__init__(self)
- self.addStep(Dummy())
+ self.addStep(Dummy(delay))
+ if triggers:
+ self.addStep(Trigger(
+ schedulerNames=triggers,
+ waitForFinish=False,
+ ))
-def makeDummyBuilder(name, slaves, category=None):
+def makeDummyBuilder(name, slaves, category=None, delay=0, triggers=None):
builder = {
'name': name,
- 'factory': DummyFactory(),
+ 'factory': DummyFactory(delay, triggers),
'builddir': name,
'slavenames': slaves,
}
@@ -689,6 +697,7 @@ def __init__(self, env, objdir, platform, configRepoPath, configSubDir,
multiLocaleScript=None,
multiLocaleConfig=None,
mozharnessMultiOptions=None,
+ signingServers=None,
**kwargs):
MozillaBuildFactory.__init__(self, **kwargs)
@@ -754,6 +763,7 @@ def __init__(self, env, objdir, platform, configRepoPath, configSubDir,
self.useSharedCheckouts = useSharedCheckouts
self.testPrettyNames = testPrettyNames
self.l10nCheckTest = l10nCheckTest
+ self.signingServers = signingServers
if self.uploadPackages:
assert productName and stageServer and stageUsername
@@ -838,6 +848,17 @@ def __init__(self, env, objdir, platform, configRepoPath, configSubDir,
self.logBaseUrl = 'http://%s/pub/mozilla.org/%s/%s' % \
( self.stageServer, self.stageProduct, self.logUploadDir)
+ if signingServers:
+ cmd = [
+ env.get('PYTHON26', 'python'), "%(toolsdir)s/release/signing/signtool.py",
+ "-t", "%(basedir)s/build/token",
+ "-n", "%(basedir)s/build/nonce",
+ "-c", "%(toolsdir)s/release/signing/host.cert",
+ ]
+ for ss, user, passwd in signingServers:
+ cmd.extend(['-H', ss])
+ self.env['MOZ_SIGN_CMD'] = WithProperties(" ".join(cmd))
+
# Need to override toolsdir as set by MozillaBuildFactory because
# we need Windows-style paths.
if self.platform.startswith('win'):
@@ -967,6 +988,8 @@ def addBuildSteps(self):
self.addSourceSteps()
self.addConfigSteps()
self.addDoBuildSteps()
+ if self.signingServers:
+ self.addGetTokenSteps()
if self.doBuildAnalysis:
self.addBuildAnalysisSteps()
@@ -1095,6 +1118,26 @@ def addConfigSteps(self):
command=['cat', '.mozconfig'],
))
+ def addGetTokenSteps(self):
+ server_cert = os.path.join(
+ os.path.dirname(build.paths.__file__),
+ '../../../release/signing/host.cert')
+ token = "build/token"
+ nonce = "build/nonce"
+ self.addStep(ShellCommand(
+ command=['rm', '-f', nonce],
+ workdir='.',
+ name='rm_nonce',
+ description=['remove', 'old', 'nonce'],
+ ))
+ self.addStep(SigningServerAuthenication(
+ servers=self.signingServers,
+ server_cert=server_cert,
+ slavedest=token,
+ workdir='.',
+ name='download_token',
+ ))
+
def addDoBuildSteps(self):
workdir=WithProperties('%(basedir)s/build')
if self.platform.startswith('win'):
@@ -1351,11 +1394,18 @@ def addCheckTestSteps(self):
)
def addL10nCheckTestSteps(self):
+ # We override MOZ_SIGN_CMD here because it's not necessary
+ # Disable signing for l10n check steps
+ env = self.env
+ if 'MOZ_SIGN_CMD' in env:
+ env = env.copy()
+ del env['MOZ_SIGN_CMD']
+
self.addStep(ShellCommand(
name='make l10n check',
command=['make', 'l10n-check'],
workdir='build/%s' % self.objdir,
- env=self.env,
+ env=env,
haltOnFailure=False,
flunkOnFailure=False,
warnOnFailure=True,
@@ -1389,13 +1439,19 @@ def addCreateUpdateSteps(self):
)
def addTestPrettyNamesSteps(self):
+ # Disable signing for l10n check steps
+ env = self.env
+ if 'MOZ_SIGN_CMD' in env:
+ env = env.copy()
+ del env['MOZ_SIGN_CMD']
+
if 'mac' in self.platform:
# Need to run this target or else the packaging targets will
# fail.
self.addStep(ShellCommand(
name='postflight_all',
command=['make', '-f', 'client.mk', 'postflight_all'],
- env=self.env,
+ env=env,
haltOnFailure=False,
flunkOnFailure=False,
warnOnFailure=False,
@@ -1407,7 +1463,7 @@ def addTestPrettyNamesSteps(self):
self.addStep(ShellCommand(
name='make %s pretty' % t,
command=['make', t, 'MOZ_PKG_PRETTYNAMES=1'],
- env=self.env,
+ env=env,
workdir='build/%s' % self.objdir,
haltOnFailure=False,
flunkOnFailure=False,
@@ -1418,7 +1474,7 @@ def addTestPrettyNamesSteps(self):
command=['make', '-C',
'%s/tools/update-packaging' % self.mozillaObjdir,
'MOZ_PKG_PRETTYNAMES=1'],
- env=self.env,
+ env=env,
haltOnFailure=False,
flunkOnFailure=False,
warnOnFailure=True,
@@ -1428,7 +1484,7 @@ def addTestPrettyNamesSteps(self):
name='make l10n check pretty',
command=['make', 'l10n-check', 'MOZ_PKG_PRETTYNAMES=1'],
workdir='build/%s' % self.objdir,
- env=self.env,
+ env=env,
haltOnFailure=False,
flunkOnFailure=False,
warnOnFailure=True,
@@ -2562,7 +2618,8 @@ def addUploadSnippetsSteps(self):
class ReleaseBuildFactory(MercurialBuildFactory):
def __init__(self, env, version, buildNumber, brandName=None,
unittestMasters=None, unittestBranch=None, talosMasters=None,
- usePrettyNames=True, enableUpdatePackaging=True, **kwargs):
+ usePrettyNames=True, enableUpdatePackaging=True,
+ signingFormats=None, **kwargs):
self.version = version
self.buildNumber = buildNumber
@@ -2795,7 +2852,7 @@ def __init__(self, project, appName, l10nRepoPath,
mozconfig=None, configRepoPath=None, configSubDir=None,
tree="notset", mozillaDir=None, l10nTag='default',
mergeLocales=True, mozconfigBranch="production",
- testPrettyNames=False, **kwargs):
+ testPrettyNames=False, signingServers=None, **kwargs):
MozillaBuildFactory.__init__(self, **kwargs)
self.env = env.copy()
@@ -2815,6 +2872,7 @@ def __init__(self, project, appName, l10nRepoPath,
self.mozconfig = mozconfig
self.mozconfigBranch = mozconfigBranch
self.testPrettyNames = testPrettyNames
+ self.signingServers = signingServers
# WinCE is the only platform that will do repackages with
# a mozconfig for now. This will be fixed in bug 518359
@@ -2825,6 +2883,17 @@ def __init__(self, project, appName, l10nRepoPath,
self.configRepo = self.getRepository(self.configRepoPath,
kwargs['hgHost'])
+ if signingServers:
+ cmd = [
+ env.get('PYTHON26', 'python'), "%(toolsdir)s/release/signing/signtool.py",
+ "-t", "%(basedir)s/build/token",
+ "-n", "%(basedir)s/build/nonce",
+ "-c", "%(toolsdir)s/release/signing/host.cert",
+ ]
+ for ss, user, passwd in signingServers:
+ cmd.extend(['-H', ss])
+ self.env['MOZ_SIGN_CMD'] = WithProperties(" ".join(cmd))
+
self.addStep(SetBuildProperty(
property_name='tree',
value=self.tree,
@@ -2911,6 +2980,8 @@ def __init__(self, project, appName, l10nRepoPath,
self.tinderboxPrintRevisions()
self.compareLocalesSetup()
self.compareLocales()
+ if self.signingServers:
+ self.addGetTokenSteps()
self.doRepack()
self.doUpload()
if self.testPrettyNames:
View
121 steps/signing.py
@@ -0,0 +1,121 @@
+from urllib import urlencode
+
+from OpenSSL.SSL import Context, TLSv1_METHOD, VERIFY_PEER,\
+ VERIFY_FAIL_IF_NO_PEER_CERT, OP_NO_SSLv2
+from OpenSSL.crypto import load_certificate, FILETYPE_PEM
+
+from twisted.python.urlpath import URLPath
+from twisted.internet.ssl import ContextFactory
+from twisted.web.client import getPage
+from twisted.python.failure import Failure
+from twisted.internet import reactor
+from buildbot.steps.transfer import StringDownload
+
+
+class HTTPSVerifyingContextFactory(ContextFactory):
+ isClient = True
+
+ def __init__(self, hostname, certfile):
+ self.hostname = hostname
+ data = open(certfile).read()
+ self.cert = load_certificate(FILETYPE_PEM, data)
+
+ def getContext(self):
+ ctx = Context(TLSv1_METHOD)
+ store = ctx.get_cert_store()
+ store.add_cert(self.cert)
+ ctx.set_verify(VERIFY_PEER | VERIFY_FAIL_IF_NO_PEER_CERT,
+ self.verifyHostname)
+ ctx.set_options(OP_NO_SSLv2)
+ return ctx
+
+ def verifyHostname(self, connection, x509, errno, depth, preverifyOK):
+ if preverifyOK:
+ if self.hostname == x509.get_subject().commonName:
+ return False
+ return preverifyOK
+
+
+class SigningServerAuthenication(StringDownload):
+ current_attempt = 0
+ stdio_log = None
+ uri = None
+ username = None
+ password = None
+ d = None
+ interrupted = False
+
+ def __init__(self, servers, server_cert, duration=6*3600, attempts=5,
+ sleeptime=60, **kwargs):
+ kwargs['s'] = ''
+ StringDownload.__init__(self, **kwargs)
+ self.addFactoryArguments(servers=servers, server_cert=server_cert,
+ duration=duration)
+ self.servers = list(servers)
+ self.server_cert = server_cert
+ self.duration = duration
+ self.attempts = attempts
+ self.sleeptime = sleeptime
+
+ def generateHeaders(self, method, credentials):
+ headers = {}
+ if method == 'POST':
+ headers['Content-Type'] = 'application/x-www-form-urlencoded'
+ base64string = '%s:%s' % (credentials[0], credentials[1])
+ base64string = base64string.encode("base64").strip()
+ headers['Authorization'] = 'Basic %s' % base64string
+ return headers
+
+ def start(self):
+ if self.interrupted:
+ self.failed(Failure(Exception('Interrupted')))
+ return
+
+ self.current_attempt += 1
+
+ if self.current_attempt > self.attempts:
+ if len(self.servers) < 1:
+ self.failed(Failure(Exception(
+ 'No more signing servers to try.')))
+ else:
+ self.current_attempt = 1
+
+ if self.current_attempt == 1:
+ uri, self.username, self.password = self.servers.pop()
+ self.uri = 'https://%s/token' % uri
+
+ slaveName = self.getSlaveName()
+ slaveIP = self.buildslave.slave.broker.transport.getPeer().host
+
+ if not self.stdio_log:
+ self.stdio_log = self.addLog('output')
+ self.stdio_log.addHeader("Slave: %s\n" % slaveName)
+ self.stdio_log.addHeader("IP: %s\n" % slaveIP)
+ self.stdio_log.addHeader("Duration: %s\n" % self.duration)
+ self.stdio_log.addStdout("URI: %s\n" % self.uri)
+
+ method = 'POST'
+ postdata = {
+ 'slave_ip': slaveIP,
+ 'duration': self.duration,
+ }
+ headers = self.generateHeaders(
+ method=method,
+ credentials=(self.username, self.password))
+ contextFactory = HTTPSVerifyingContextFactory(
+ URLPath(self.uri).netloc, self.server_cert)
+ d = getPage(self.uri, method=method, headers=headers,
+ postdata=urlencode(postdata), contextFactory=contextFactory)
+ d.addCallbacks(
+ self.downloadSignature,
+ lambda e: reactor.callLater(self.sleeptime, self.start)
+ )
+
+ def downloadSignature(self, res):
+ self.s = res
+ StringDownload.start(self)
+
+ def interrupt(self, reason='Interrupted'):
+ if not self.interrupted:
+ self.interrupted = True
+ StringDownload.interrupt(self, 'Interrupted')
Please sign in to comment.
Something went wrong with that request. Please try again.