Browse files

Merge from default

changeset:   2110:cc74a82b64a3
parent:      2108:f90c75b98eb7
user:        Aki Sasaki <asasaki@mozilla.com>
date:        Mon Feb 13 12:48:34 2012 -0800
summary:     bug 719511 - add optional reboot command to ScriptFactory. r=lsblakk

changeset:   2111:7fea4233c693
user:        Rail Aliiev <rail@mozilla.com>
date:        Tue Feb 14 10:56:56 2012 -0500
summary:     Bug 725292 - some repacks failed in 11.0b2 because of missing tokens. r=bhearsum

changeset:   2114:73437b4c315b
parent:      2111:7fea4233c693
user:        Rail Aliiev <rail@mozilla.com>
date:        Wed Feb 15 11:07:52 2012 -0500
summary:     Bug 607392 - split tagging into en-US and other. Backed out revisions 60b06686c840 and f84a95d878d6

changeset:   2118:8d4e66f79f5d
parent:      2114:73437b4c315b
user:        Rail Aliiev <rail@mozilla.com>
date:        Fri Feb 17 09:45:37 2012 -0500
summary:     Bug 728104 - AggregatingScheduler resets its state on reconfig. r=bhearsum

changeset:   2119:6fe26978479a
user:        Rail Aliiev <rail@mozilla.com>
date:        Fri Feb 17 11:39:33 2012 -0500
summary:     Bug 722608 - Remove android signature verification with Fennec 11.0 release. r=aki

changeset:   2120:dbee78ea932f
user:        Justin Wood <Callek@gmail.com>
date:        Fri Feb 17 13:20:03 2012 -0500
summary:     Bug 719260 - Investigate why updates builder triggered twice for 10.0b5. Use threading.Lock. r=rail f=dustin

changeset:   2122:7c1e450fe214
parent:      2120:dbee78ea932f
user:        Justin Wood <Callek@gmail.com>
date:        Sat Feb 18 16:49:24 2012 -0500
summary:     Update generateCCBranchObjects for changes in generateBranchObjects

changeset:   2124:e56221a93eff
tag:         tip
parent:      2122:7c1e450fe214
user:        Rail Aliiev <rail@mozilla.com>
date:        Tue Feb 21 00:04:46 2012 -0500
summary:     Bug 719260 - Investigate why updates builder triggered twice for 10.0b5. Bustage fix.

--HG--
branch : production-0.8
  • Loading branch information...
2 parents d92925f + 18cd1a2 commit a06b113f0839d68d15489afd99129181f61c4cf0 @rail rail committed Feb 21, 2012
Showing with 71 additions and 77 deletions.
  1. +12 −18 misc.py
  2. +24 −1 process/factory.py
  3. +0 −38 process/release.py
  4. +35 −20 scheduler.py
View
30 misc.py
@@ -415,6 +415,7 @@ def generateTestBuilder(config, branch_name, platform, name_prefix,
scriptName=suites['script_path'],
hg_bin=suites['hg_bin'],
extra_args=suites.get('extra_args', []),
+ reboot_command=suites.get('reboot_command'),
log_eval_func=lambda c,s: regex_log_evaluator(c, s, (
(re.compile('# TBPL WARNING #'), WARNINGS),
(re.compile('# TBPL FAILURE #'), FAILURE),
@@ -2275,9 +2276,8 @@ def generateCCBranchObjects(config, name, secrets=None):
factory_class = CCNightlyBuildFactory
uploadSymbols = False
- #stageBasePath = '%s/%s' % (config['stage_base_path'],
- # pf['stage_product'])
- stageBasePath = '%s' % (config['stage_base_path'])
+ stageBasePath = '%s/%s' % (config['stage_base_path'],
+ pf['stage_product'])
# For the 'per-checkin' pgo strategy, we want PGO
# enabled on what would be 'opt' builds.
@@ -2442,26 +2442,21 @@ def generateCCBranchObjects(config, name, secrets=None):
triggeredSchedulers=None
if config['enable_l10n'] and pf.get('is_mobile_l10n') and pf.get('l10n_chunks'):
mobile_l10n_scheduler_name = '%s-%s-l10n' % (name, platform)
- builder_env = platform_env.copy()
- builder_env.update({
- 'BUILDBOT_CONFIGS': '%s%s' % (config['hgurl'],
- config['config_repo_path']),
- 'CLOBBERER_URL': config['base_clobber_url'],
- })
mobile_l10n_builders = []
+ builder_env = platform_env.copy()
for n in range(1, int(pf['l10n_chunks']) + 1):
builddir='%s-%s-l10n_%s' % (name, platform, str(n))
builderName = "%s l10n nightly %s/%s" % \
(pf['base_name'], n, pf['l10n_chunks'])
mobile_l10n_builders.append(builderName)
factory = ScriptFactory(
scriptRepo='%s%s' % (config['hgurl'],
- config['build_tools_repo_path']),
- interpreter='bash',
- scriptName='scripts/l10n/nightly_mobile_repacks.sh',
- extra_args=[platform, stage_platform,
- getRealpath('localconfig.py'),
- str(pf['l10n_chunks']), str(n)]
+ config['mozharness_repo_path']),
+ scriptName='scripts/mobile_l10n.py',
+ extra_args=['--cfg',
+ 'single_locale/%s_%s.py' % (name, platform),
+ '--total-chunks', str(pf['l10n_chunks']),
+ '--this-chunk', str(n)]
)
slavebuilddir = reallyShort(builddir)
branchObjects['builders'].append({
@@ -3007,9 +3002,8 @@ def generateCCBranchObjects(config, name, secrets=None):
# -- end of per-platform loop --
if config['enable_weekly_bundle']:
- #stageBasePath = '%s/%s' % (config['stage_base_path'],
- # pf['stage_product'])
- stageBasePath = config['stage_base_path']
+ stageBasePath = '%s/%s' % (config['stage_base_path'],
+ pf['stage_product'])
bundle_factory = ScriptFactory(
config['hgurl'] + config['build_tools_repo_path'],
'scripts/bundle/hg-bundle.sh',
View
25 process/factory.py
@@ -8196,11 +8196,12 @@ class ScriptFactory(BuildFactory):
def __init__(self, scriptRepo, scriptName, cwd=None, interpreter=None,
extra_data=None, extra_args=None,
script_timeout=1200, script_maxtime=None, log_eval_func=None,
- hg_bin='hg'):
+ reboot_command=None, hg_bin='hg'):
BuildFactory.__init__(self)
self.script_timeout = script_timeout
self.log_eval_func = log_eval_func
self.script_maxtime = script_maxtime
+ self.reboot_command = reboot_command
if scriptName[0] == '/':
script_path = scriptName
else:
@@ -8257,6 +8258,7 @@ def __init__(self, scriptRepo, scriptName, cwd=None, interpreter=None,
workdir='scripts'
))
self.runScript()
+ self.reboot()
def runScript(self):
self.addStep(ShellCommand(
@@ -8280,6 +8282,27 @@ def runScript(self):
flunkOnFailure=False,
))
+ def reboot(self):
+ def do_disconnect(cmd):
+ try:
+ if 'SCHEDULED REBOOT' in cmd.logs['stdio'].getText():
+ return True
+ except:
+ pass
+ return False
+ if self.reboot_command:
+ self.addStep(DisconnectStep(
+ name='reboot',
+ flunkOnFailure=False,
+ warnOnFailure=False,
+ alwaysRun=True,
+ workdir='.',
+ description="reboot",
+ command=self.reboot_command,
+ force_disconnect=do_disconnect,
+ env=self.env,
+ ))
+
class SigningScriptFactory(ScriptFactory):
def __init__(self, signingServers, env, enableSigning=True,
View
38 process/release.py
@@ -1310,36 +1310,6 @@ def majorUpdateVerifyBuilders(platform):
}
})
- if releaseConfig['productName'] == 'fennec':
- # TODO: remove android_signature_verification related parts when the
- # verification procedure moved to post singing scripts
- envJava = builder_env.copy()
- envJava['PATH'] = '/tools/jdk6/bin:%s' % envJava.get(
- 'PATH', ':'.join(('/opt/local/bin', '/tools/python/bin',
- '/tools/buildbot/bin', '/usr/kerberos/bin',
- '/usr/local/bin', '/bin', '/usr/bin',
- '/home/cltbld/bin')))
- signature_verification_factory = ScriptFactory(
- scriptRepo=tools_repo,
- scriptName='release/signing/verify-android-signature.sh',
- extra_args=['--tools-dir=scripts/', '--release',
- WithProperties('--apk=%(who)s')]
- )
- builders.append({
- 'name': builderPrefix('android_signature_verification'),
- 'slavenames': branchConfig['platforms']['linux']['slaves'],
- 'category': builderPrefix(''),
- 'builddir': builderPrefix('android_verify_sig'),
- 'factory': signature_verification_factory,
- 'env': envJava,
- 'properties': {
- 'builddir': builderPrefix('android_verify_sig'),
- 'slavebuilddir':
- reallyShort(builderPrefix('android_verify_sig')),
- },
- })
-
-
##### Change sources and Schedulers
if not releaseConfig.get('enableSigningAtBuildTime', True) and \
@@ -1540,14 +1510,6 @@ def majorUpdateVerifyBuilders(platform):
schedulers.append(mirror_scheduler2)
- if releaseConfig['productName'] == 'fennec':
- android_signature_verification_scheduler = Scheduler(
- name=builderPrefix('android_signature_verification_scheduler'),
- branch=builderPrefix('android_post_signing'),
- treeStableTimer=None,
- builderNames=[builderPrefix('android_signature_verification')],
- )
- schedulers.append(android_signature_verification_scheduler)
if releaseConfig.get('enableAutomaticPushToMirrors') and \
releaseConfig.get('verifyConfigs'):
View
55 scheduler.py
@@ -3,6 +3,7 @@
# Contributor(s):
# Chris AtLee <catlee@mozilla.com>
+from threading import Lock
from twisted.internet import defer, reactor
from twisted.python import log
from twisted.internet.task import LoopingCall
@@ -352,18 +353,26 @@ class AggregatingScheduler(BaseScheduler, Triggerable):
compare_attrs = ('name', 'branch', 'builderNames', 'properties',
'upstreamBuilders', 'okResults')
- working = False
def __init__(self, name, branch, builderNames, upstreamBuilders,
okResults=(SUCCESS,WARNINGS), properties={}):
BaseScheduler.__init__(self, name, builderNames, properties)
self.branch = branch
+ self.lock = Lock()
+ assert isinstance(upstreamBuilders, (list, tuple))
self.upstreamBuilders = upstreamBuilders
self.reason = "AccumulatingScheduler(%s)" % name
self.okResults = okResults
+ self.log_prefix = '%s(%s) <id=%s>' % (self.__class__.__name__, name,
+ id(self))
def get_initial_state(self, max_changeid):
+ log.msg('%s: get_initial_state()' % self.log_prefix)
+ # Keep initial state of builders in upstreamBuilders
+ # and operate on remainingBuilders to simplify comparison
+ # on reconfig
return {
+ "upstreamBuilders": self.upstreamBuilders,
"remainingBuilders": self.upstreamBuilders,
"lastCheck": now(),
}
@@ -379,36 +388,38 @@ def _startService(self, t):
for b in list(state['remainingBuilders']):
if b not in self.upstreamBuilders:
state['remainingBuilders'].remove(b)
- # Add new upstream builders
+ # Add new upstream builders. New builders shouln't be in
+ # state['upstreamBuilders'] which contains old self.upstreamBuilders.
+ # Since state['upstreamBuilders'] was introduced after
+ # state['remainingBuilders'], it may be absent from the scheduler
+ # database.
for b in self.upstreamBuilders:
- if b not in state['remainingBuilders']:
+ if b not in state.get('upstreamBuilders', []) and \
+ b not in state['remainingBuilders']:
state['remainingBuilders'].append(b)
- log.msg('%s <id=%s>: reloaded' % (self.__class__.__name__,
- id(self)))
+ state['upstreamBuilders'] = self.upstreamBuilders
+ log.msg('%s: reloaded' % self.log_prefix)
if old_state != state:
- log.msg('%s <id=%s>: old state: %s' % (self.__class__.__name__,
- id(self), old_state))
- log.msg('%s <id=%s>: new state: %s' % (self.__class__.__name__,
- id(self), state))
+ log.msg('%s: old state: %s' % (self.log_prefix, old_state))
+ log.msg('%s: new state: %s' % (self.log_prefix, state))
self.set_state(t, state)
def trigger(self, ss, set_props=None):
"""Reset scheduler state"""
self.parent.db.runInteractionNow(self._trigger)
def _trigger(self, t):
+ log.msg('%s: _trigger attempting to acquire Lock' % self.log_prefix)
+ self.lock.acquire()
+ log.msg('%s: _trigger acquired Lock' % self.log_prefix)
state = self.get_initial_state(None)
state['lastReset'] = state['lastCheck']
- log.msg('%s <id=%s>: reset state: %s' % (self.__class__.__name__,
- id(self), state))
+ log.msg('%s: reset state: %s' % (self.log_prefix, state))
self.set_state(t, state)
+ self.lock.release()
+ log.msg('%s: _trigger released Lock' % self.log_prefix)
def run(self):
- if self.working:
- log.msg('%s <id=%s>: another instance is still running, skipping.' \
- % (self.__class__.__name__, id(self)))
- return
- self.working = True
d = self.parent.db.runInteraction(self._run)
return d
@@ -430,10 +441,14 @@ def findNewBuilds(self, db, t, lastCheck):
return t.fetchall()
def _run(self, t):
+ log.msg('%s: _run attempting to acquire Lock' % self.log_prefix)
+ self.lock.acquire()
+ log.msg('%s: _run acquired Lock' % self.log_prefix)
try:
self.processRequest(t)
finally:
- self.working = False
+ self.lock.release()
+ log.msg('%s: _run released Lock' % self.log_prefix)
def processRequest(self, t):
db = self.parent.db
@@ -458,9 +473,9 @@ def processRequest(self, t):
# Start a build!
log.msg(
- '%s <id=%s>: new buildset: name=%s, branch=%s, ssid=%s, builders: %s' \
- % (self.__class__.__name__, id(self), self.name,
- self.branch, ssid, ', '.join(self.builderNames)))
+ '%s: new buildset: branch=%s, ssid=%s, builders: %s' \
+ % (self.log_prefix, self.branch, ssid,
+ ', '.join(self.builderNames)))
self.create_buildset(ssid, "downstream", t)
# Reset the list of builders we're waiting for

0 comments on commit a06b113

Please sign in to comment.