Skip to content
This repository has been archived by the owner on Jun 11, 2019. It is now read-only.

Commit

Permalink
Merge from default
Browse files Browse the repository at this point in the history 
changeset:   2110:cc74a82b64a3
parent:      2108:f90c75b98eb7
user:        Aki Sasaki <asasaki@mozilla.com>
date:        Mon Feb 13 12:48:34 2012 -0800
summary:     bug 719511 - add optional reboot command to ScriptFactory. r=lsblakk

changeset:   2111:7fea4233c693
user:        Rail Aliiev <rail@mozilla.com>
date:        Tue Feb 14 10:56:56 2012 -0500
summary:     Bug 725292 - some repacks failed in 11.0b2 because of missing tokens. r=bhearsum

changeset:   2114:73437b4c315b
parent:      2111:7fea4233c693
user:        Rail Aliiev <rail@mozilla.com>
date:        Wed Feb 15 11:07:52 2012 -0500
summary:     Bug 607392 - split tagging into en-US and other. Backed out revisions 60b06686c840 and f84a95d878d6

changeset:   2118:8d4e66f79f5d
parent:      2114:73437b4c315b
user:        Rail Aliiev <rail@mozilla.com>
date:        Fri Feb 17 09:45:37 2012 -0500
summary:     Bug 728104 - AggregatingScheduler resets its state on reconfig. r=bhearsum

changeset:   2119:6fe26978479a
user:        Rail Aliiev <rail@mozilla.com>
date:        Fri Feb 17 11:39:33 2012 -0500
summary:     Bug 722608 - Remove android signature verification with Fennec 11.0 release. r=aki

changeset:   2120:dbee78ea932f
user:        Justin Wood <Callek@gmail.com>
date:        Fri Feb 17 13:20:03 2012 -0500
summary:     Bug 719260 - Investigate why updates builder triggered twice for 10.0b5. Use threading.Lock. r=rail f=dustin

changeset:   2122:7c1e450fe214
parent:      2120:dbee78ea932f
user:        Justin Wood <Callek@gmail.com>
date:        Sat Feb 18 16:49:24 2012 -0500
summary:     Update generateCCBranchObjects for changes in generateBranchObjects

changeset:   2124:e56221a93eff
tag:         tip
parent:      2122:7c1e450fe214
user:        Rail Aliiev <rail@mozilla.com>
date:        Tue Feb 21 00:04:46 2012 -0500
summary:     Bug 719260 - Investigate why updates builder triggered twice for 10.0b5. Bustage fix.

--HG--
branch : production-0.8
  • Loading branch information
Rail Aliiev committed Feb 21, 2012
2 parents d92925f + 18cd1a2 commit a06b113
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 77 deletions.
30 changes: 12 additions & 18 deletions misc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,7 @@ def generateTestBuilder(config, branch_name, platform, name_prefix,
scriptName=suites['script_path'],
hg_bin=suites['hg_bin'],
extra_args=suites.get('extra_args', []),
reboot_command=suites.get('reboot_command'),
log_eval_func=lambda c,s: regex_log_evaluator(c, s, (
(re.compile('# TBPL WARNING #'), WARNINGS),
(re.compile('# TBPL FAILURE #'), FAILURE),
Expand Down Expand Up @@ -2275,9 +2276,8 @@ def generateCCBranchObjects(config, name, secrets=None):
factory_class = CCNightlyBuildFactory
uploadSymbols = False

#stageBasePath = '%s/%s' % (config['stage_base_path'],
# pf['stage_product'])
stageBasePath = '%s' % (config['stage_base_path'])
stageBasePath = '%s/%s' % (config['stage_base_path'],
pf['stage_product'])

# For the 'per-checkin' pgo strategy, we want PGO
# enabled on what would be 'opt' builds.
Expand Down Expand Up @@ -2442,26 +2442,21 @@ def generateCCBranchObjects(config, name, secrets=None):
triggeredSchedulers=None
if config['enable_l10n'] and pf.get('is_mobile_l10n') and pf.get('l10n_chunks'):
mobile_l10n_scheduler_name = '%s-%s-l10n' % (name, platform)
builder_env = platform_env.copy()
builder_env.update({
'BUILDBOT_CONFIGS': '%s%s' % (config['hgurl'],
config['config_repo_path']),
'CLOBBERER_URL': config['base_clobber_url'],
})
mobile_l10n_builders = []
builder_env = platform_env.copy()
for n in range(1, int(pf['l10n_chunks']) + 1):
builddir='%s-%s-l10n_%s' % (name, platform, str(n))
builderName = "%s l10n nightly %s/%s" % \
(pf['base_name'], n, pf['l10n_chunks'])
mobile_l10n_builders.append(builderName)
factory = ScriptFactory(
scriptRepo='%s%s' % (config['hgurl'],
config['build_tools_repo_path']),
interpreter='bash',
scriptName='scripts/l10n/nightly_mobile_repacks.sh',
extra_args=[platform, stage_platform,
getRealpath('localconfig.py'),
str(pf['l10n_chunks']), str(n)]
config['mozharness_repo_path']),
scriptName='scripts/mobile_l10n.py',
extra_args=['--cfg',
'single_locale/%s_%s.py' % (name, platform),
'--total-chunks', str(pf['l10n_chunks']),
'--this-chunk', str(n)]
)
slavebuilddir = reallyShort(builddir)
branchObjects['builders'].append({
Expand Down Expand Up @@ -3007,9 +3002,8 @@ def generateCCBranchObjects(config, name, secrets=None):
# -- end of per-platform loop --

if config['enable_weekly_bundle']:
#stageBasePath = '%s/%s' % (config['stage_base_path'],
# pf['stage_product'])
stageBasePath = config['stage_base_path']
stageBasePath = '%s/%s' % (config['stage_base_path'],
pf['stage_product'])
bundle_factory = ScriptFactory(
config['hgurl'] + config['build_tools_repo_path'],
'scripts/bundle/hg-bundle.sh',
Expand Down
25 changes: 24 additions & 1 deletion process/factory.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8196,11 +8196,12 @@ class ScriptFactory(BuildFactory):
def __init__(self, scriptRepo, scriptName, cwd=None, interpreter=None,
extra_data=None, extra_args=None,
script_timeout=1200, script_maxtime=None, log_eval_func=None,
hg_bin='hg'):
reboot_command=None, hg_bin='hg'):
BuildFactory.__init__(self)
self.script_timeout = script_timeout
self.log_eval_func = log_eval_func
self.script_maxtime = script_maxtime
self.reboot_command = reboot_command
if scriptName[0] == '/':
script_path = scriptName
else:
Expand Down Expand Up @@ -8257,6 +8258,7 @@ def __init__(self, scriptRepo, scriptName, cwd=None, interpreter=None,
workdir='scripts'
))
self.runScript()
self.reboot()

def runScript(self):
self.addStep(ShellCommand(
Expand All @@ -8280,6 +8282,27 @@ def runScript(self):
flunkOnFailure=False,
))

def reboot(self):
def do_disconnect(cmd):
try:
if 'SCHEDULED REBOOT' in cmd.logs['stdio'].getText():
return True
except:
pass
return False
if self.reboot_command:
self.addStep(DisconnectStep(
name='reboot',
flunkOnFailure=False,
warnOnFailure=False,
alwaysRun=True,
workdir='.',
description="reboot",
command=self.reboot_command,
force_disconnect=do_disconnect,
env=self.env,
))

class SigningScriptFactory(ScriptFactory):

def __init__(self, signingServers, env, enableSigning=True,
Expand Down
38 changes: 0 additions & 38 deletions process/release.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1310,36 +1310,6 @@ def majorUpdateVerifyBuilders(platform):
}
})

if releaseConfig['productName'] == 'fennec':
# TODO: remove android_signature_verification related parts when the
# verification procedure moved to post singing scripts
envJava = builder_env.copy()
envJava['PATH'] = '/tools/jdk6/bin:%s' % envJava.get(
'PATH', ':'.join(('/opt/local/bin', '/tools/python/bin',
'/tools/buildbot/bin', '/usr/kerberos/bin',
'/usr/local/bin', '/bin', '/usr/bin',
'/home/cltbld/bin')))
signature_verification_factory = ScriptFactory(
scriptRepo=tools_repo,
scriptName='release/signing/verify-android-signature.sh',
extra_args=['--tools-dir=scripts/', '--release',
WithProperties('--apk=%(who)s')]
)
builders.append({
'name': builderPrefix('android_signature_verification'),
'slavenames': branchConfig['platforms']['linux']['slaves'],
'category': builderPrefix(''),
'builddir': builderPrefix('android_verify_sig'),
'factory': signature_verification_factory,
'env': envJava,
'properties': {
'builddir': builderPrefix('android_verify_sig'),
'slavebuilddir':
reallyShort(builderPrefix('android_verify_sig')),
},
})


##### Change sources and Schedulers

if not releaseConfig.get('enableSigningAtBuildTime', True) and \
Expand Down Expand Up @@ -1540,14 +1510,6 @@ def majorUpdateVerifyBuilders(platform):

schedulers.append(mirror_scheduler2)

if releaseConfig['productName'] == 'fennec':
android_signature_verification_scheduler = Scheduler(
name=builderPrefix('android_signature_verification_scheduler'),
branch=builderPrefix('android_post_signing'),
treeStableTimer=None,
builderNames=[builderPrefix('android_signature_verification')],
)
schedulers.append(android_signature_verification_scheduler)

if releaseConfig.get('enableAutomaticPushToMirrors') and \
releaseConfig.get('verifyConfigs'):
Expand Down
55 changes: 35 additions & 20 deletions scheduler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# Contributor(s):
# Chris AtLee <catlee@mozilla.com>

from threading import Lock
from twisted.internet import defer, reactor
from twisted.python import log
from twisted.internet.task import LoopingCall
Expand Down Expand Up @@ -352,18 +353,26 @@ class AggregatingScheduler(BaseScheduler, Triggerable):

compare_attrs = ('name', 'branch', 'builderNames', 'properties',
'upstreamBuilders', 'okResults')
working = False

def __init__(self, name, branch, builderNames, upstreamBuilders,
okResults=(SUCCESS,WARNINGS), properties={}):
BaseScheduler.__init__(self, name, builderNames, properties)
self.branch = branch
self.lock = Lock()
assert isinstance(upstreamBuilders, (list, tuple))
self.upstreamBuilders = upstreamBuilders
self.reason = "AccumulatingScheduler(%s)" % name
self.okResults = okResults
self.log_prefix = '%s(%s) <id=%s>' % (self.__class__.__name__, name,
id(self))

def get_initial_state(self, max_changeid):
log.msg('%s: get_initial_state()' % self.log_prefix)
# Keep initial state of builders in upstreamBuilders
# and operate on remainingBuilders to simplify comparison
# on reconfig
return {
"upstreamBuilders": self.upstreamBuilders,
"remainingBuilders": self.upstreamBuilders,
"lastCheck": now(),
}
Expand All @@ -379,36 +388,38 @@ def _startService(self, t):
for b in list(state['remainingBuilders']):
if b not in self.upstreamBuilders:
state['remainingBuilders'].remove(b)
# Add new upstream builders
# Add new upstream builders. New builders shouln't be in
# state['upstreamBuilders'] which contains old self.upstreamBuilders.
# Since state['upstreamBuilders'] was introduced after
# state['remainingBuilders'], it may be absent from the scheduler
# database.
for b in self.upstreamBuilders:
if b not in state['remainingBuilders']:
if b not in state.get('upstreamBuilders', []) and \
b not in state['remainingBuilders']:
state['remainingBuilders'].append(b)
log.msg('%s <id=%s>: reloaded' % (self.__class__.__name__,
id(self)))
state['upstreamBuilders'] = self.upstreamBuilders
log.msg('%s: reloaded' % self.log_prefix)
if old_state != state:
log.msg('%s <id=%s>: old state: %s' % (self.__class__.__name__,
id(self), old_state))
log.msg('%s <id=%s>: new state: %s' % (self.__class__.__name__,
id(self), state))
log.msg('%s: old state: %s' % (self.log_prefix, old_state))
log.msg('%s: new state: %s' % (self.log_prefix, state))
self.set_state(t, state)

def trigger(self, ss, set_props=None):
"""Reset scheduler state"""
self.parent.db.runInteractionNow(self._trigger)

def _trigger(self, t):
log.msg('%s: _trigger attempting to acquire Lock' % self.log_prefix)
self.lock.acquire()
log.msg('%s: _trigger acquired Lock' % self.log_prefix)
state = self.get_initial_state(None)
state['lastReset'] = state['lastCheck']
log.msg('%s <id=%s>: reset state: %s' % (self.__class__.__name__,
id(self), state))
log.msg('%s: reset state: %s' % (self.log_prefix, state))
self.set_state(t, state)
self.lock.release()
log.msg('%s: _trigger released Lock' % self.log_prefix)

def run(self):
if self.working:
log.msg('%s <id=%s>: another instance is still running, skipping.' \
% (self.__class__.__name__, id(self)))
return
self.working = True
d = self.parent.db.runInteraction(self._run)
return d

Expand All @@ -430,10 +441,14 @@ def findNewBuilds(self, db, t, lastCheck):
return t.fetchall()

def _run(self, t):
log.msg('%s: _run attempting to acquire Lock' % self.log_prefix)
self.lock.acquire()
log.msg('%s: _run acquired Lock' % self.log_prefix)
try:
self.processRequest(t)
finally:
self.working = False
self.lock.release()
log.msg('%s: _run released Lock' % self.log_prefix)

def processRequest(self, t):
db = self.parent.db
Expand All @@ -458,9 +473,9 @@ def processRequest(self, t):

# Start a build!
log.msg(
'%s <id=%s>: new buildset: name=%s, branch=%s, ssid=%s, builders: %s' \
% (self.__class__.__name__, id(self), self.name,
self.branch, ssid, ', '.join(self.builderNames)))
'%s: new buildset: branch=%s, ssid=%s, builders: %s' \
% (self.log_prefix, self.branch, ssid,
', '.join(self.builderNames)))
self.create_buildset(ssid, "downstream", t)

# Reset the list of builders we're waiting for
Expand Down

0 comments on commit a06b113

Please sign in to comment.