Browse files

Bug 464165 - merge of Axel's buildbotcustom repo, including changes f…

…or l10n rep

ack-on-change - r=bhearsum
  • Loading branch information...
1 parent d1f6fc0 commit 62251183249545ee7eb1c8c664ef5d1141df79f7 @ccooper ccooper committed Mar 6, 2009
Showing with 2,108 additions and 362 deletions.
  1. +31 −0 bin/l10n-stage/README
  2. +24 −0 bin/l10n-stage/botmaster
  3. +251 −0 bin/l10n-stage/create-buildbot
  4. +138 −0 bin/l10n-stage/create-stage
  5. +284 −118 changes/hgpoller.py
  6. +968 −0 l10n.py
  7. 0 l10n/__init__.py
  8. +0 −226 l10n/scheduler.py
  9. +35 −18 l10n/l10n.py → log.py
  10. +183 −0 test/test_l10n.py
  11. +194 −0 test/utils.py
View
31 bin/l10n-stage/README
@@ -0,0 +1,31 @@
+The scripts
+
+create-stage,
+create-buildbot, and
+botmaster
+
+provide help in setting up and running a testing environment for how we
+schedule l10n builds.
+
+Use create-stage to create the required repos as well as one en-US and
+four l10n dummy repositories. Those will already have an initial push in
+the pushlog database. The repositories are in the repos directory.
+
+Clones to work on these and to then push changes are in the workdir directory
+tree. When pushing, you need to make sure that mozhghooks in
+repos/hghooks is in your PYTHONPATH.
+
+create-buildbot sets up a master and a number of slaves for each of the
+three platforms. The master is set up to have en-US dep builds, nightlies,
+and l10n builds, with build properties depending on whether it's an en-US
+change-triggered build, l10n triggered or nightly.
+
+Last but not least, botmaster offers start and stop commands to start and
+stop the master and all slaves.
+
+All commands take the staging dir as argument.
+
+After creating the hg repositories, you want to set up a
+hg serve -p 8080 --webdir-conf webdir.conf
+in the staging dir, that will offer the repositories on the localhost.
+Make sure to map the port here with what you specify in create-buildbot.
View
24 bin/l10n-stage/botmaster
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+from optparse import OptionParser
+import os
+import subprocess
+
+if __name__ == "__main__":
+ p = OptionParser()
+ (options, args) = p.parse_args()
+
+ dest = args[1]
+ command = args[0]
+
+ for d in sorted(os.listdir(dest)):
+ dir = os.path.join(dest, d)
+ pid = os.path.join(dir, 'twistd.pid')
+ if command == 'start':
+ if os.path.isfile(pid):
+ subprocess.call(['buildbot', 'restart', dir])
+ else:
+ subprocess.call(['buildbot', 'start', dir])
+ if command == 'stop':
+ if os.path.isfile(pid):
+ subprocess.call(['buildbot', 'stop', dir])
View
251 bin/l10n-stage/create-buildbot
@@ -0,0 +1,251 @@
+#!/usr/bin/env python
+
+from optparse import OptionParser
+import os
+import subprocess
+
+master_cfg = '''c = BuildmasterConfig = {}
+from buildbot.buildslave import BuildSlave
+from buildbot import locks, manhole, scheduler
+from buildbot.process import factory
+from buildbot.steps.dummy import Dummy
+from buildbot.status import html
+
+from twisted.internet import reactor
+
+from buildbotcustom.l10n import Scheduler, NightlyL10n
+from buildbotcustom.changes.hgpoller import HgAllLocalesPoller, HgPoller
+
+from twisted.web.client import HTTPClientFactory
+HTTPClientFactory.noisy = False
+
+c['slaves'] = [
+%(slaves)s
+ ]
+
+c['slavePortnum'] = %(port)s
+
+hga = HgAllLocalesPoller('http://localhost:%(hg_port)s/',
+ 'l10n', pollInterval = 60)
+hga.parallelRequests = 1
+c['change_source'] = [
+ hga,
+ HgPoller('http://localhost:%(hg_port)s/', 'mozilla',
+ pollInterval = 90)
+]
+
+c['schedulers'] = [
+ Scheduler('l10n', 'l10nbuilds.ini'),
+ NightlyL10n('l10n nightly',
+ ['Firefox test linux', 'Firefox test mac',
+ 'Firefox test win'],
+ 'hg',
+ repo='http://localhost:%(hg_port)s/',
+ branch='mozilla',
+ localesFile='browser/locales/all-locales',
+ minute=5),
+ scheduler.Scheduler('enUS dep', 'mozilla', 30,
+ ['Linux mozilla build',
+ 'OS X 10.5.2 mozilla build',
+ 'WINNT 5.2 mozilla build']),
+ scheduler.Nightly('enUS nightly',
+ ['Linux mozilla nightly',
+ 'OS X 10.5.2 mozilla nightly',
+ 'WINNT 5.2 mozilla nightly'])
+]
+
+bigSlaveLock = locks.SlaveLock("bigSlaveLock")
+
+f = factory.BuildFactory()
+class L10nReporter(Dummy):
+ def start(self):
+ p = self.step_status.build.getProperties()
+ desc = [p['locale']]
+ if p.has_key('tree'):
+ desc += [p['tree'], p['app']]
+ self.desc = desc
+ if p.has_key('compareOnly'):
+ desc.append('compareOnly')
+ if p.has_key('nightly'):
+ desc.append('nightly')
+ desc.append('building')
+ desc.append('on %%s' %% p['slavename'])
+ self.step_status.setText(desc)
+ self.desc = desc[:]
+ timeout = self.timeout
+ if 'win' in p['buildername'].lower():
+ timeout *= 3
+ if 'mac' in p['buildername'].lower():
+ timeout *= 2
+ self.timer = reactor.callLater(timeout, self.done)
+ def done(self):
+ self.desc[-2] = 'built'
+ self.step_status.setText(self.desc)
+ Dummy.done(self)
+
+f.addStep(L10nReporter, timeout=2, name='l10nstep')
+
+enBuild = factory.BuildFactory()
+class EnBuildStep(Dummy):
+ def start(self):
+ p = self.step_status.build.getProperties()
+ desc = ['en-US', 'building',
+ 'on %%s' %% p['slavename']]
+ self.desc = desc[:]
+ self.step_status.setText(desc)
+ timeout = self.timeout
+ if 'win' in p['buildername'].lower():
+ timeout *= 3
+ if 'os x' in p['buildername'].lower():
+ timeout *= 2
+ self.timer = reactor.callLater(timeout, self.done)
+ def done(self):
+ self.desc[1] = 'built'
+ self.step_status.setText(self.desc)
+ Dummy.done(self)
+
+enBuild.addStep(EnBuildStep, timeout=15, name='enstep')
+
+enNightly = factory.BuildFactory()
+enNightly.addStep(EnBuildStep, timeout=15, name='enstep')
+
+c['builders'] = [
+ {'name': 'Firefox test linux',
+ 'slavenames': [%(slaves_linux)s],
+ 'builddir': 'fx_test_linux',
+ 'factory': f,
+ 'category': 'l10n',
+ 'locks': [bigSlaveLock]},
+ {'name': 'Linux mozilla build',
+ 'slavenames': [%(slaves_linux)s],
+ 'builddir': 'fx_en_linux',
+ 'factory': enBuild,
+ 'category': 'mozilla',
+ 'locks': [bigSlaveLock]},
+ {'name': 'Linux mozilla nightly',
+ 'slavenames': [%(slaves_linux)s],
+ 'builddir': 'fx_en_nightly_linux',
+ 'factory': enNightly,
+ 'category': 'mozilla',
+ 'locks': [bigSlaveLock]},
+ {'name': 'Firefox test mac',
+ 'slavenames': [%(slaves_mac)s],
+ 'builddir': 'fx_test_mac',
+ 'factory': f,
+ 'category': 'l10n',
+ 'locks': [bigSlaveLock]},
+ {'name': 'OS X 10.5.2 mozilla build',
+ 'slavenames': [%(slaves_mac)s],
+ 'builddir': 'fx_en_mac',
+ 'factory': enBuild,
+ 'category': 'mozilla',
+ 'locks': [bigSlaveLock]},
+ {'name': 'OS X 10.5.2 mozilla nightly',
+ 'slavenames': [%(slaves_mac)s],
+ 'builddir': 'fx_en_nightly_mac',
+ 'factory': enNightly,
+ 'category': 'mozilla',
+ 'locks': [bigSlaveLock]},
+ {'name': 'Firefox test win',
+ 'slavenames': [%(slaves_win)s],
+ 'builddir': 'fx_test_win',
+ 'factory': f,
+ 'category': 'l10n',
+ 'locks': [bigSlaveLock]},
+ {'name': 'WINNT 5.2 mozilla build',
+ 'slavenames': [%(slaves_win)s],
+ 'builddir': 'fx_en_win',
+ 'factory': enBuild,
+ 'category': 'mozilla',
+ 'locks': [bigSlaveLock]},
+ {'name': 'WINNT 5.2 mozilla nightly',
+ 'slavenames': [%(slaves_win)s],
+ 'builddir': 'fx_en_nightly_win',
+ 'factory': enNightly,
+ 'category': 'mozilla',
+ 'locks': [bigSlaveLock]},
+ ]
+
+c['manhole'] = manhole.TelnetManhole(%(telnet_port)s, "%(telnet_user)s",
+ "%(telnet_pass)s")
+
+c['status'] = []
+
+c['status'].append(html.WebStatus(http_port=%(http_port)s,
+ allowForce=True))
+c['buildbotURL'] = 'http://localhost:%(http_port)s/'
+c['projectName'] = 'l10n testbed'
+'''
+
+l10nini = '''[fx]
+app = browser
+type = hg
+locales = all
+mozilla = mozilla
+l10n = l10n
+repo = http://localhost:%(hgport)s/
+l10n.ini = browser/locales/l10n.ini
+builders = Firefox\ test\ linux Firefox\ test\ mac Firefox\ test\ win
+'''
+
+def createMaster(dest, opts):
+ if os.path.isdir(dest):
+ raise RuntimeError("Upgrading not supported")
+ os.makedirs(dest)
+ rv = subprocess.call(['buildbot', 'create-master', 'master'],
+ cwd=dest)
+ if rv:
+ raise RuntimeError('creation of master failed')
+ l_s = ['"sl%d"' % i for i in xrange(opts.slaves)]
+ m_s = ['"sm%d"' % i for i in xrange(opts.slaves)]
+ w_s = ['"sw%d"' % i for i in xrange(opts.slaves)]
+ def formatSlave(s):
+ return ' BuildSlave(%s, "pwd"),\n' % s
+ open(os.path.join(dest, 'master', 'master.cfg'),
+ 'w').write(master_cfg % {
+ 'slaves': ''.join(map(formatSlave, l_s + m_s + w_s)),
+ 'slaves_linux': ','.join(l_s),
+ 'slaves_mac': ','.join(m_s),
+ 'slaves_win': ','.join(w_s),
+ 'port': options.port,
+ 'http_port': options.http_port,
+ 'hg_port': options.hg_port,
+ 'telnet_port': options.telnet_port,
+ 'telnet_user': options.telnet_user,
+ 'telnet_pass': options.telnet_pass,
+ })
+ open(os.path.join(dest, 'master', 'l10nbuilds.ini'),
+ 'w').write(l10nini % {'hgport': opts.hg_port})
+
+def createSlave(dest, name, opts):
+ if os.path.isdir(os.path.join(dest, name)):
+ return
+ rv = subprocess.call(['buildbot', 'create-slave', name,
+ 'localhost:%s' % opts.port,
+ name, 'pwd'],
+ cwd=dest)
+ if rv:
+ raise RuntimeError('creation of master failed')
+
+
+if __name__ == "__main__":
+ p = OptionParser()
+ p.add_option('-v', dest='verbose', action='store_true')
+ p.add_option('-p', '--port', default='9876')
+ p.add_option('-q', '--http-port', dest='http_port',
+ default='8010')
+ p.add_option('-g', '--hg-port', dest='hg_port',
+ default='8000')
+ p.add_option('-n', '--slaves', type='int', default=3)
+ p.add_option('--telnet-port', dest='telnet_port', default='9875')
+ p.add_option('--telnet-user', dest='telnet_user', default='god')
+ p.add_option('--telnet-pass', dest='telnet_pass', default='knows')
+ (options, args) = p.parse_args()
+
+ dest = args[0]
+
+ createMaster(dest, options)
+ for i in xrange(options.slaves):
+ createSlave(dest, "sl%d" % i, options)
+ createSlave(dest, "sm%d" % i, options)
+ createSlave(dest, "sw%d" % i, options)
View
138 bin/l10n-stage/create-stage
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+
+from optparse import OptionParser
+import os
+import subprocess
+
+repos = (
+ 'http://hg.mozilla.org/users/bsmedberg_mozilla.com/hghooks/',
+ 'http://hg.mozilla.org/users/bsmedberg_mozilla.com/hgpoller/',
+ 'http://hg.mozilla.org/hg_templates/',
+ )
+
+downstreams = (
+ 'mozilla',
+ 'l10n/ab',
+ 'l10n/de',
+ 'l10n/ja-JP-mac',
+ 'l10n/x-testing',
+ )
+
+def ensureUpstreamRepo(r, dest):
+ base = os.path.join(dest, 'repos')
+ if not os.path.isdir(base):
+ os.makedirs(base)
+ leaf = r.rsplit('/', 2)[1]
+ if os.path.isdir(os.path.join(base, leaf, '.hg')):
+ return
+ rv = subprocess.call(['hg', 'clone', r], cwd = base)
+ if rv:
+ raise RuntimeError('hg failed to clone %s' % leaf)
+
+
+def ensureRepo(leaf, dest):
+ base = os.path.join(dest, 'repos')
+ if not os.path.isdir(base):
+ os.makedirs(base)
+ if os.path.isdir(os.path.join(base, leaf)):
+ return
+
+ os.makedirs(os.path.join(base, leaf))
+ rv = subprocess.call(['hg', 'init', leaf], cwd = base)
+ if rv:
+ raise RuntimeError('Couldnt hg init %s' % leaf)
+ tail = '''
+[hooks]
+pretxnchangegroup.a_singlehead = python:mozhghooks.single_head_per_branch.hook
+pretxnchangegroup.z_linearhistory = python:mozhghooks.pushlog.log
+
+[extensions]
+pushlog-feed = %(dest)s/repos/hgpoller/pushlog-feed.py
+buglink = %(dest)s/repos/hgpoller/buglink.py
+
+[web]
+style = gitweb_mozilla
+templates = %(dest)s/repos/hg_templates
+'''
+ hgrc = open(os.path.join(base, leaf, '.hg', 'hgrc'), 'a')
+ hgrc.write(tail % {'dest': os.path.abspath(dest)})
+ hgrc.close()
+
+ rv = subprocess.call(['hg', 'clone', leaf,
+ os.path.join('..', 'workdir', leaf)],
+ cwd=base)
+ if rv:
+ raise RuntimeError('clone for %s failed' % leaf)
+ browserdir = os.path.join(dest, 'workdir', leaf, 'browser')
+ if leaf.startswith('l10n'):
+ # create initial content for l10n
+ os.makedirs(browserdir)
+ open(os.path.join(browserdir, 'file.properties'),
+ 'w').write('''k_e_y: %s value
+''' % leaf)
+ else:
+ # create initial content for mozilla
+ os.makedirs(os.path.join(browserdir, 'locales', 'en-US'))
+ open(os.path.join(browserdir, 'locales', 'en-US', 'file.properties'),
+ 'w').write('''k_e_y: en-US value
+''')
+ open(os.path.join(browserdir, 'locales', 'all-locales'),
+ 'w').write('''ab
+de
+ja-JP-mac
+x-testing
+''')
+ open(os.path.join(browserdir, 'locales', 'l10n.ini'),
+ 'w').write('''[general]
+depth = ../..
+all = browser/locales/all-locales
+
+[compare]
+dirs = browser
+''')
+ env = dict(os.environ)
+ if 'PYTHONPATH' in env:
+ env['PYTHONPATH'] += ':%s/hghooks' % os.path.abspath(base)
+ else:
+ env['PYTHONPATH'] = '%s/hghooks' % os.path.abspath(base)
+ rv = subprocess.call(['hg', 'add', '.'], cwd=browserdir)
+ if rv:
+ raise RuntimeError('failed to add initial content')
+ rv = subprocess.call(['hg', 'ci', '-mInitial commit for %s' % leaf],
+ cwd=browserdir)
+ if rv:
+ raise RuntimeError('failed to check in initian content to %s' %
+ leaf)
+ rv = subprocess.call(['hg', 'push'], cwd=browserdir, env=env)
+ if rv:
+ raise RuntimeError('failed to push to %s' % leaf)
+
+
+def createWebDir(dest, port):
+ content = '''[collections]
+repos = repos
+'''
+ if not os.path.isfile(os.path.join(dest, 'webdir.conf')):
+ open(os.path.join(dest, 'webdir.conf'),
+ 'w').write(content % {'port': port,
+ 'dest': os.path.abspath(dest)})
+
+if __name__ == "__main__":
+ p = OptionParser()
+ p.add_option('-v', dest='verbose', action='store_true')
+ p.add_option('-p', '--port', default='8000')
+ (options, args) = p.parse_args()
+
+ dest = args[0]
+
+ if not os.path.isdir(os.path.join(dest, 'workdir', 'l10n')):
+ os.makedirs(os.path.join(dest, 'workdir', 'l10n'))
+
+ for r in repos:
+ ensureUpstreamRepo(r, dest)
+
+
+ for l in downstreams:
+ ensureRepo(l, dest)
+
+ createWebDir(dest, options.port)
View
402 changes/hgpoller.py
@@ -1,6 +1,34 @@
+"""hgpoller provides Pollers to work on single hg repositories as well
+as on a group of hg repositories. It's polling the RSS feed of pushlog,
+which is XML of the form
+
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ <id>http://hg.mozilla.org/l10n-central/repo/pushlog</id>
+ <link rel="self" href="http://hg.mozilla.org/l10n-central/repo/pushlog" />
+ <updated>2009-02-09T23:10:59Z</updated>
+ <title>repo Pushlog</title>
+ <entry>
+ <title>Changeset 3dd5e26f1334ad08a333a7acbe7649af7450feda</title>
+ <id>http://www.selenic.com/mercurial/#changeset-3dd5e26f1334ad08a333a7acbe7649af7450feda</id>
+ <link href="http://hg.mozilla.org/l10n-central/repo/rev/3dd5e26f1334ad08a333a7acbe7649af7450feda" />
+ <updated>2009-02-09T23:10:59Z</updated>
+ <author>
+ <name>ldap@domain.tld</name>
+ </author>
+ <content type="xhtml">
+ <div xmlns="http://www.w3.org/1999/xhtml">
+ <ul class="filelist"><li class="file">some/file/path</li></ul>
+ </div>
+ </content>
+ </entry>
+</feed>
+"""
+
import time
from calendar import timegm
from xml.dom import minidom, Node
+import operator
from twisted.python import log, failure
from twisted.internet import defer, reactor
@@ -180,7 +208,7 @@ def parse_date(datestring, default_timezone=UTC):
def parse_date_string(dateString):
return timegm(parse_date(dateString).utctimetuple())
-def _parse_changes(query, lastChange):
+def _parse_changes(query):
dom = minidom.parseString(query)
items = dom.getElementsByTagName("entry")
@@ -190,40 +218,171 @@ def _parse_changes(query, lastChange):
for k in ["title", "updated"]:
d[k] = i.getElementsByTagName(k)[0].firstChild.wholeText
d["updated"] = parse_date_string(d["updated"])
- d["changeset"] = d["title"].split(" ")[1]
+ _cs = d["title"].split(" ")[1]
+ assert _cs == str(_cs)
+ d["changeset"] = str(_cs)
nameNode = i.getElementsByTagName("author")[0].childNodes[1]
d["author"] = nameNode.firstChild.wholeText
d["link"] = i.getElementsByTagName("link")[0].getAttribute("href")
- if d["updated"] > lastChange:
- changes.append(d)
+ # Get all <li class="file"> elements
+ files = filter(lambda e: 'file' in e.getAttribute('class').split(),
+ i.getElementsByTagName('li'))
+ # For each <li class="file"> element, concat the data of all
+ # text node children.
+ # This way, we don't get confused if the DOM has split the file
+ # paths.
+ # We end up with a list of paths by using map()
+ d["files"] = map(lambda e: reduce(operator.add,
+ map(lambda t:t.data, e.childNodes),
+ ''),
+ files)
+ changes.append(d)
changes.reverse() # want them in chronological order
return changes
-
-class BaseHgPoller(object):
- """Common base of HgPoller, HgLocalePoller, and HgAllLocalesPoller.
- Subclasses should implement getData, processData, and __str__"""
- working = False
-
- def poll(self):
- if self.working:
- log.msg("Not polling %s because last poll is still working" % self)
+class Pluggable(object):
+ '''The Pluggable class implements a forward for Deferred's that
+ can be thrown away.
+
+ This is in particular useful when a network request doesn't really
+ error in a reasonable time, and you want to make sure that if it
+ answers after you tried to give up on it, it's not confusing the
+ rest of your app by calling back with data twice or something.
+ '''
+ def __init__(self, d):
+ self.d = defer.Deferred()
+ self.dead = False
+ d.addCallbacks(self.succeeded, self.failed)
+ def succeeded(self, result):
+ if self.dead:
+ log.msg("Dead pluggable got called")
+ else:
+ self.d.callback(result)
+ def failed(self, fail = None):
+ if self.dead:
+ log.msg("Dead pluggable got errbacked")
else:
- self.working = True
- d = self.getData()
- d.addCallback(self.processData)
- d.addCallbacks(self.dataFinished, self.dataFailed)
+ self.d.errback(fail)
- def dataFinished(self, res):
- assert self.working
- self.working = False
+class BasePoller(object):
+ attemptLimit = 3
+ def __init__(self):
+ self.attempts = 0
+ self.startLoad = 0
+ self.loadTime = None
+
+ def poll(self):
+ if self.attempts:
+ if self.attempts > self.attemptLimit:
+ self.plug.dead = True
+ self.attempts = 0
+ log.msg("dropping the ball on %s, starting new" % self)
+ else:
+ self.attempts += 1
+ log.msg("Not polling %s because last poll is still working" % self)
+ reactor.callLater(0, self.pollDone, None)
+ return
+ self.attempts = 1
+ self.startLoad = time.time()
+ self.loadTime = None
+ self.plug = Pluggable(self.getData())
+ d = self.plug.d
+ d.addCallback(self.stopLoad)
+ d.addCallback(self.processData)
+ d.addCallbacks(self.dataFinished, self.dataFailed)
+ d.addCallback(self.pollDone)
+
+ def stopLoad(self, res):
+ self.loadTime = time.time() - self.startLoad
return res
+ def dataFinished(self, res):
+ assert self.attempts
+ self.attempts = 0
+
def dataFailed(self, res):
- assert self.working
- self.working = False
- log.msg("%s: polling failed, result %s" % (self, res))
- return None
+ assert self.attempts
+ self.attempts = 0
+ log.msg("%s: polling failed, result %s" % (self, res.value.message))
+ res.printTraceback()
+
+ def pollDone(self, res):
+ pass
+
+
+
+class BaseHgPoller(BasePoller):
+ """Common base of HgPoller, HgLocalePoller, and HgAllLocalesPoller.
+
+ Subclasses should implement getData, processData, and __str__"""
+ verbose = True
+ timeout = 30
+
+ def __init__(self, hgURL, branch, pushlogUrlOverride=None,
+ tipsOnly=False, tree = None):
+ BasePoller.__init__(self)
+ self.hgURL = hgURL
+ self.branch = branch
+ self.tree = tree
+ if hgURL.endswith("/"):
+ hgURL = hgURL[:-1]
+ fragments = [hgURL, branch]
+ if tree is not None:
+ fragments.append(tree)
+ self.baseURL = "/".join(fragments)
+ self.pushlogUrlOverride = pushlogUrlOverride
+ self.tipsOnly = tipsOnly
+ self.lastChange = time.time()
+ self.lastChangeset = None
+ self.startLoad = 0
+ self.loadTime = None
+
+ def getData(self):
+ url = self._make_url()
+ if self.verbose:
+ log.msg("Polling Hg server at %s" % url)
+ return getPage(url, timeout = self.timeout)
+
+ def _make_url(self):
+ url = None
+ if self.pushlogUrlOverride:
+ url = self.pushlogUrlOverride
+ else:
+ url = "/".join((self.baseURL, 'pushlog'))
+
+ args = []
+ if self.lastChangeset is not None:
+ args.append('fromchange=' + self.lastChangeset)
+ if self.tipsOnly:
+ args.append('tipsonly=1')
+ if args:
+ url += '?' + '&'.join(args)
+
+ return url
+
+ def processData(self, query):
+ change_list = _parse_changes(query)
+ if self.lastChangeset is not None:
+ for change in change_list:
+ adjustedChangeTime = change["updated"]
+ c = changes.Change(who = change["author"],
+ files = change["files"],
+ revision = change["changeset"],
+ comments = change["link"],
+ when = adjustedChangeTime,
+ branch = self.branch)
+ self.changeHook(c)
+ self.parent.addChange(c)
+ if len(change_list) > 0:
+ self.lastChange = max(self.lastChange, *[c["updated"]
+ for c in change_list])
+ self.lastChangeset = change_list[-1]["changeset"]
+ if self.verbose:
+ log.msg("last changeset %s on %s" %
+ (self.lastChangeset, self.baseURL))
+
+ def changeHook(self, change):
+ pass
class HgPoller(base.ChangeSource, BaseHgPoller):
"""This source will poll a Mercurial server over HTTP using
@@ -252,12 +411,9 @@ def __init__(self, hgURL, branch, pushlogUrlOverride=None,
as *one* changeset
"""
- self.hgURL = hgURL
- self.branch = branch
- self.pushlogUrlOverride = pushlogUrlOverride
- self.tipsOnly = tipsOnly
+ BaseHgPoller.__init__(self, hgURL, branch, pushlogUrlOverride,
+ tipsOnly)
self.pollInterval = pollInterval
- self.lastChange = time.time()
def startService(self):
self.loop = LoopingCall(self.poll)
@@ -270,39 +426,6 @@ def stopService(self):
def describe(self):
return "Getting changes from: %s" % self._make_url()
-
- def _make_url(self):
- url = None
- if self.pushlogUrlOverride:
- url = self.pushlogUrlOverride
- else:
- url = "%s%s/pushlog" % (self.hgURL, self.branch)
-
- if self.tipsOnly:
- url += '?tipsonly=1'
-
- return url
-
-
- def getData(self):
- url = self._make_url()
- log.msg("Polling Hg server at %s" % url)
- return pollThrottler.getPage(url)
-
- def processData(self, query):
- change_list = _parse_changes(query, self.lastChange)
- for change in change_list:
- adjustedChangeTime = change["updated"]
- c = changes.Change(who = change["author"].encode("utf-8", "replace"),
- files = [], # sucks
- revision = change["changeset"].encode("utf-8", "replace"),
- comments = change["link"],
- when = adjustedChangeTime,
- branch = self.branch)
- self.parent.addChange(c)
- if len(change_list) > 0:
- self.lastChange = max(self.lastChange, *[c["updated"]
- for c in change_list])
def __str__(self):
return "<HgPoller for %s%s>" % (self.hgURL, self.branch)
@@ -311,68 +434,59 @@ class HgLocalePoller(BaseHgPoller):
"""This helper class for HgAllLocalesPoller polls a single locale and
submits changes if necessary."""
- def __init__(self, locale, parent, branch, url):
+ timeout = 30
+ verbose = False
+
+ def __init__(self, locale, parent, branch, hgURL):
+ BaseHgPoller.__init__(self, hgURL, branch, tree = locale)
self.locale = locale
self.parent = parent
self.branch = branch
- self.url = url
- self.lastChange = time.time()
- def getData(self):
- log.msg("Polling l10n Hg server at %s" % self.url)
- return pollThrottler.getPage(self.url)
+ def changeHook(self, change):
+ change.locale = self.locale
- def processData(self, query):
- change_list = _parse_changes(query, self.lastChange)
- for change in change_list:
- adjustedChangeTime = change["updated"]
- c = changes.Change(who = change["author"],
- files = [], # sucks
- revision = change["changeset"],
- comments = change["link"],
- when = adjustedChangeTime,
- branch = self.branch)
- c.locale = self.locale
- self.parent.addChange(c)
- if len(change_list) > 0:
- self.lastChange = max(self.lastChange, *[c["updated"]
- for c in change_list])
+ def pollDone(self, res):
+ self.parent.localeDone(self.locale)
def __str__(self):
- return "<HgLocalePoller for %s>" % self.url
+ return "<HgLocalePoller for %s>" % self.baseURL
-class HgAllLocalesPoller(base.ChangeSource, BaseHgPoller):
- """Poll every locale from an all-locales file."""
+class HgAllLocalesPoller(base.ChangeSource, BasePoller):
+ """Poll all localization repositories from an index page.
- compare_attrs = ['allLocalesURL', 'pollInterval',
- 'localePushlogURL', 'branch']
+ For a index page like http://hg.mozilla.org/releases/l10n-mozilla-1.9.1/,
+ all links look like /releases/l10n-mozilla-1.9.1/af/, where the last
+ path step will be the locale code, and the others will be passed
+ as branch for the changes, i.e. 'releases/l10n-mozilla-1.9.1'.
+ """
+
+ compare_attrs = ['repositoryIndex', 'pollInterval']
parent = None
loop = None
volatile = ['loop']
- def __init__(self, allLocalesURL, localePushlogURL, branch=None,
- pollInterval=120):
+ timeout = 10
+ parallelRequests = 2
+ verboseChilds = False
+
+ def __init__(self, hgURL, repositoryIndex, pollInterval=120):
"""
- @type allLocalesURL: string
- @param allLocalesURL: The URL of the all-locales file
- @type localePushlogURL: string
- @param localePushlogURL: The URL of the localized pushlogs.
- %(locale)s will be substituted.
+ @type repositoryIndex: string
+ @param repositoryIndex: The URL listing all locale repos
@type pollInterval int
@param pollInterval The time (in seconds) between queries for
changes
- @type branch string or None
- @param branch The name of the branch to report changes on.
- This only affects the Change, it doesn't
- affect the polling URLs at all!
"""
- self.allLocalesURL = allLocalesURL
- self.localePushlogURL = localePushlogURL
- self.branch = branch
+ BasePoller.__init__(self)
+ self.hgURL = hgURL
+ self.repositoryIndex = repositoryIndex
self.pollInterval = pollInterval
- self.lastChange = time.time()
self.localePollers = {}
+ self.locales = []
+ self.pendingLocales = []
+ self.activeRequests = 0
def startService(self):
self.loop = LoopingCall(self.poll)
@@ -383,25 +497,77 @@ def stopService(self):
self.loop.stop()
return base.ChangeSource.stopService(self)
+ def addChange(self, change):
+ self.parent.addChange(change)
+
def describe(self):
- return "Getting changes from all-locales at %s for repositories at %s" % (self.allLocalesURL, self.localePushlogURL)
+ return "Getting changes from all locales at %s" % self.repositoryIndex
def getData(self):
- log.msg("Polling all-locales at %s" % self.allLocalesURL)
- return pollThrottler.getPage(self.allLocalesURL)
-
- def getLocalePoller(self, locale):
- if locale not in self.localePollers:
- self.localePollers[locale] = \
- HgLocalePoller(locale, self.parent, self.branch,
- self.localePushlogURL % {'locale': locale})
- return self.localePollers[locale]
+ log.msg("Polling all locales at %s%s/" % (self.hgURL,
+ self.repositoryIndex))
+ return getPage(self.hgURL + self.repositoryIndex + '/?style=raw',
+ timeout = self.timeout)
+
+ def getLocalePoller(self, locale, branch):
+ if (locale, branch) not in self.localePollers:
+ lp = HgLocalePoller(locale, self, branch,
+ self.hgURL)
+ lp.verbose = self.verboseChilds
+ self.localePollers[(locale, branch)] = lp
+ return self.localePollers[(locale, branch)]
def processData(self, data):
- for l in data.splitlines():
- l = l.strip()
- if l == '': continue
- self.getLocalePoller(l).poll()
+ locales = filter(None, data.split())
+ # get locales and branches
+ def brancher(link):
+ steps = filter(None, link.split('/'))
+ loc = steps.pop()
+ branch = '/'.join(steps)
+ return (loc, branch)
+ # locales is now locale code / branch tuple
+ locales = map(brancher, locales)
+ if locales != self.locales:
+ log.msg("new locale list: " + " ".join(map(str, locales)))
+ self.locales = locales
+ self.pendingLocales = locales[:]
+ # prune removed locales from pollers
+ for oldLoc in self.localePollers.keys():
+ if oldLoc not in locales:
+ self.localePollers.pop(oldLoc)
+ log.msg("not polling %s on %s anymore, dropped from repositories" %
+ oldLoc)
+ for i in xrange(self.parallelRequests):
+ self.activeRequests += 1
+ reactor.callLater(0, self.pollNextLocale)
+
+ def pollNextLocale(self):
+ if not self.pendingLocales:
+ self.activeRequests -= 1
+ if not self.activeRequests:
+ msg = "%s done with all locales" % str(self)
+ loadTimes = map(lambda p: p.loadTime, self.localePollers.values())
+ goodTimes = filter(lambda t: t is not None, loadTimes)
+ if not goodTimes:
+ msg += ". All %d locale pollers failed" % len(loadTimes)
+ else:
+ msg += ", min: %.1f, max: %.1f, mean: %.1f" % \
+ (min(goodTimes), max(goodTimes),
+ sum(goodTimes) / len(goodTimes))
+ if len(loadTimes) > len(goodTimes):
+ msg += ", %d failed" % (len(loadTimes) - len(goodTimes))
+ log.msg(msg)
+ log.msg("Total time: %.1f" % (time.time() - self.startLoad))
+ return
+ loc, branch = self.pendingLocales.pop(0)
+ poller = self.getLocalePoller(loc, branch)
+ poller.poll()
+
+ def localeDone(self, loc):
+ if self.verboseChilds:
+ log.msg("done with " + loc)
+ reactor.callLater(0, self.pollNextLocale)
def __str__(self):
- return "<HgAllLocalesPoller for %s>" % self.allLocalesURL
+ return "<HgAllLocalesPoller for %s%s/>" % (self.hgURL,
+ self.repositoryIndex)
View
968 l10n.py
@@ -0,0 +1,968 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla-specific Buildbot steps.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2007
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+# Armen Zambrano Gasparnian <armenzg@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+from twisted.python import log as log2
+from buildbotcustom import log
+from buildbot.scheduler import BaseUpstreamScheduler, Nightly, Dependent
+from buildbot.sourcestamp import SourceStamp
+from buildbot import buildset, process
+from buildbot.changes import changes
+from buildbot.process import properties
+from twisted.internet import protocol, utils, reactor, error, defer
+from twisted.web.client import HTTPClientFactory, getPage
+from itertools import izip
+from ConfigParser import ConfigParser, NoSectionError, NoOptionError
+from cStringIO import StringIO
+import os
+import os.path
+import re
+import shlex
+from urlparse import urljoin, urlparse
+from urllib import pathname2url, url2pathname
+import subprocess
+
+
+# NOTE: The following class has been copied from compare-locales
+# keep in sync and remove once compare-locales is on the master
+class L10nConfigParser(object):
+ '''Helper class to gather application information from ini files.
+
+ This class is working on synchronous open to read files or web data.
+ Subclass this and overwrite loadConfigs and addChild if you need async.
+ '''
+ def __init__(self, inipath, **kwargs):
+ """Constructor for L10nConfigParsers
+
+ inipath -- l10n.ini path
+ Optional keyword arguments are fowarded to the inner ConfigParser as
+ defaults.
+ """
+ if os.path.isabs(inipath):
+ self.inipath = 'file:%s' % pathname2url(inipath)
+ else:
+ pwdurl = 'file:%s/' % pathname2url(os.getcwd())
+ self.inipath = urljoin(pwdurl, inipath)
+ # l10n.ini files can import other l10n.ini files, store the
+ # corresponding L10nConfigParsers
+ self.children = []
+ # we really only care about the l10n directories described in l10n.ini
+ self.dirs = []
+ # optional defaults to be passed to the inner ConfigParser (unused?)
+ self.defaults = kwargs
+
+ def loadConfigs(self):
+ """Entry point to load the l10n.ini file this Parser refers to.
+
+ This implementation uses synchronous loads, subclasses might overload
+ this behaviour. If you do, make sure to pass a file-like object
+ to onLoadConfig.
+ """
+ self.onLoadConfig(urlopen(self.inipath))
+
+ def onLoadConfig(self, inifile):
+ """Parse a file-like object for the loaded l10n.ini file."""
+ cp = ConfigParser(self.defaults)
+ cp.readfp(inifile)
+ try:
+ depth = cp.get('general', 'depth')
+ except:
+ depth = '.'
+ self.baseurl = urljoin(self.inipath, depth)
+ # create child loaders for any other l10n.ini files to be included
+ try:
+ for title, path in cp.items('includes'):
+ # skip default items
+ if title in self.defaults:
+ continue
+ # add child config parser
+ self.addChild(title, path, cp)
+ except NoSectionError:
+ pass
+ # try to load the "dirs" defined in the "compare" section
+ try:
+ self.dirs.extend(cp.get('compare', 'dirs').split())
+ except (NoOptionError, NoSectionError):
+ pass
+ # try to set "all_path" and "all_url"
+ try:
+ self.all_path = cp.get('general', 'all')
+ self.all_url = urljoin(self.inipath, 'all-locales')
+ except (NoOptionError, NoSectionError):
+ self.all_path = None
+ self.all_url = None
+
+ def addChild(self, title, path, orig_cp):
+ """Create a child L10nConfigParser and load it.
+
+ title -- indicates the module's name
+ path -- indicates the path to the module's l10n.ini file
+ orig_cp -- the configuration parser of this l10n.ini
+ """
+ cp = L10nConfigParser(urljoin(self.baseurl, path), **self.defaults)
+ cp.loadConfigs()
+ self.children.append(cp)
+
+ def dirsIter(self):
+ """Iterate over all dirs and our base path for this l10n.ini"""
+ url = urlparse(self.baseurl)
+ basepath = None
+ if url[0] == 'file':
+ basepath = url2pathname(url[2])
+ for dir in self.dirs:
+ yield (dir, basepath)
+
+ def directories(self):
+ """Iterate over all dirs and base paths for this l10n.ini as well
+ as the included ones.
+ """
+ for t in self.dirsIter():
+ yield t
+ for child in self.children:
+ for t in child.directories():
+ yield t
+
+ def allLocales(self):
+ """Return a list of all the locales of this project"""
+ return urlopen(self.all_url).read().splitlines()
+
+class AsyncLoader(L10nConfigParser):
+ """AsyncLoader extends L10nConfigParser to do IO with asynchronous twisted
+ loads.
+ """
+ timeout = 30
+
+ def __init__(self, inipath, branch, _type='hg'):
+ L10nConfigParser.__init__(self, inipath)
+ self.branch = branch
+ self.pendingLoads = 0
+ self.type = _type
+
+ def loadConfigs(self):
+ """Load the l10n.ini file asynchronously.
+
+ Parses the contents in the callback, and fires off further
+ loads as needed for included l10n.inis for other modules.
+ """
+ self.d = defer.Deferred()
+ self.pendingLoads += 1
+ d = self._getPage(self.inipath)
+ def _cbDone(contents):
+ """Callback passing the contents of the l10n.ini file to onLoadConfig
+ """
+ self.onLoadConfig(StringIO(contents))
+ return defer.succeed(True)
+ def _cbErr(rv):
+ """Errback when loading of l10n.ini fails.
+
+ Decrements the pending loads and forwards the error.
+ """
+ self.pendingLoads -= 1
+ return self.d.errback(rv)
+ d.addCallback(_cbDone)
+ d.addCallbacks(self._loadDone, _cbErr)
+ return self.d
+
+ def onLoadConfig(self, f):
+ """Overloaded method of L10nConfigParser
+
+ Just adds debug information."""
+ log2.msg("onLoadConfig called")
+ L10nConfigParser.onLoadConfig(self,f)
+
+ def addChild(self, title, path, orig_cp):
+ """Create a child L10nConfigParser and load it.
+
+ title -- indicates the module's name
+ path -- indicates the path to the module's l10n.ini file
+ orig_cp -- the configuration parser this l10n.ini
+
+ Extends L10nConfigParser.addChild to keep track of all
+ asynchronous loads
+ """
+
+ # check if there's a section with details for this include
+ # we might have to check a different repo, or even VCS
+ # for example, projects like "mail" indicate in
+ # an "include_" section where to find the l10n.ini for "toolkit"
+ details = 'include_' + title
+ if orig_cp.has_section(details):
+ type = orig_cp.get(details, 'type')
+ if type not in ['hg', 'cvs']:
+ log2.msg("Cannot load l10n.ini for %s with type %s" % (title, type))
+ return
+ branch = orig_cp.get(details, 'mozilla')
+ # Create an asynchronous loader depending if "hg" or "cvs" module
+ if type == 'cvs':
+ cp = CVSAsyncLoader(orig_cp.get(details, 'l10n.ini'), branch)
+ else:
+ l10n_ini_temp = '%(repo)s%(mozilla)s/raw-file/tip/%(l10n.ini)s'
+ cp = AsyncLoader(l10n_ini_temp % dict(orig_cp.items(details)), branch,
+ _type=type)
+ else:
+ # instantiates the same class as the current one
+ # it can be AsyncLoader or CVSAsyncLoader
+ cp = self.__class__(urljoin(self.baseurl, path), self.branch,
+ **self.defaults)
+ d = cp.loadConfigs()
+ self.pendingLoads += 1
+ d.addCallbacks(self._loadDone, self.d.errback)
+ self.children.append(cp)
+
+ def getAllLocales(self):
+ return self._getPage(self.all_url)
+
+ def _getPage(self, path):
+ return getPage(path, timeout = self.timeout)
+
+ def _loadDone(self, result):
+ self.pendingLoads -= 1
+ if not self.pendingLoads:
+ self.d.callback(True)
+
+class CVSProtocol(protocol.ProcessProtocol):
+
+ def __init__(self, cmd):
+ self.d = defer.Deferred()
+ self.data = ''
+ self.errdata = ''
+ self.cmd = cmd
+
+ def connectionMade(self):
+ self.transport.closeStdin()
+
+ def outReceived(self, data):
+ self.data += data
+
+ def errReceived(self, data):
+ # chew away what cvs blurbs at us
+ self.errdata += data
+ pass
+
+ def processEnded(self, reason):
+ if isinstance(reason.value, error.ProcessDone):
+ self.d.callback(self.data)
+ else:
+ reason.value.args = (self.errdata,)
+ self.d.errback(reason)
+
+class CVSAsyncLoader(AsyncLoader):
+ """CVSAsyncLoader subclasses AsyncLoader to get data via
+ cvs co -p
+ """
+ CVSROOT = ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot'
+
+ def __init__(self, inipath, branch, **kwargs):
+ AsyncLoader.__init__(self, inipath, branch, **kwargs)
+ self.inipath = inipath
+ self.type = 'cvs'
+
+ def _getPage(self, path):
+ args = ['cvs', '-d', self.CVSROOT, 'co']
+ if self.branch is not None:
+ args += ['-r', self.branch]
+ args += ['-p', path]
+ pp = CVSProtocol(' '.join(args))
+ reactor.spawnProcess(pp, 'cvs', args, {})
+ return pp.d
+
+
+class repositories(object):
+ """
+ Helper to store some information about the cvs repositories we use.
+
+ It provides the cvsroot and the bonsai url. For l10n purposes, there
+ are two functions, expand, and expandLists, which take the locale
+ and the module, or lists for both, resp., and returns the list of
+ directories, suitable both for cvs check out and bonsai.
+
+ Predefined are the repositories mozilla and l10n.
+ """
+ class _repe:
+ def __init__(self, root, base, bonsai):
+ self.cvsroot = root
+ self.base = base
+ self.bonsai = bonsai
+ self.expand = lambda loc, mod: 'mozilla/%s/locales/'%mod
+ self.repository = 'l10nbld@cvs.mozilla.org:/cvsroot'
+ def expandLists(self, locs, list):
+ return [self.expand(l, m) for m in list for l in locs]
+ mozilla = _repe('/cvsroot', 'mozilla', 'http://bonsai.mozilla.org/')
+ l10n = _repe('/l10n', 'l10n', 'http://bonsai-l10n.mozilla.org/')
+ l10n.expand = lambda loc, mod: 'l10n/%s/%s/'%(loc,mod)
+ l10n.repository = 'l10nbld@cvs.mozilla.org:/l10n'
+
+
+def configureDispatcher(config, section, scheduler):
+ """
+ Add the Dispatchers for the given section of l10nbuilds.ini to the scheduler.
+
+ section -- the name of the section inside of l10nbuilds.ini, aka the tree
+ config -- the config parser that loaded l10nbuilds.ini
+ scheduler -- scheduler to which we are adding dispatchers
+
+ Types of dispatchers:
+ - AllLocalesWatcher:
+ every time all-locales for the app changes, the list of locales is updated
+ - EnDispatcher:
+ checks for en-US changes and triggers compareOnly builds for all locales
+ more than one can be given for a given tree to check multiple repos
+ - [Hg]L10nDispatcher:
+ checks for changes in a locale and triggers a full build for that locale
+ """
+
+ log2.msg('configureDispatcher for ' + section)
+ buildtype = config.get(section, 'type')
+ if buildtype not in ['cvs', 'hg', 'single-module-hg']:
+ raise RuntimeError('type needs to be either cvs, hg, or single-module-hg')
+ en_branch = config.get(section, 'mozilla')
+ l10n_branch = config.get(section, 'l10n')
+ props = {'en_branch': en_branch,
+ 'l10n_branch': l10n_branch}
+ if config.has_option(section, 'en_us_binary'):
+ props['en_us_binary'] = config.get(section, 'en_us_binary')
+ if buildtype == 'cvs':
+ CVSAsyncLoader.CVSROOT = repositories.mozilla.repository
+ cp = CVSAsyncLoader(config.get(section, 'l10n.ini'), en_branch)
+ else:
+ l10n_ini_temp = '%(repo)s%(mozilla)s/raw-file/tip/%(l10n.ini)s'
+ # substitute "repo", "mozilla" and "l10n.ini" from the l10nbuilds.ini file
+ cp = AsyncLoader(l10n_ini_temp % dict(config.items(section)), en_branch,
+ _type = buildtype)
+
+ # Declaring few functions that are going to be used as callback functions
+ # immediatelly after being declared
+ # 1) _addDispatchers(dirs, builders)
+ # 2) _cbLoadedConfig(rv)
+ # 3) _errBack(msg)
+ def _addDispatchers(dirs, builders):
+ """
+ This function is called at the end of _cbLoadedConfig's execution
+ Let's add the EnDispatcher(s) and the (Hg)L10nDispatcher for this project/branch
+ """
+ alldirs = []
+ if dirs['cvs']:
+ # Let's add an EnDispatcher to watch for en-US entities' changes
+ # We can have more than one EnDispatcher
+ for branch, endirs in dirs['cvs'].iteritems():
+ scheduler.addDispatcher(EnDispatcher(endirs, branch, section,
+ prefix = 'mozilla/'))
+ alldirs += endirs
+ log2.msg('en cvs dispatchers added for ' + section)
+ if dirs['hg'] or dirs['single-module-hg']:
+ revisions = ['en', 'l10n']
+ if config.has_option(section, 'revisions'):
+ # we're given a list of foo_revision properties to set to 'default'
+ revisions = map(None, config.get(section, 'revisions').split())
+ props.update(dict.fromkeys((s+'_revision' for s in revisions), 'default'))
+ for branch, endirs in dirs['hg'].iteritems():
+ log2.msg('adding EnDispatcher for %s on %s for %s' %
+ (section, branch, ' '.join(endirs)))
+ scheduler.addDispatcher(EnDispatcher(endirs, branch, section))
+ alldirs += endirs
+ for branch, endirs in dirs['single-module-hg'].iteritems():
+ log2.msg('adding EnDispatcher for single module %s on %s for %s' %
+ (section, branch, ' '.join(endirs)))
+ if len(endirs) > 1:
+ log2.msg('WARNING: More than one dir for a single module?')
+ # the EnDispatcher for a single module just listens to
+ # 'locales/en-US/foo', i.e., has only a single empty dir
+ scheduler.addDispatcher(EnDispatcher([''], branch, section))
+ alldirs += endirs
+
+
+ # if we have one hg dispatcher, l10n is on hg
+ scheduler.addDispatcher(HgL10nDispatcher(alldirs, l10n_branch,
+ section))
+ log2.msg('both hg dispatchers added for ' + section)
+ else:
+ # only pure cvs projects have l10n on cvs
+ scheduler.addDispatcher(L10nDispatcher(alldirs, l10n_branch,
+ section))
+ log2.msg('l10n cvs dispatchers added for ' + section)
+
+ scheduler.treeprops[section] = props
+
+ # This adds in the builder's column a "%(section)s set up" message to
+ # indicate that all loading has been complete for that section
+ buildermap = scheduler.parent.botmaster.builders
+ for b in builders:
+ try:
+ buildermap[b].builder_status.addPointEvent([section, "set", "up"])
+ except KeyError:
+ log2.msg("Can't find builder %s for %s" % (b, section))
+ def _cbLoadedConfig(rv):
+ # All the l10n.ini required files have been loaded asynchronously and
+ # we are ready to add the required dispatchers
+ log2.msg('config loaded for ' + section)
+ dirs = {'hg':{}, 'cvs':{}, 'single-module-hg':{}}
+ # Let's iterate through the ConfigParser (aka loader)
+ # to get all the directories
+ # l.type indicates if the module lives in "cvs" or "hg"
+ loaders = [cp]
+ while loaders:
+ l = loaders.pop(0)
+ ldirs = dict(l.dirsIter()).keys()
+ if l.branch not in dirs[l.type]:
+ dirs[l.type][l.branch] = ldirs
+ else:
+ dirs[l.type][l.branch] += ldirs
+ # Let's append at the end any children (which are loaders)
+ # that the current loader has
+ loaders += l.children
+ # Let's sort the keys for debugging purposes
+ for d in dirs.itervalues():
+ for dd in d.itervalues():
+ dd.sort()
+
+ builders = shlex.split(config.get(section, 'builders'))
+ scheduler.builders[section] = builders
+ scheduler.apps[section] = config.get(section, 'app')
+
+ locales = config.get(section, 'locales')
+ if locales == 'all':
+ # add AllLocalesWatcher, cvs and hg have different paths, i.e.,
+ # cvs has a leading 'mozilla/'
+ if cp.type == 'cvs':
+ path = cp.all_url
+ else:
+ # e.g. browser/locales/all-locales
+ path = cp.all_path
+ scheduler.addDispatcher(AllLocalesWatcher(en_branch,
+ path,
+ cp,
+ section))
+ else:
+ # Just use the given list of locales, picks up changes on reconfig
+ scheduler.locales[section] = locales.split()
+ # Let's add the rest of the dispatchers
+ _addDispatchers(dirs, builders)
+ return
+ def _errBack(msg):
+ log2.msg("loading %s failed with %s" % (section, msg.value.message))
+ log2.msg(section + " has inipath " + cp.inipath)
+ buildermap = scheduler.parent.botmaster.builders
+ # for b in builders:
+ # buildermap[b].builder_status.addPointEvent([section, "setup", "failed"])
+ d = cp.loadConfigs()
+ d.addCallbacks(_cbLoadedConfig, _errBack)
+ return d
+
+"""
+Dispatchers
+
+The dispatchers know about which changes impact which localizations and
+applications. They're used in the Scheduler, and enable the scheduler to
+work on multiple branch combinations and multiple applications.
+"""
+class IBaseDispatcher(object):
+ """
+ Interface for dispatchers.
+ """
+ log = "dispatcher"
+
+ def dispatchChange(self, change):
+ """
+ Scheduler calls dispatch for each change, the dispatcher is expected
+ to call sink.queueBuild for each build that is supposed to run.
+ Scheduler will coalescent duplicate builders.
+ """
+ raise NotImplented
+
+ def setParent(self, parent):
+ self.parent = parent
+
+ def debug(self, msg):
+ log.debug(self.log, msg)
+
+class AllLocalesWatcher(IBaseDispatcher):
+ """
+ Dispatcher to watch for changes for changes to all-locales files.
+
+ If such a change comes in, it reloads the all-locales file, and
+ updates the scheduler's locales map.
+ """
+
+ def __init__(self, branch, all_urls_path, loader, tree):
+ self.branch = branch
+ self.path = all_urls_path
+ self.loader = loader
+ self.tree = tree
+ self.log += '.'+ tree
+
+ def setParent(self, parent):
+ IBaseDispatcher.setParent(self, parent)
+ # do the initial loading of all-locales
+ d = self.loader.getAllLocales()
+ d.addCallbacks(self.callback, self.errback)
+
+
+ def dispatchChange(self, change):
+ self.debug("changed %s for %s?" % (self.path, self.tree))
+ # Let's verify that the change is for our branch
+ if change.branch != self.branch:
+ return
+ if self.path not in change.files:
+ return
+ # Let's get the contents of all-locales in the repo
+ self.debug("update all-locales for %s" % self.tree)
+ d = self.loader.getAllLocales()
+ d.addCallbacks(self.callback, self.errback)
+
+ def callback(self, locales):
+ """Callback for content of all-locales.
+
+ split() the content and update the scheduler's locales map
+ """
+ self.debug("all-locales loaded for %s" % self.tree)
+ locales = locales.split()
+ self.parent.locales[self.tree] = locales
+
+ def errback(self, failure):
+ self.debug("loading all locales for %s failed with %s" % (self.tree,
+ failure.value.message))
+
+
+class L10nDispatcher(IBaseDispatcher):
+ """
+ Dispatcher taking care about one branch in the l10n rep.
+
+ It's using
+ - an array of module-app tuples and
+ - a hash mapping apps to locales
+ to figure out which apps-locale combos need to be built. It ignores
+ changes that are not affecting its combos.
+ It can pass around a tree name, too.
+ """
+
+ def __init__(self, paths, l10n_branch, tree,
+ props = {}):
+ self.paths = paths
+ self.l10n_branch = l10n_branch
+ self.tree = tree
+ self.parent = None
+ self.log += '.l10n'
+ self.log += '.' + tree
+ self.props = props
+
+ def dispatchChange(self, change):
+ self.debug("adding change %d" % change.number)
+ toBuild = {}
+ if self.l10n_branch and self.l10n_branch != change.branch:
+ self.debug("not our branch, ignore, %s != %s" %
+ (self.l10n_branch, change.branch))
+ return
+ if self.tree not in self.parent.locales:
+ # we're probably still waiting for all-locales to load, ignore
+ # this change and go on
+ self.debug("locales list for %s not set up, ignoring change" % self.tree)
+ return
+ for file in change.files:
+ pathparts = file.split('/',2)
+ if pathparts[0] != 'l10n':
+ self.debug("non-l10n changeset ignored")
+ return
+ if len(pathparts) != 3:
+ self.debug("l10n path doesn't contain locale and path")
+ return
+ loc, path = pathparts[1:]
+ if loc not in self.parent.locales[self.tree]:
+ continue
+ for basepath in self.paths:
+ if not path.startswith(basepath):
+ continue
+ toBuild[loc] = True
+ self.debug("adding %s for %s" % (self.tree, loc))
+ for loc in toBuild.iterkeys():
+ self.parent.queueBuild(loc, change, self.tree, self.props)
+
+class HgL10nDispatcher(L10nDispatcher):
+ def dispatchChange(self, change):
+ self.debug("adding change %d" % change.number)
+ if self.l10n_branch and self.l10n_branch != change.branch:
+ self.debug("not our branch, ignore, %s != %s" %
+ (self.l10n_branch, change.branch))
+ return
+ if not hasattr(change, 'locale'):
+ log2.msg("I'm confused, the branches match, but this is not a locale change")
+ return
+ if self.tree not in self.parent.locales:
+ # we're probably still waiting for all-locales to load, ignore
+ # this change and go on
+ self.debug("locales list for %s not set up, ignoring change" % self.tree)
+ return
+ if change.locale not in self.parent.locales[self.tree]:
+ # this is a change to a locale that this build doesn't pay attention to
+ return
+ # Only changes in the modules mentioned in self.paths should trigger a build
+ doBuild = False
+ for file in change.files:
+ for basepath in self.paths:
+ if file.startswith(basepath):
+ doBuild = True
+ break
+ if not doBuild:
+ self.debug("dropping change %d, not our app" % change.number)
+ self.debug("%s listens to %s" % (self.tree, ' '.join(self.paths)))
+ return
+ self.parent.queueBuild(change.locale, change, self.tree, self.props)
+
+class EnDispatcher(IBaseDispatcher):
+ """
+ Dispatcher watching one branch on the main mozilla repository.
+ It's using
+ - an array of module-app tuples and
+ - a hash mapping apps to locales
+ to figure out which apps-locale combos need to be built. For each application
+ affected by a change, it queues a build for all locales for that app.
+ It can pass around a tree name, too.
+ """
+
+ def __init__(self, paths, branch, tree, prefix = '',
+ props = {}):
+ self.paths = paths
+ self.branch = branch
+ self.tree = tree
+ self.parent = None
+ self.log += '.en'
+ self.log += '.' + tree
+ self.prefix = prefix
+ self.props = dict(props)
+ self.props['compareOnly'] = True
+ self.en_pattern = re.compile('(?P<module>.*?)/?locales/en-US/')
+
+ def setParent(self, parent):
+ self.parent = parent
+
+ def dispatchChange(self, change):
+ self.debug("adding change %d" % change.number)
+ if self.branch and self.branch != change.branch:
+ self.debug("not our branch, ignore, %s != %s" %
+ (self.branch, change.branch))
+ return
+ if self.tree not in self.parent.locales:
+ self.debug("parent locales not set up, ignoring change")
+ return
+ needsBuild = False
+ for file in change.files:
+ if not file.startswith(self.prefix):
+ self.debug("Ignoring change %d, not our rep" % change.number)
+ return
+ file = file.replace(self.prefix, '', 1)
+ m = self.en_pattern.match(file)
+ if not m:
+ continue
+ basedir = m.group('module')
+ if basedir in self.paths:
+ needsBuild = True
+ break
+ if needsBuild:
+ for loc in self.parent.locales[self.tree]:
+ self.parent.queueBuild(loc, change, self.tree, self.props)
+
+
+class Scheduler(BaseUpstreamScheduler):
+ """
+ Scheduler used for l10n builds.
+
+ It's using several Dispatchers to create build items depending
+ on the submitted changes.
+ """
+
+ compare_attrs = ('name', 'treeStableTimer', 'inipath',
+ 'builders', 'apps', 'locales', 'treeprops',
+ 'properties')
+
+ def __init__(self, name, inipath, treeStableTimer = None):
+ """
+ @param name: the name of this Scheduler
+ @param treeStableTimer: the duration, in seconds, for which the tree
+ must remain unchanged before a build will be
+ triggered. This is intended to avoid builds
+ of partially-committed fixes.
+ """
+
+ BaseUpstreamScheduler.__init__(self, name)
+ # path to the l10nbuilds.ini file that is read synchronously
+ self.inipath = inipath
+ self.treeStableTimer = treeStableTimer
+ self.nextBuildTime = None
+ self.timer = None
+
+ # will hold the dispatchers for each tree
+ self.dispatchers = []
+ # list of locales for each tree
+ self.locales = {}
+ # list of builders for each tree
+ self.builders = {}
+ # app per tree
+ self.apps = {}
+ # properties per tree
+ self.treeprops = {}
+
+ def startService(self):
+ log2.msg("starting l10n scheduler")
+ cp = ConfigParser()
+ cp.read(self.inipath)
+ # Configure the dispatchers for our trees as soon as the reactor is running
+ for tree in cp.sections():
+ reactor.callWhenRunning(configureDispatcher,
+ cp, tree, self)
+
+ class NoMergeStamp(SourceStamp):
+ """
+ We're going to submit a bunch of build requests for each change. That's
+ how l10n goes. This source stamp impl keeps them from being merged by
+ the build master.
+ """
+ def canBeMergedWith(self, other):
+ return False
+
+ # dispatching routines
+ def addDispatcher(self, dispatcher):
+ """
+ Add an IBaseDispatcher instance to this Scheduler.
+ """
+ self.dispatchers.append(dispatcher)
+ dispatcher.setParent(self)
+
+ def queueBuild(self, locale, change_or_changes, tree, misc_props={}):
+ """
+ Callback function for dispatchers to tell us what to build.
+ This function actually submits the buildsets on non-mergable
+ sourcestamps.
+ """
+ if isinstance(change_or_changes, changes.Change):
+ _changes = [change_or_changes]
+ else:
+ _changes = change_or_changes
+ log.debug("scheduler", "queueBuild: build %s for change %d" %
+ (', '.join(self.builders[tree]), _changes[0].number))
+ props = properties.Properties()
+ props.updateFromProperties(self.properties)
+ if tree in self.treeprops:
+ props.update(self.treeprops[tree], 'Scheduler')
+ props.update(dict(app=self.apps[tree], locale=locale, tree=tree,
+ needsCheckout = True), 'Scheduler')
+ if misc_props:
+ props.update(misc_props, 'Scheduler')
+ bs = buildset.BuildSet(self.builders[tree],
+ Scheduler.NoMergeStamp(changes=_changes),
+ reason = "%s %s" % (tree, locale),
+ properties = props)
+ self.submitBuildSet(bs)
+
+ # Implement IScheduler
+ def addChange(self, change):
+ log.debug("scheduler",
+ "addChange: Change %d, %s" % (change.number, change.asText()))
+ for dispatcher in self.dispatchers:
+ dispatcher.dispatchChange(change)
+
+ def listBuilderNames(self):
+ builders = set()
+ for bs in self.builders.itervalues():
+ builders.update(bs)
+ return list(builders)
+
+ def getPendingBuildTimes(self):
+ if self.nextBuildTime is not None:
+ return [self.nextBuildTime]
+ return []
+
+
+class L10nMixin(object):
+ """
+ This class helps any of the L10n custom made schedulers
+ to submit BuildSets as specified per list of locales, or either a
+ 'all-locales' or 'shipped-locales' file via a call to createL10nBuilds.
+
+ For each locale, there will be a build property 'locale' set to the
+ inidividual locale to be built for that BuildSet.
+ """
+
+ def __init__(self,
+ repo = 'http://hg.mozilla.org/',
+ branch = None,
+ repoType = 'cvs',
+ baseTag = 'default',
+ localesFile = None,
+ cvsRoot = ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot',
+ locales = None):
+ self.repoType = repoType
+ self.baseTag = baseTag
+ self.cvsRoot = cvsRoot
+ # set a default localesURL accordingly to the repoType if none has been set
+ if repoType.find('hg') >= 0:
+ if not localesFile:
+ localesFile = "browser/locales/all-locales"
+ self.localesURL = "%s%s/raw-file/%s/%s" \
+ % (repo, branch, baseTag, localesFile)
+ elif repoType.find('cvs') >= 0:
+ if not localesFile:
+ self.localesURL = "mozilla/browser/locales/all-locales"
+ else:
+ self.localesURL = localesFile
+ # if the user wants to use something different than all locales
+ if locales:
+ self.locales = locales[:]
+ else:
+ self.locales = None
+
+ class NoMergeStamp(SourceStamp):
+ """
+ This source stamp implementation keeps them from being merged by
+ the build master.
+ """
+ def canBeMergedWith(self, other):
+ return False
+
+ def _cbLoadedLocales(self, locales):
+ """
+ This is the callback function that gets called once the list
+ of locales are ready to be processed
+ Let's fill the queues per builder and submit the BuildSets per each locale
+ """
+ log2.msg("L10nMixin:: loaded locales' list")
+ for locale in locales:
+ if locale == "en-US":
+ continue
+ props = properties.Properties()
+ props.updateFromProperties(self.properties)
+ #I do not know exactly what to pass as the source parameter
+ props.update(dict(locale=locale),"Scheduler")
+ props.setProperty("en_revision",self.baseTag,"Scheduler")
+ props.setProperty("l10n_revision",self.baseTag,"Scheduler")
+ log2.msg('Submitted '+locale+' locale')
+ # let's submit the BuildSet for this locale
+ self.submitBuildSet(
+ buildset.BuildSet(self.builderNames,
+ self.NoMergeStamp(branch=self.branch),
+ self.reason,
+ properties = props))
+
+ def getLocales(self):
+ """
+ It returns a list of locales if the user has set a list of locales
+ in the scheduler OR it returns a Deferred.
+
+ You want to call this method via defer.maybeDeferred().
+ """
+ if self.locales:
+ log2.msg('L10nMixin.getLocales():: The user has set a list of locales')
+ return self.locales
+ else:
+ log2.msg("L10nMixin:: Getting locales from: "+self.localesURL)
+ # we expect that getPage will return the output of "all-locales"
+ # or "shipped-locales" or any file that contains a locale per line
+ # in the begining of the line e.g. "en-GB" or "ja linux win32"
+ # this code will only care about the first argument appearing in the line
+ if self.repoType == 'cvs':
+ args = ['cvs', '-q', '-d', self.cvsRoot, 'co', '-p', self.localesURL]
+ # communicate() returns a tuple - stdout, stderr
+ # the output of cvs has a '\n' element at the end
+ # a last '' string is generated that we get rid of
+ return (lambda lines: [line.split(' ')[0] for line in lines]) \
+ (subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0].split('\n')[0:-1])
+ else: # the repoType is 'hg'
+ # getPage returns a defered that will return a string
+ d = getPage(self.localesURL, timeout = 5 * 60)
+ d.addCallback(lambda data: [line.split(' ')[0]
+ for line in data.split("\n") if line])
+ return d
+
+ def createL10nBuilds(self):
+ """
+ We request to get the locales that we have to process and which
+ method to call once they are ready
+ """
+ log2.msg('L10nMixin:: A list of locales is going to be requested')
+ d = defer.maybeDeferred(self.getLocales)
+ d.addCallback(self._cbLoadedLocales)
+ return d
+
+class NightlyL10n(Nightly, L10nMixin):
+ """
+ NightlyL10n is used to paralellize the generation of l10n builds.
+
+ NightlyL10n is designed to be used with a Build factory that gets the
+ locale to build from the 'locale' build property.
+ """
+
+ compare_attrs = ('name', 'builderNames',
+ 'minute', 'hour', 'dayOfMonth', 'month',
+ 'dayOfWeek', 'branch')
+
+ def __init__(self, name, builderNames, repoType, minute=0, hour='*', dayOfMonth='*', month='*', dayOfWeek='*',
+ repo = 'http://hg.mozilla.org/', branch=None, baseTag='default', localesFile=None,
+ cvsRoot=None, locales=None):
+
+ Nightly.__init__(self, name, builderNames, minute, hour, dayOfMonth, month, dayOfWeek, branch,
+ properties={'nightly': True})
+ L10nMixin.__init__(self, repoType = repoType, repo = repo, branch = branch,
+ baseTag = baseTag, localesFile = localesFile,
+ cvsRoot = cvsRoot, locales = locales)
+
+ def doPeriodicBuild(self):
+ # Schedule the next run (as in Nightly's doPeriodicBuild)
+ self.setTimer()
+ self.createL10nBuilds()
+
+class DependentL10n(Dependent, L10nMixin):
+ """
+ This scheduler runs some set of 'downstream' builds when the
+ 'upstream' scheduler has completed successfully.
+ """
+
+ compare_attrs = ('name', 'upstream', 'builders')
+
+ def __init__(self, name, upstream, builderNames,
+ repoType, repo = 'http://hg.mozilla.org/', branch=None,
+ baseTag='default', localesFile=None,
+ cvsRoot=None, locales=None):
+ Dependent.__init__(self, name, upstream, builderNames)
+ # The next two lines has been added because of:
+ # _cbLoadedLocales's BuildSet submit needs them
+ self.branch = None
+ self.reason = None
+ L10nMixin.__init__(self, repoType = repoType, branch = branch,
+ baseTag = baseTag, localesFile = localesFile,
+ cvsRoot = cvsRoot, locales = locales)
+
+ # ss is the source stamp that we don't use currently
+ def upstreamBuilt(self, ss):
+ self.createL10nBuilds()
View
0 l10n/__init__.py
No changes.
View
226 l10n/scheduler.py
@@ -1,226 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
-#
-# The Initial Developer of the Original Code is
-# Mozilla Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2007
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Axel Hecht <axel@mozilla.com>
-# Armen Zambrano Gasparnian <armenzg@mozilla.com>
-# ***** END LICENSE BLOCK *****
-
-from twisted.python import log
-from twisted.web.client import getPage
-from twisted.internet import defer, reactor
-from buildbot.steps.shell import ShellCommand
-from buildbot.scheduler import Nightly, Periodic, Dependent
-from buildbot.sourcestamp import SourceStamp
-from buildbot.process import properties
-from buildbot import buildset
-import subprocess
-
-class L10nMixin(object):
- """
- This class helps any of the L10n custom made schedulers
- to generate build objects as specified per list of locales
- """
-
- def __init__(self, scheduler,
- repoPath = None,
- repoType = 'cvs',
- baseTag = 'default',
- localesFile = None,
- cvsRoot = ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot',
- locales = None):
- self.scheduler = scheduler
- self.repoType = repoType
- self.baseTag = baseTag
- self.cvsRoot = cvsRoot
- # set a default localesURL accordingly to the repoType if none has been set
- if repoType.find('hg') >= 0:
- if not localesFile:
- localesFile = "browser/locales/all-locales"
- self.localesURL = "http://hg.mozilla.org/%s/raw-file/%s/%s" \
- % (repoPath, baseTag, localesFile)
- elif repoType.find('cvs') >= 0:
- if not localesFile:
- self.localesURL = "mozilla/browser/locales/all-locales"
- else:
- self.localesURL = localesFile
-
- # we are going to have a queue per builder attached to the scheduler
- self.queue = {}
- for builderName in scheduler.builderNames:
- self.queue[builderName] = []
- # if the user wants to use something different than all locales
- if locales:
- self.locales = locales[:]
- else:
- self.locales = None
-
- class NoMergeStamp(SourceStamp):
- """
- This source stamp implementation keeps them from being merged by
- the build master.
- """
- def canBeMergedWith(self, other):
- return False
-
- class BuildDesc(object):
- """
- All it does is to associate the self.locale property
- """
- def __init__(self, locale):
- self.locale = locale
-
- def __eq__(self, other):
- return self.locale == other.locale
-
- def __repr__(self):
- return "Build: %s" % (self.locale)
-
- def _cbLoadedLocales(self, locales):
- """
- This is the callback function that gets called once the list
- of locales are ready to be processed
- Let's fill the queues per builder and submit the BuildSets per each locale
- """
- log.msg("L10nMixin:: loaded locales' list")
- for locale in locales:
- if locale == "en-US":
- continue
- props = properties.Properties()
- props.updateFromProperties(self.scheduler.properties)
- #I do not know exactly what to pass as the source parameter
- props.update(dict(locale=locale),"Scheduler")
- props.setProperty("en_revision",self.baseTag,"Scheduler")
- props.setProperty("l10n_revision",self.baseTag,"Scheduler")
- log.msg('Submitted '+locale+' locale')
- # let's submit the BuildSet for this locale
- self.scheduler.submitBuildSet(
- buildset.BuildSet(self.scheduler.builderNames,
- self.NoMergeStamp(branch=self.scheduler.branch),
- self.scheduler.reason,
- properties = props))
-
- def getLocales(self):
- """
- It returns a list of locales if the user has set a list of locales
- in the scheduler OR it returns a defered
- """
- if self.locales:
- log.msg('L10nMixin.getLocales():: The user has set a list of locales')
- return self.locales
- else:
- log.msg("L10nMixin:: Getting locales from: "+self.localesURL)
- if self.repoType.find('cvs') >= 0:
- args = ['cvs', '-q', '-d', self.cvsRoot, 'co', '-p', self.localesURL]
- # communicate() returns a tuple - stdio, stderr
- # the output of cvs has a '\n' element at the end
- # a last '' string is generated that we get rid of
- return (lambda lines: [line.split(' ')[0] for line in lines]) \
- (subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0].split('\n')[0:-1])
- else: # the repoType is 'hg'
- # getPage returns a defered that will return a string
- d = getPage(self.localesURL, timeout = 5 * 60)
- # we expect that getPage will return the output of "all-locales"
- # or "shipped-locales" or any file that contains a locale per line
- # in the begining of the line e.g. "en-GB" or "ja linux win32"
- # this code will only care about the first argument appearing in the line
- d.addCallback(lambda data: [line.split(' ')[0] \
- for line in data.split("\n") if line])
- return d
-
- def createL10nBuilds(self):
- """
- We request to get the locales that we have to process and which
- method to call once they are ready
- """
- log.msg('L10nMixin:: A list of locales is going to be requested')
- d = defer.maybeDeferred(self.getLocales)
- d.addCallback(self._cbLoadedLocales)
-
- def getLocale(self, builderName):
- """
- This is the method that the schedulers call to request
- the next locale to build by the builder that is doing the request
- """
- buildDescription = self.queue[builderName].pop()
- log.msg('%s requests next locale and %s was given' % (builderName, buildDescription.locale))
- return buildDescription
-
-class NightlyL10n(Nightly):
- """
- NightlyL10n is used to paralellize the generation of l10n builds.
-
- NightlyL10n is designed to be used with its special Build class,
- which actually pops the build items and moves the relevant information
- onto build properties for steps to use.
- """
-
- compare_attrs = ('name', 'builderNames',
- 'minute', 'hour', 'dayOfMonth', 'month',
- 'dayOfWeek', 'branch')
-
- def __init__(self, name, builderNames, minute=0, hour='*', dayOfMonth='*', month='*', dayOfWeek='*',
- repoType=None, repoPath=None, baseTag=None, localesFile=None,
- branch=None, cvsRoot=None, locales=None):
-
- Nightly.__init__(self, name, builderNames, minute, hour, dayOfMonth, month, dayOfWeek, branch)
- # To avoid breakage in 1.9.0 that uses this scheduler
- # TODO: change repoType=None to repoType when fixed
- if not repoType:
- repoType = 'cvs'
- self.helper = L10nMixin(self,
- repoType = repoType, repoPath = repoPath,
- baseTag = baseTag,
- localesFile = localesFile,
- cvsRoot = cvsRoot,
- locales = locales)
-
- def doPeriodicBuild(self):
- # Schedule the next run (as in Nightly's doPeriodicBuild)
- self.setTimer()
- self.helper.createL10nBuilds()
-
-class DependentL10n(Dependent):
- """
- This scheduler runs some set of 'downstream' builds when the
- 'upstream' scheduler has completed successfully.
- """
-
- compare_attrs = ('name', 'upstream', 'builders')
-
- def __init__(self, name, upstream, builderNames,
- repoType, repoPath=None,
- baseTag='default', localesFile=None,
- cvsRoot=None, locales=None):
- Dependent.__init__(self, name, upstream, builderNames)
- # The next two lines has been added because of:
- # _cbLoadedLocales's BuildSet submit needs them
- self.branch = None
- self.reason = None
- self.helper = L10nMixin(self,
- repoType = repoType, repoPath = repoPath,
- baseTag = baseTag,
- localesFile = localesFile,
- cvsRoot = cvsRoot,
- locales = locales)
-
- # ss is the source stamp that we don't use currently
- def upstreamBuilt(self, ss):
- self.helper.createL10nBuilds()
View
53 l10n/l10n.py → log.py
@@ -20,25 +20,42 @@
#
# Contributor(s):
# Axel Hecht <axel@mozilla.com>
-# Armen Zambrano Gasparnian <armenzg@mozilla.com>
# ***** END LICENSE BLOCK *****
+from twisted.python import log
+import logging
+from logging import DEBUG
-from buildbot.steps.shell import ShellCommand
-from buildbot import process
+class LogFwd(object):
+ @classmethod
+ def write(cls, msg):
+ log.msg(msg.rstrip())
+ pass
+ @classmethod
+ def flush(cls):
+ pass
-class BuildL10n(process.base.Build):
- """
- I subclass process.Build just to set some properties I get from
- the scheduler in setupBuild when I call "getNextLocale".
- """
- # this is the scheduler, needs to be set on the class in master.cfg
- buildQueue = None
+def init(**kw):
+ logging.basicConfig(stream = LogFwd,
+ format = '%(name)s: (%(levelname)s) %(message)s')
+ for k, v in kw.iteritems():
+ logging.getLogger(k).setLevel(v)
- def setupBuild(self, expectations):
- #The L10n schedulers have a queue for each builder
- bd = self.buildQueue.getNextLocale(self.builder.name)
- if not bd:
- raise Exception("No build found for %s on %s, bad mojo" % \
- (self.builder.name, self.slavename))
- process.base.Build.setupBuild(self, expectations)
- self.setProperty('locale', bd.locale, "Build")
+def critical(cat, msg):
+ logging.getLogger(cat).critical(msg)
+ pass
+
+def error(cat, msg):
+ logging.getLogger(cat).error(msg)
+ pass
+
+def warning(cat, msg):
+ logging.getLogger(cat).warning(msg)
+ pass
+
+def info(cat, msg):
+ logging.getLogger(cat).info(msg)
+ pass
+
+def debug(cat, msg):
+ logging.getLogger(cat).debug(msg)
+ pass
View
183 test/test_l10n.py
@@ -0,0 +1,183 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2008
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+'''Tests for buildbot integration on l10n builds.
+
+The tests in this module use the helpers from utils to test
+the l10n build logic. For each scenario, a number of changes is
+fed to the system, and after the dummy builds are completed,
+both the buildbot status and the database are checked for
+expected results.
+'''
+
+import utils
+from utils import Request
+
+__all__ = ['L10nTest', 'EnTest', 'ChangeCoalescenceTest']
+
+config = """
+import buildbotcustom.l10n
+import buildbotcustom.log
+buildbotcustom.log.init(scheduler = buildbotcustom.log.DEBUG,
+ dispatcher = buildbotcustom.log.DEBUG)
+
+from buildbot.process import factory
+from buildbot.steps import dummy
+from buildbot.buildslave import BuildSlave
+s = factory.s
+
+f = factory.BuildFactory([
+ s(dummy.Dummy, timeout=%(timeout)i),
+ ])
+
+BuildmasterConfig = c = {}
+c['slaves'] = [BuildSlave('bot1', 'sekrit')]
+c['schedulers'] = []
+
+from buildbotcustom.l10n import Scheduler, L10nDispatcher, EnDispatcher
+
+s = Scheduler("l10n", "")
+c['schedulers'].append(s)
+
+paths = ['browser', 'toolkit']
+locales = ['af', 'ar']
+s.addDispatcher(L10nDispatcher(paths, 'HEAD', 'fx_tree'))
+s.addDispatcher(EnDispatcher(paths, 'HEAD', 'fx_tree', 'mozilla/'))
+s.locales['fx_tree'] = locales
+s.builders['fx_tree'] = ['dummy']
+s.apps['fx_tree'] = 'browser'
+
+c['builders'] = []
+c['builders'].append({'name': 'dummy', 'slavename': 'bot1',
+ 'builddir': 'dummy1', 'factory': f})
+c['slavePortnum'] = 0
+"""
+
+testconfig = config
+
+class L10nTest(utils.TimedChangesQueue):
+ """Test case to verify that a single l10n check-in actually
+ triggers a single l10n build.
+
+ """
+ config = testconfig % dict(timeout=1)
+ requests = (
+ Request(when = 1200000000,
+ delay = 0,
+ branch = 'HEAD',
+ files = "l10n/af/browser/file"),
+ )
+ def allBuildsDone(self