From c775dcfac6ef7367e089d8ee0107f39732157c2a Mon Sep 17 00:00:00 2001 From: Adam Groszer Date: Mon, 3 Sep 2012 07:26:07 +0000 Subject: [PATCH] testing distribute --- master.cfg | 229 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 135 insertions(+), 94 deletions(-) diff --git a/master.cfg b/master.cfg index eb2e70d..d9a7cb8 100644 --- a/master.cfg +++ b/master.cfg @@ -86,6 +86,11 @@ class Git(source.Git): show_revno = False # the LastChange step does it better +class Mercurial(source.Mercurial): + pass + #show_revno = False # the LastChange step does it better + + class LastChange(shell.ShellCommand): command = ['svn', 'log', '--limit', '1'] @@ -344,7 +349,7 @@ class Platform(object): withcompiler = '' def __init__(self, **kw): - for k,v in kw.items(): + for k, v in kw.items(): setattr(self, k, v) self.bits = int(self.name[-2:]) @@ -680,16 +685,16 @@ def setupZTK_11_tests(slow_lock, hour=02, minute=10): ###################################### # zc.buildout -def get_same_platforms(bits, platforms, exclude): - data = [] - for platform in platforms.values(): - if platform.bits == bits and platform.name != exclude: - data.append("set PYTHON%s=%s" % ( - platform.majorVersion, platform.python)) - return data +ZC_BUILDOUT_PLATFORMS = CLEAN_PLATFORMS.copy() +del ZC_BUILDOUT_PLATFORMS['py_244_win32'] +del ZC_BUILDOUT_PLATFORMS['py_254_win32'] + +ZC_BUILDOUT_PLATFORMS_16 = CLEAN_PLATFORMS.copy() +del ZC_BUILDOUT_PLATFORMS_16['py_244_win32'] def zc_buildout_dev_builder(name, slavename, platform, locks, branch='HEAD'): + # XXX: leave spaces later builddir = name.replace(' ', '_') f = factory.BuildFactory() f.addStep(Git( @@ -717,36 +722,21 @@ def zc_buildout_dev_builder(name, slavename, platform, locks, branch='HEAD'): haltOnFailure=True, name="dev.py", description="dev.py")) - #f.addStep(shell.ShellCommand( - # command="%s" % platform.buildout, - # haltOnFailure=True, - # name="buildout", - # description="buildout", - # timeout=3600)) - #others = get_same_platforms(platform.bits, CLEAN_PLATFORMS, platform.name) - #others.append(r"bin\test.exe --exit-with-status -1") - #cmd = ' && '.join(others) cmd = r"bin\test.exe --exit-with-status -1" + cmd = platform.withcompiler % cmd f.addStep(Test( #command=[r"bin\test.exe", "--exit-with-status", "-1"], command=cmd, haltOnFailure=False, name="test", - description="test trunk")) + description="test " + branch)) return dict(name=name, slavename=slavename, builddir=builddir, factory=f, locks=locks) -ZC_BUILDOUT_PLATFORMS = CLEAN_PLATFORMS.copy() -del ZC_BUILDOUT_PLATFORMS['py_244_win32'] -del ZC_BUILDOUT_PLATFORMS['py_254_win32'] - -ZC_BUILDOUT_PLATFORMS_16 = CLEAN_PLATFORMS.copy() -del ZC_BUILDOUT_PLATFORMS_16['py_244_win32'] - def setup_zc_buildout_dev_tests(slow_lock, hour=02, minute=20, branch='HEAD', platforms=ZC_BUILDOUT_PLATFORMS): @@ -774,6 +764,68 @@ def setup_zc_buildout_dev_tests(slow_lock, hour=02, minute=20, branch='HEAD', ) +# zc.buildout +###################################### + +###################################### +# distribute + +DISTRIBUTE_PLATFORMS = CLEAN_PLATFORMS.copy() + + +def distribute_dev_builder(name, slavename, platform, locks): + builddir = name.replace(' ', '_') + f = factory.BuildFactory() + f.addStep(Mercurial( + repourl="https://bitbucket.org/tarek/distribute", + haltOnFailure=True, + mode="copy")) + + # XXX: LastChangeMercurial ??? + #lc = LastChangeGit() + #f.addStep(lc) + + cmd = platform.python + " setup.py test" + cmd = platform.withcompiler % cmd + f.addStep(Test( + #command=[r"bin\test.exe", "--exit-with-status", "-1"], + command=cmd, + haltOnFailure=False, + name="test", + description="test")) + return dict(name=name, + slavename=slavename, + builddir=builddir, + factory=f, + locks=locks) + + +def setup_distribute_dev_tests(slow_lock, hour=02, minute=20, + platforms=DISTRIBUTE_PLATFORMS): + #hour = 02 + #minute = 20 + builders = [] + for pname in sorted(platforms.keys()): + platform = platforms[pname] + name = "distribute_dev %s" % (platform.name, ) + builders.append(name) + c['builders'].append( + distribute_dev_builder(name, 'local', platform, [slow_lock])) + + c['schedulers'].append( + Nightly("%s_nightly" % name, [name], hour=hour, minute=minute)) + minute += 1 + + c['status'].append( + MailNotifier(mode="all", + fromaddr=FROMADDR, + extraRecipients=["agroszer@gmail.com"], + sendToInterestedUsers=False, + builders=builders, + messageFormatter=message_formatter) + ) + + # zc.buildout ###################################### @@ -992,9 +1044,7 @@ def message_formatter(mode, name, build, results, master_status): ###################################### # stuff taken from gocept.bsquare - -bsquare_platform = PLATFORMS['py_265_win32'] -bsquare_platname = 'py_265_32' +# but heavily modified now def bsquare_split_file(path): pieces = path.split("/") @@ -1006,7 +1056,7 @@ def bsquare_split_file(path): return ("%s/%s" % (project, branch), "/".join(pieces[2:])) -def bsquare_make_factory(svn_url): +def bsquare_make_factory(svn_url, platform): f = factory.BuildFactory() f.addStep(SVN(baseURL=svn_url, mode='copy')) @@ -1015,34 +1065,20 @@ def bsquare_make_factory(svn_url): # description=['bootstrapping'], # descriptionDone=['bootstrap'])) f.addStep(shell.ShellCommand( - command=[bsquare_platform.python, r"c:\buildmaster\bootstrap.py"], + command=[platform.python, r"c:\buildmaster\bootstrap.py"], haltOnFailure=True, name="bootstrap", description="bootstrap")) - #if is_win32: - # command = "bin\\buildout.exe" - #else: - # command = 'bin/buildout' - #f.addStep(Compile(name="buildout", - # command=command, - # description=['building'], - # descriptionDone=['build'])) - f.addStep(shell.ShellCommand( - command=bsquare_platform.withcompiler % r"bin\buildout.exe", + command=platform.withcompiler % r"bin\buildout.exe", haltOnFailure=True, name="buildout", description="buildout", timeout=3600)) - if is_win32: - command = "bin\\test.exe --exit-with-status -1" - else: - command = 'bin/test --exit-with-status -1' #-1 == stop on first error - f.addStep(Test(name="test", - command=command, + command="bin\\test.exe --exit-with-status -1", description=['testing'], descriptionDone=['tests'])) @@ -1052,12 +1088,12 @@ def bsquare_make_factory(svn_url): def bsquare_configure(c, svn_url, http_port=8010, allowForce=False, - svnuser = None, svnpasswd = None, - pollinterval = 30, nightlyhour=None, - poller = None, makefactory = bsquare_make_factory, - maxConcurrent = 2, - svnbin = 'svn', - slow_lock = None): + svnuser=None, svnpasswd=None, + pollinterval=30, nightlyhour=None, + poller=None, makefactory=bsquare_make_factory, + maxConcurrent=2, + svnbin='svn', + slow_lock=None): """Creates a buildout master configuration. The configuration returned is almost functional. You just need to add @@ -1099,47 +1135,51 @@ def bsquare_configure(c, projects = open("project-list.cfg", "rb").readlines() projects = [x.strip() for x in projects] + platforms = {'py_265_32': PLATFORMS['py_265_win32']} + for project in projects: if not project or project.startswith('#'): - continue #comment or empty line - - if isinstance(makefactory, dict): - f = makefactory.get(project, - makefactory.get('__default__', makefactory)) - f = f(svn_url) - else: - f = makefactory(svn_url) - - bname = '%s_%s' % (project, bsquare_platname) - - c['builders'].append({ - 'name': bname, - 'slavename': 'local', - 'builddir': project, - 'factory': f, - 'locks': [slow_lock], - }) - del f - - c['schedulers'].append(Scheduler( - name = bname, - branch = "%s/trunk" % project, - treeStableTimer = pollinterval*2+10, - builderNames = [bname])) - - if nightlyhour is not None: - c['schedulers'].append(Nightly( - "%s nightly" % project, [bname], hour=[nightlyhour], - branch="%s/trunk" % project)) - - c['status'].append( - MailNotifier(mode="failing", - fromaddr=FROMADDR, - extraRecipients=["agroszer@gmail.com", "zope-tests@zope.org"], - sendToInterestedUsers=False, - builders=[bname], - messageFormatter=message_formatter) - ) + continue # comment or empty line + + for bsquare_platname, bsquare_platform in platforms.items(): + if isinstance(makefactory, dict): + f = makefactory.get(project, + makefactory.get('__default__', makefactory)) + f = f(svn_url, platform=bsquare_platform) + else: + f = makefactory(svn_url, platform=bsquare_platform) + + bname = '%s_%s' % (project, bsquare_platname) + + c['builders'].append({ + 'name': bname, + 'slavename': 'local', + 'builddir': project, + 'factory': f, + 'locks': [slow_lock], + }) + del f + + c['schedulers'].append(Scheduler( + name=bname, + branch="%s/trunk" % project, + treeStableTimer=pollinterval * 2 + 10, + builderNames=[bname])) + + if nightlyhour is not None: + c['schedulers'].append(Nightly( + "%s nightly" % project, [bname], hour=[nightlyhour], + branch="%s/trunk" % project)) + + c['status'].append( + MailNotifier(mode="failing", + fromaddr=FROMADDR, + extraRecipients=["agroszer@gmail.com", + "zope-tests@zope.org"], + sendToInterestedUsers=False, + builders=[bname], + messageFormatter=message_formatter) + ) return c # stuff taken from gocept.bsquare @@ -1164,10 +1204,11 @@ setupZTK_dev_tests(slow_lock, hour=20, minute=10) setupZTK_10_tests(slow_lock, hour=20, minute=20) setupZTK_11_tests(slow_lock, hour=20, minute=30) setup_zc_buildout_dev_tests(slow_lock, hour=20, minute=40, branch='master') -setup_zc_buildout_dev_tests(slow_lock, hour=20, minute=40, branch='1.6.x', +setup_zc_buildout_dev_tests(slow_lock, hour=20, minute=50, branch='1.6.x', platforms=ZC_BUILDOUT_PLATFORMS_16) -setup_ZODB_dev_tests(slow_lock, hour=20, minute=50) -setup_BB_dev_tests(slow_lock, hour=20, minute=55) +setup_distribute_dev_tests(slow_lock, hour=21, minute=10) +setup_ZODB_dev_tests(slow_lock, hour=21, minute=50) +setup_BB_dev_tests(slow_lock, hour=21, minute=55) bsquare_configure(c, 'svn://svn.zope.org/repos/main/',