update metabuildbot to build newly split packages (refs buildbot#880)

This allows some builders to skip the buildmaster tests

builders.py

@@ -8,6 +8,67 @@
 
 from metabbotcfg.slaves import slaves
 
+builders = []
+
+#### docs
+
+# build on any of the slaves that can build docs
+# TODO: make the results available somewhere
+
+docs_factory = factory.BuildFactory()
+docs_factory.addStep(Git(repourl='git://github.com/djmitche/buildbot.git', mode="update"))
+docs_factory.addStep(ShellCommand(command="make docs", name="create docs"))
+builders.append({
+	'name' : 'docs',
+	'slavenames' : [ sl.slavename for sl in slaves if sl.has_texinfo ],
+	'workdir' : 'docs',
+	'factory' : docs_factory,
+	'category' : 'docs' })
+
+#### simple slaves
+
+# some slaves just do "simple" builds: get the source, run the tests.  These are mostly
+# windows machines where we don't have a lot of flexibility to mess around with virtualenv
+def mksimplefactory(slave):
+	f = factory.BuildFactory()
+	f.addSteps([
+	Git(repourl='git://github.com/djmitche/buildbot.git', mode="copy"),
+	#FileDownload(mastersrc="bbimport.py", slavedest="bbimport.py", flunkOnFailure=True),
+	#ShellCommand(workdir="build/master", env={'PYTHONPATH' : '.;.'}, command=r"python ..\bbimport.py"),
+	# use workdir instead of testpath because setuptools sticks its own eggs (including
+	# the running version of buildbot) into sys.path *before* PYTHONPATH, but includes
+	# "." in sys.path even before the eggs
+	Trial(workdir="build/slave", testpath=".",
+		env={ 'PYTHON_EGG_CACHE' : '../' },
+		tests='buildslave.test',
+		usePTY=False,
+		name='test slave'),
+	])
+	if slave.test_master:
+		f.addStep(
+		Trial(workdir="build/master", testpath=".",
+			env={ 'PYTHON_EGG_CACHE' : '../' },
+			tests='buildbot.test',
+			usePTY=False,
+			name='test master'),
+		)
+	return f
+
+for sl in slaves:
+	if not sl.use_simple: continue
+	name = sl.slavename
+	builders.append({
+		'name' : 'slave-%s' % name,
+		'slavenames' : [ name ],
+		'workdir' : 'slave-%s' % name,
+		'factory' : mksimplefactory(sl),
+		'category' : 'slave' })
+
+#### full slaves
+
+# this will eventually run against a variety of Twisted and Python versions on
+# slaves that can support it, but for now it's a lot like the simple builders, except
+# that it uses virtualenv
 def mkfactory(*tests):
 	f = factory.BuildFactory()
 	f.addSteps([
@@ -19,62 +80,52 @@ def mkfactory(*tests):
 		PYTHON=../sandbox/bin/python; PATH=../sandbox/bin:$PATH; 
 		export PYTHON_EGG_CACHE=$PWD/..;
 		# and somehow the install_requires in setup.py doesn't always work:
-		$PYTHON -c 'import json' || $PYTHON -c 'import simplejson' ||
+		$PYTHON -c 'import json' 2>/dev/null || $PYTHON -c 'import simplejson' ||
 					../sandbox/bin/easy_install simplejson || exit 1;
-		$PYTHON -c 'import sqlite3' || $PYTHON -c 'import pysqlite2.dbapi2' ||
+		$PYTHON -c 'import sqlite3' 2>/dev/null || $PYTHON -c 'import pysqlite2.dbapi2' ||
 					../sandbox/bin/easy_install pysqlite || exit 1;
+		../sandbox/bin/easy_install twisted || exit 1;
+		../sandbox/bin/easy_install jinja2 || exit 1;
 		../sandbox/bin/easy_install mock || exit 1;
-		$PYTHON setup.py build install || exit 1;
 	"""),
 		flunkOnFailure=True,
-		name="virtualenv install and build"),
+		haltOnFailure=True,
+		name="virtualenv setup"),
 	ShellCommand(usePTY=False, command=textwrap.dedent("""
 		PYTHON=../sandbox/bin/python; PATH=../sandbox/bin:$PATH; 
 		export PYTHON_EGG_CACHE=$PWD/..;
-		$PYTHON -c 'print "USING VIRTUALENV"; import sys; print sys.version; import twisted; print twisted.version'
+		$PYTHON -c 'import sys; print "Python:", sys.version; import twisted; print "Twisted:", twisted.version'
 	"""),
 		name="versions"),
-	Trial(testpath=".",
-		env={ 'PYTHON_EGG_CACHE' : '../' },
-		tests=list(tests),
-		trial="../sandbox/bin/trial", usePTY=False),
-	])
-	return f
-
-def mksimplefactory(*tests):
-	f = factory.BuildFactory()
-	f.addSteps([
-	Git(repourl='git://github.com/djmitche/buildbot.git', mode="copy"),
-	Trial(testpath=".",
-		env={ 'PYTHON_EGG_CACHE' : '../' },
-		tests=list(tests),
-		usePTY=False),
+	# see note above about workdir vs. testpath
+	Trial(workdir="build/slave", testpath='.',
+		env={ 'PYTHON_EGG_CACHE' : '../../' },
+		tests='buildslave.test',
+		trial="../../sandbox/bin/trial",
+		usePTY=False,
+		name='test slave'),
+	Trial(workdir="build/master", testpath='.',
+		env={ 'PYTHON_EGG_CACHE' : '../../' },
+		tests='buildbot.test',
+		trial="../../sandbox/bin/trial",
+		usePTY=False,
+		name='test master'),
 	])
 	return f
 
-virtualenv_factory = mkfactory('buildbot.test')
-simple_factory = mksimplefactory('buildbot.test')
-
-docs_factory = factory.BuildFactory()
-docs_factory.addStep(Git(repourl='git://github.com/djmitche/buildbot.git', mode="update"))
-docs_factory.addStep(ShellCommand(command="make docs", name="create docs"))
-
-builders = []
-builders.append({
-	'name' : 'docs',
-	'slavenames' : [ sl.slavename for sl in slaves if sl.has_texinfo ],
-	'builddir' : 'docs',
-	'factory' : docs_factory,
-	'category' : 'docs' })
 
 for sl in slaves:
+	if sl.use_simple: continue
 	name = sl.slavename
-	fact = virtualenv_factory
-	if sl.use_simple:
-		fact = simple_factory
 	builders.append({
 		'name' : 'slave-%s' % name,
 		'slavenames' : [ name ],
-		'builddir' : 'slave-%s' % name,
-		'factory' : fact,
+		'workdir' : 'slave-%s' % name,
+		'factory' : mkfactory(sl),
 		'category' : 'full' })
+
+##### sdist
+
+# TODO
+# build buildbot and buildslave source distributions, then untar them, build/install them into a new
+# virtualenv, and run the tests there

slaves.py

@@ -2,15 +2,20 @@
 from buildbot.buildslave import BuildSlave
 
 class MySlave(BuildSlave):
-	def __init__(self, name, os, tw, py, has_texinfo=False, use_simple=False, **kwargs):
+	def __init__(self, name, os, py, has_texinfo=False, use_simple=False,
+			test_master=True, **kwargs):
 		password = self.get_pass(name)
 		BuildSlave.__init__(self, name, password, **kwargs)
 		self.slavename = name
 		self.os = os
-		self.tw = tw
 		self.py = py
+		# true if this box can build docs
 		self.has_texinfo = has_texinfo
+		# true if this box should use a 'simple' factory, meaning no virtualenv
+		# (basically good for windows)
 		self.use_simple = use_simple
+		# true if this box can test the buildmaster
+		self.test_master = test_master
 
 	def get_pass(self, name):
 		# get the password based on the name
@@ -20,18 +25,21 @@ def get_pass(self, name):
 
 slaves = [
 	# Steve 'Ashcrow' Milner
-	MySlave('centos_5_python2_4', "linux", "801", "23",
+	MySlave('centos_5_python2_4', "linux", "24",
 		),
 
  	# Dustin Mitchell
-	MySlave('knuth.r.igoro.us', "linux", "820", "25",
+	MySlave('knuth.r.igoro.us', "linux", "25",
 		has_texinfo=True,
 		),
 
-	MySlave('xp-cygwin-1.7', 'winxp', '1000', '25',
-		use_simple=True),
-	MySlave('win7-py26', 'win7', '1000', '26',
-		use_simple=True),
+	MySlave('xp-cygwin-1.7', 'winxp', '25',
+		use_simple=True,
+		test_master=False, # master doesn't work on cygwin
+		),
+	MySlave('win7-py26', 'win7', '26',
+		use_simple=True,
+		),
 ]
 
 # these are slaves that haven't been up and from whose owners I have not heard in a while

status.py

@@ -1,7 +1,17 @@
 status = []
 
 from buildbot.status import html
-status.append(html.WebStatus(http_port=8010, allowForce=False, order_console_by_time=True,
+from buildbot.status.web.authz import Authz
+from buildbot.status.web.auth import BasicAuth
+
+users = [ ('dev', 'bbot!')] # it's not *that* secret..
+authz = Authz(auth=BasicAuth(users),
+	forceBuild='auth',
+)
+status.append(html.WebStatus(
+		http_port=8010,
+		authz=authz,
+		order_console_by_time=True,
 		revlink="http://github.com/djmitche/buildbot/commit/%s",
 		changecommentlink=(r'\b#(\d+)\b', r'http://buildbot.net/trac/ticket/\1',
 				   r'Ticket \g<0>')))