Add a new latent buildslave using libvirt

contrib/libvirt/network.xml

@@ -0,0 +1,16 @@
+<network>
+  <name>buildbot-network</name>
+  <bridge name="virbr1" />
+  <forward />
+  <ip address="192.168.201.1" netmask="255.255.255.0">
+    <dhcp>
+      <range start="192.168.201.2" end="192.168.201.254" />
+      <host mac="00:16:3e:77:e2:01" name="minion1" ip="192.168.201.2">
+      <host mac="00:16:3e:77:e2:02" name="minion2" ip="192.168.201.3">
+      <host mac="00:16:3e:77:e2:03" name="minion3" ip="192.168.201.4">
+      <host mac="00:16:3e:77:e2:04" name="minion4" ip="192.168.201.5">
+      <host mac="00:16:3e:77:e2:05" name="minion5" ip="192.168.201.6">
+      <host mac="00:16:3e:77:e2:06" name="minion6" ip="192.168.201.7">
+    </dhcp>
+  </ip>
+</network>

contrib/libvirt/vmbuilder

@@ -0,0 +1,147 @@
+#! /usr/bin/env python
+
+"""
+This script can be used to generate an Ubuntu VM that is suitable for use by the libvirt backend of buildbot.
+
+It creates a buildbot slave and then changes the buildbot.tac to get its username from the hostname. The hostname is set by
+changing the DHCP script.
+
+See network.xml for how to map a MAC address to an IP address and a hostname. You can load that configuration on to your master by running::
+
+    virsh net-define network.xml
+
+Note that the VM's also need their MAC address set, and configuring to use the new network, or this won't work..
+"""
+
+import os, platform, tempfile
+
+if platform.machine() == "x86_64":
+    arch = "amd64"
+else:
+    arch = "i386"
+
+postboot = """\
+#!/bin/sh
+chroot $1 update-rc.d -f buildbot remove
+
+chroot $1 addgroup --system minion
+chroot $1 adduser --system --home /var/local/buildbot --shell /bin/bash --ingroup zope --disabled-password --disabled-login minion
+
+mkdir -p $1/var/local/buildbot
+chroot $1 chown minion: /var/local/buildbot
+
+chroot $1 sudo -u minion /usr/bin/buildbot create-slave /var/local/buildbot %(master_host)s:%(master_port)s %(slave)s %(slave_password)s
+
+cat > $1/etc/default/buildbot << HERE
+BB_NUMBER[0]=0
+BB_NAME[0]="minion"
+BB_USER[0]="minion"
+BB_BASEDIR[0]="/var/local/buildbot"
+BB_OPTIONS[0]=""
+BB_PREFIXCMD[0]=""
+HERE
+
+cat > $1/var/local/buildbot/buildbot.tac << HERE
+from twisted.application import service
+from buildbot.slave.bot import BuildSlave
+import socket
+
+basedir = r'/var/local/buildbot'
+buildmaster_host = '%(master_host)s'
+port = %(master_port)s
+slavename = socket.gethostname()
+passwd = "%(slave_password)s"
+keepalive = 600
+usepty = 0
+umask = None
+maxdelay = 300
+rotateLength = 1000000
+maxRotatedFiles = None
+
+application = service.Application('buildslave')
+s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
+               keepalive, usepty, umask=umask, maxdelay=maxdelay)
+s.setServiceParent(application)
+HERE
+
+cat > $1/etc/dhcp3/dhclient-exit-hooks.d/update-hostname << HERE
+if [ x\$reason != xBOUND ] && [ x\$reason != xREBIND ] && [ x\$reason != xREBOOT ]; then exit; fi
+echo Updating hostname: \$new_host_name
+hostname \$new_host_name
+echo Starting buildbot
+/etc/init.d/buildbot stop || true
+/etc/init.d/buildbot start
+HERE
+
+cat > $1/etc/udev/rules.d/virtio.rules << HERE
+KERNEL=="vda*", SYMLINK+="sda%%n"
+HERE
+"""
+
+class VMBuilder:
+
+    """ Class that executes ubuntu-vm-builder with appropriate options """
+
+    postboot = postboot
+
+    defaults = {
+        "rootsize": 8192,
+        "mem": 1024,
+        "domain": 'yourdomain.com',
+        "hostname": "ubuntu",
+        "arch": arch,
+        "variant": "minbase",
+        "components": "main,universe,multiverse,restricted",
+        "lang": "en_GB.UTF-8",
+        "timezone": "Europe/London",
+        "execscript": os.path.realpath(os.path.join(os.curdir, "postboot.sh")),
+        "addpkg": [
+            "standard^", "server^", "gpgv", "openssh-server", "buildbot", "subversion",
+            ],
+    }
+
+    def __init__(self, hypervisor="kvm", suite="karmic", destdir="ubuntu", **kw):
+        self.hypervisor = hypervisor
+        self.suite = suite
+        self.destdir = destdir
+        self.options = self.defaults.copy()
+        self.options.update(**kw)
+        f = tempfile.NamedTemporaryFile(delete=False, prefix="/var/tmp/")
+        print >>f, self.postboot % {
+            'master_host': '192.168.201.1',
+            'master_port': '8081',
+            'slave': 'slave',
+            'slave_password': 'password',
+        }
+        f.close()
+        os.chmod(f.name, 0755)
+        self.options['execscript'] = f.name
+
+    def build(self):
+        optstring = []
+        for k, v in self.options.items():
+            if type(v) == type([]):
+                for i in v:
+                    if i:
+                        optstring.append("--%s=%s" % (k, i))
+            else:
+                if v:
+                    optstring.append("--%s=%s" % (k, v))
+        execute=("ubuntu-vm-builder %s %s -d%s %s" % (
+                    self.hypervisor,
+                    self.suite,
+                    self.destdir,
+                    " ".join(optstring)))
+        print execute
+        os.system(execute)
+
+if __name__ == "__main__":
+    import sys, socket, optparse
+
+    parser = optparse.OptionParser(usage="%prog [options] project")
+    parser.add_option("-p", "--proxy", help="http proxy URL")
+    (options, args) = parser.parse_args()
+    builder = VMBuilder(proxy=options.proxy)
+    builder.build()
+
+

docs/cfg-buildslaves.texinfo

@@ -128,6 +128,7 @@ latent buildslaves, and a concrete implementation for AWS EC2.
 
 @menu
 * Amazon Web Services Elastic Compute Cloud ("AWS EC2")::
+* Libvirt::
 * Dangers with Latent Buildslaves::
 * Writing New Latent Buildslaves::
 @end menu
@@ -348,6 +349,58 @@ instance.  It defaults to 10 minutes.
 "keypair_name" and "security_name" allow you to specify different names for
 these AWS EC2 values.  They both default to "latent_buildbot_slave".
 
+@node Libvirt
+@subsection Libvirt
+
+@url{http://www.libvirt.org/,,libvirt} is a virtualization API for interacting
+with the virtualization capabilities of recent versions of Linux and other OSes.
+It is LGPL and comes with a stable C API, and python bindings.
+
+This means we know have an API which when tied to buildbot allows us to have slaves
+that run under Xen, QEMU, KVM, LXC, OpenVZ, User Mode Linux, VirtualBox and VMWare.
+
+@heading Setting up libvirt
+
+We won't show you how to set up libvirt, but there are a few things you should keep
+in mind.
+
+@itemize @bullet
+@item
+If you are using the system libvirt, your buildbot master user will need to be in the
+libvirtd group.
+
+@item
+You need to think carefully about your virtual network *first*. Will NAT be enough?
+What IP will my VM's need to connect to for connecting to the master?
+@end itemize
+
+@heading Configuring your base image
+
+You need to create a base image for your builds that has everything needed to build
+your software. You need to configure base image with a buildbot slave that is configured
+to connect to the master on boot.
+
+If you want to have multiple slaves using the same base image it can be annoying to duplicate
+the image just to change the buildbot credentials. One option is to use libvirt's DHCP
+server to allocate an identity to the slave: DHCP sets a hostname, and the slave takes its
+identity from that.
+
+Doing all this is really beyond the scope of the manual, but there is a vmbuilder script
+and a network.xml file to create such a DHCP server in contrib/ that should get you started.
+
+@heading Configuring your Master
+
+If you want to add a simple on demand VM to your setup, you only need the following. We
+set the username to minion1, the password to sekrit. The base image is caleld base_image
+and a copy of it will be made for the duration of the VM's life. That copy will be thrown
+away every time a build is complete.
+
+@example
+from buildbot.libvirtbuildslave import LibVirtBuildSlave
+c['slaves'] = [LibVirtBuildSlave('minion1', 'sekrit',
+                                   '/home/buildbot/images/minion1', '/home/buildbot/images/base_image')]
+@end example
+
 @node Dangers with Latent Buildslaves
 @subsubsection Dangers with Latent Buildslaves