Hyper support for buildbot

Signed-off-by: Pierre Tardy <tardyp@gmail.com>

master/buildbot/test/fake/bworkermanager.py

@@ -65,6 +65,9 @@ def __init__(self, worker):
         self.unregistered = False
         self.worker = worker
 
+    def getPBPort(self):
+        return 1234
+
     def unregister(self):
         assert not self.unregistered, "called twice"
         self.unregistered = True

master/buildbot/test/fake/hyper.py

@@ -0,0 +1,50 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+
+class Client(object):
+    instance = None
+
+    def __init__(self, config):
+        self.config = config
+        self.containers = {}
+        #assert Client.instance is None
+        Client.instance = self
+
+    def start(self, container):
+        self.containers[container["Id"]]['started'] = True
+
+    def close(self):
+        # we should never close if we have live containers
+        assert len(self.containers) == 0, self.containers
+        Client.instance = None
+
+    def stop(self, id):
+        self.containers[id]['started'] = False
+
+    def wait(self, id):
+        return 0
+
+    def create_container(self, image, *args, **kwargs):
+        if 'buggy' in image:
+            raise Exception("we could not create this container")
+
+        ret = {'Id': '8a61192da2b3bb2d922875585e29b74ec0dc4e0117fcbf84c962204e97564cd7',
+               'Warnings': None}
+        self.containers[ret['Id']] = {'started': False, 'image': image}
+        return ret
+
+    def remove_container(self, id, **kwargs):
+        del self.containers[id]

master/buildbot/test/unit/test_worker_hyper.py

@@ -0,0 +1,131 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+from twisted.internet import defer
+from twisted.python import threadpool
+from twisted.trial import unittest
+
+from buildbot import config
+from buildbot.process.properties import Properties
+from buildbot.test.fake import fakemaster
+from buildbot.test.fake import hyper
+from buildbot.test.fake.reactor import NonThreadPool
+from buildbot.test.fake.reactor import TestReactor
+from buildbot.util.eventual import _setReactor
+from buildbot.worker import hyper as workerhyper
+from buildbot.worker.hyper import HyperLatentWorker
+
+
+class FakeBuild(object):
+    def render(self, r):
+        return "rendered:" + r
+
+
+class FakeBot(object):
+    info = {}
+
+    def notifyOnDisconnect(self, n):
+        self.n = n
+
+    def remoteSetBuilderList(self, builders):
+        return defer.succeed(None)
+
+    def loseConnection(self):
+        print "looseConnection"
+        self.n()
+
+
+class TestHyperLatentWorker(unittest.SynchronousTestCase):
+
+    def setUp(self):
+        self.patch(threadpool, 'ThreadPool', NonThreadPool)
+        self.reactor = TestReactor()
+        _setReactor(self.reactor)
+        self.patch(workerhyper, 'Hyper', hyper.Client)
+        self.build = Properties(
+            image="busybox:latest", builder="docker_worker")
+        self.worker = None
+
+    def tearDown(self):
+        if self.worker is not None:
+            self.worker.stopService()
+            self.reactor.pump([.1])
+        self.assertIsNone(hyper.Client.instance)
+        _setReactor(None)
+
+    def test_constructor_normal(self):
+        worker = HyperLatentWorker('bot', 'pass', 'tcp://hyper.sh/', 'foo', 'bar', 'debian:wheezy')
+        # class instanciation configures nothing
+        self.assertEqual(worker.client, None)
+        self.assertEqual(worker.client_args, None)
+
+    def test_constructor_nohyper(self):
+        self.patch(workerhyper, 'Hyper', None)
+        self.assertRaises(config.ConfigErrors, HyperLatentWorker,
+                          'bot', 'pass', 'tcp://hyper.sh/', 'foo', 'bar', 'debian:wheezy')
+
+    def test_constructor_badsize(self):
+        self.assertRaises(config.ConfigErrors, HyperLatentWorker,
+                          'bot', 'pass', 'tcp://hyper.sh/', 'foo', 'bar', 'debian:wheezy', hyper_size="big")
+
+    def makeWorker(self, **kwargs):
+        kwargs.setdefault('image', 'debian:wheezy')
+        worker = HyperLatentWorker('bot', 'pass', 'tcp://hyper.sh/', 'foo', 'bar', **kwargs)
+        self.worker = worker
+        master = fakemaster.make_master(testcase=self, wantData=True)
+        worker.setServiceParent(master)
+        worker.reactor = self.reactor
+        self.successResultOf(worker.startService())
+        return worker
+
+    def test_start_service(self):
+        worker = self.worker = self.makeWorker()
+        self.assertEqual(worker.client_args, {'clouds': {'tcp://hyper.sh/': {
+            'secretkey': 'bar', 'accesskey': 'foo'}}})
+        # client is lazily created on worker substantiation
+        self.assertEqual(worker.client, None)
+
+    def test_start_worker(self):
+        worker = self.makeWorker()
+
+        d = worker.substantiate(None, FakeBuild())
+        # we simulate a connection
+        worker.attached(FakeBot())
+        self.successResultOf(d)
+
+        self.assertIsNotNone(worker.client)
+        self.assertEqual(worker.instance, {
+            'Id': '8a61192da2b3bb2d922875585e29b74ec0dc4e0117fcbf84c962204e97564cd7',
+            'Warnings': None,
+            'image': 'rendered:debian:wheezy'})
+        # teardown makes sure all containers are cleaned up
+
+    def test_start_worker_but_no_connection_and_shutdown(self):
+        worker = self.makeWorker()
+        worker.substantiate(None, FakeBuild())
+        self.assertIsNotNone(worker.client)
+        self.assertEqual(worker.instance, {
+            'Id': '8a61192da2b3bb2d922875585e29b74ec0dc4e0117fcbf84c962204e97564cd7',
+            'Warnings': None,
+            'image': 'rendered:debian:wheezy'})
+        # teardown makes sure all containers are cleaned up
+
+    def test_start_worker_but_error(self):
+        worker = self.makeWorker(image="buggy")
+        d = worker.substantiate(None, FakeBuild())
+        self.reactor.pump([.1])
+        r = self.failureResultOf(d)
+        self.assertIsNotNone(worker.client)
+        self.assertEqual(worker.instance, None)
+        # teardown makes sure all containers are cleaned up

master/buildbot/worker/hyper.py

@@ -0,0 +1,160 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from __future__ import absolute_import
+
+import socket
+
+from twisted.internet import reactor as global_reactor
+from twisted.internet import defer
+from twisted.internet import threads
+from twisted.python import log
+from twisted.python import threadpool
+
+from buildbot import config
+from buildbot.interfaces import LatentWorkerFailedToSubstantiate
+from buildbot.worker import AbstractLatentWorker
+
+try:
+    import docker
+    from hypercompose.api import Hyper
+    _hush_pyflakes = [docker, Hyper]
+except ImportError:
+    Hyper = None
+
+
+class HyperLatentWorker(AbstractLatentWorker):
+    """hyper.sh is a docker CaaS company"""
+    instance = None
+    ALLOWED_SIZES = ['xxs', 'xs', 's', 'm', 'l', 'xl', 'xxl', 'xxl']
+    threadPool = None
+    client = None
+    reactor = global_reactor
+    client_args = None
+
+    def checkConfig(self, name, password, hyper_host,
+                    hyper_accesskey, hyper_secretkey, image, hyper_size="xs", masterFQDN=None, **kwargs):
+
+        # Set build_wait_timeout to 0s if not explicitely set: Starting a
+        # container is almost immediate, we can affort doing so for each build.
+
+        if 'build_wait_timeout' not in kwargs:
+            kwargs['build_wait_timeout'] = 0
+
+        AbstractLatentWorker.checkConfig(self, name, password, **kwargs)
+
+        if not Hyper:
+            config.error("The python modules 'docker-py>=1.4' and 'hypercompose' are needed to use a"
+                         " HyperLatentWorker")
+
+        if hyper_size not in self.ALLOWED_SIZES:
+            config.error("Size is not valid %s vs %r".format(hyper_size, self.ALLOWED_SIZES))
+
+    def reconfigService(self, name, password, hyper_host,
+                        hyper_accesskey, hyper_secretkey, image, hyper_size="xs", masterFQDN=None, **kwargs):
+
+        AbstractLatentWorker.reconfigService(self, name, password, **kwargs)
+        self.size = hyper_size
+        self.image = image
+
+        # Prepare the parameters for the Docker Client object.
+        self.client_args = {'clouds': {
+            hyper_host: {
+                "accesskey": hyper_accesskey,
+                "secretkey": hyper_secretkey
+            }
+        }}
+        if not masterFQDN:  # also match empty string (for UI)
+            masterFQDN = socket.getfqdn()
+        self.masterFQDN = masterFQDN
+
+    @defer.inlineCallbacks
+    def stopService(self):
+        print "stopping service", self.client
+        # stopService will call stop_instance if the worker was up.
+        yield AbstractLatentWorker.stopService(self)
+        print "stopped service", self.client
+        # we cleanup our thread and session (or reactor.stop will hang)
+        if self.client is not None:
+            print "closing client"
+            self.client.close()
+            self.client = None
+        if self.threadPool is not None:
+            yield self.threadPool.stop()
+            self.threadPool = None
+
+    def createEnvironment(self):
+        result = {
+            "BUILDMASTER": self.masterFQDN,
+            "WORKERNAME": self.name,
+            "WORKERPASS": self.password
+        }
+        if self.registration is not None:
+            result["BUILDMASTER_PORT"] = str(self.registration.getPBPort())
+        return result
+
+    @defer.inlineCallbacks
+    def start_instance(self, build):
+        if self.instance is not None:
+            raise ValueError('instance active')
+
+        if self.threadPool is None:
+            # requests_aws4 is documented to not be thread safe, so we must serialize access
+            self.threadPool = threadpool.ThreadPool(minthreads=1, maxthreads=1, name='hyper')
+            self.threadPool.start()
+
+        if self.client is None:
+            self.client = Hyper(self.client_args)
+
+        image = yield build.render(self.image)
+        res = yield threads.deferToThreadPool(self.reactor, self.threadPool, self._thd_start_instance, image)
+        defer.returnValue(res)
+
+    def _thd_start_instance(self, image):
+        instance = self.client.create_container(
+            image,
+            name=('%s%s' % (self.workername, id(self))).replace("_", "-"),
+            environment=self.createEnvironment(),
+        )
+
+        if instance.get('Id') is None:
+            raise LatentWorkerFailedToSubstantiate(
+                'Failed to start container'
+            )
+        shortid = instance['Id'][:6]
+        log.msg('Container created, Id: %s...' % (shortid,))
+        instance['image'] = image
+        self.instance = instance
+        self.client.start(instance)
+        return [instance['Id'], image]
+
+    def stop_instance(self, fast=False):
+        if self.instance is None:
+            # be gentle. Something may just be trying to alert us that an
+            # instance never attached, and it's because, somehow, we never
+            # started.
+            return defer.succeed(None)
+        instance = self.instance
+        self.instance = None
+        return threads.deferToThreadPool(self.reactor, self.threadPool,
+                                         self._thd_stop_instance, instance, fast)
+
+    def _thd_stop_instance(self, instance, fast):
+        log.msg('Stopping container %s...' % instance['Id'][:6])
+        print "stopping"
+        self.client.stop(instance['Id'])
+        if not fast:
+            self.client.wait(instance['Id'])
+        self.client.remove_container(instance['Id'], v=True, force=True)