Merge branch 'clean_shutdown' of git://github.com/catlee/buildbot

* 'clean_shutdown' of git://github.com/catlee/buildbot:
  Adding some tests for clean shutdown
  Implemented clean master shutdown.

buildbot/master.py

@@ -16,7 +16,7 @@
 
 import buildbot
 # sibling imports
-from buildbot.util import now, safeTranslate
+from buildbot.util import now, safeTranslate, eventual
 from buildbot.pbutil import NewCredPerspective
 from buildbot.process.builder import Builder, IDLE
 from buildbot.status.builder import Status, BuildSetStatus
@@ -42,6 +42,7 @@ class BotMaster(service.MultiService):
     """
 
     debug = 0
+    reactor = reactor
 
     def __init__(self):
         service.MultiService.__init__(self)
@@ -75,10 +76,60 @@ def __init__(self):
         self.loop = DelegateLoop(self._get_processors)
         self.loop.setServiceParent(self)
 
+        self.shuttingDown = False
+
     def setMasterName(self, name, incarnation):
         self.master_name = name
         self.master_incarnation = incarnation
 
+    def cleanShutdown(self):
+        if self.shuttingDown:
+            return
+        log.msg("Initiating clean shutdown")
+        self.shuttingDown = True
+
+        # Wait for all builds to finish
+        l = []
+        for builder in self.builders.values():
+            for build in builder.builder_status.getCurrentBuilds():
+                l.append(build.waitUntilFinished())
+        if len(l) == 0:
+            log.msg("No running jobs, starting shutdown immediately")
+            self.loop.trigger()
+            d = self.loop.when_quiet()
+        else:
+            log.msg("Waiting for %i build(s) to finish" % len(l))
+            d = defer.DeferredList(l)
+            d.addCallback(lambda ign: self.loop.when_quiet())
+
+        # Flush the eventual queue
+        d.addCallback(eventual.flushEventualQueue)
+
+        # Finally, shut the whole process down
+        def shutdown(ign):
+            # Double check that we're still supposed to be shutting down
+            # The shutdown may have been cancelled!
+            if self.shuttingDown:
+                # Check that there really aren't any running builds
+                for builder in self.builders.values():
+                    n = len(builder.builder_status.getCurrentBuilds())
+                    if n > 0:
+                        log.msg("Not shutting down, builder %s has %i builds running" % (builder, n))
+                        log.msg("Trying shutdown sequence again")
+                        self.shuttingDown = False
+                        self.cleanShutdown()
+                        return
+                log.msg("Stopping reactor")
+                self.reactor.stop()
+        d.addCallback(shutdown)
+        return d
+
+    def cancelCleanShutdown(self):
+        if not self.shuttingDown:
+            return
+        log.msg("Cancelling clean shutdown")
+        self.shuttingDown = False
+
     def _sortfunc(self, b1, b2):
         t1 = b1.getOldestRequestTime()
         t2 = b2.getOldestRequestTime()
@@ -94,6 +145,8 @@ def _sort_builders(self, parent, builders):
         return sorted(builders, self._sortfunc)
 
     def _get_processors(self):
+        if self.shuttingDown:
+            return []
         builders = self.builders.values()
         sorter = self.prioritizeBuilders or self._sort_builders
         try:

buildbot/status/builder.py

@@ -2235,6 +2235,16 @@ def __init__(self, botmaster, basedir):
         self._buildset_success_waiters = collections.KeyedSets()
         self._buildset_finished_waiters = collections.KeyedSets()
 
+    @property
+    def shuttingDown(self):
+        return self.botmaster.shuttingDown
+
+    def cleanShutdown(self):
+        return self.botmaster.cleanShutdown()
+
+    def cancelCleanShutdown(self):
+        return self.botmaster.cancelCleanShutdown()
+
     def setDB(self, db):
         self.db = db
         self.db.subscribe_to("add-build", self._db_builds_changed)

buildbot/status/web/authz.py

@@ -13,6 +13,7 @@ class Authz(object):
             'stopBuild',
             'stopAllBuilds',
             'cancelPendingBuild',
+            'cleanShutdown',
     ]
 
     def __init__(self,

buildbot/status/web/base.py

@@ -170,7 +170,9 @@ def getContext(self, request):
                     tz = locale_tz,
                     metatags = [],
                     title = self.getTitle(request),
-                    welcomeurl = rootpath)
+                    welcomeurl = rootpath,
+                    authz = self.getAuthz(request),
+                    )
 
     def getStatus(self, request):
         return request.site.buildbot_service.getStatus()    

buildbot/status/web/baseweb.py

@@ -398,7 +398,10 @@ def setupSite(self):
             os.mkdir(htmldir)
 
         root = StaticFile(htmldir)
-        root.putChild("", RootPage())
+        root_page = RootPage()
+        root.putChild("", root_page)
+        root.putChild("shutdown", root_page)
+        root.putChild("cancel_shutdown", root_page)
 
         for name, child_resource in self.childrenToBeAdded.iteritems():
             root.putChild(name, child_resource)

buildbot/status/web/root.py

@@ -1,9 +1,33 @@
-from buildbot.status.web.base import HtmlResource
+from twisted.web.util import redirectTo
+from twisted.python import log
+from twisted.internet import reactor
+
+from buildbot.status.web.base import HtmlResource, path_to_root, path_to_authfail
+from buildbot.util.eventual import eventually
 
 class RootPage(HtmlResource):
     title = "Buildbot"
 
     def content(self, request, cxt):
+        status = self.getStatus(request)
+
+        if request.path == '/shutdown':
+            if self.getAuthz(request).actionAllowed("cleanShutdown", request):
+                eventually(status.cleanShutdown)
+                return redirectTo("/", request)
+            else:
+                return redirectTo(path_to_authfail(request), request)
+        elif request.path == '/cancel_shutdown':
+            if self.getAuthz(request).actionAllowed("cleanShutdown", request):
+                eventually(status.cancelCleanShutdown)
+                return redirectTo("/", request)
+            else:
+                return redirectTo(path_to_authfail(request), request)
+
+        cxt.update(
+                shutting_down = status.shuttingDown,
+                shutdown_url = request.childLink("shutdown"),
+                cancel_shutdown_url = request.childLink("cancel_shutdown"),
+                )
         template = request.site.buildbot_service.templates.get_template("root.html")
         return template.render(**cxt)
-

buildbot/status/web/templates/forms.html

@@ -128,6 +128,32 @@
  </form>
 {% endmacro %}
 
+{% macro clean_shutdown(shutdown_url, authz) %}
+  <form method="post" action="{{ shutdown_url }}" class='command clean_shutdown'>
+  <p>To cause this master to shut down cleanly, push the 'Clean Shutdown' button.</p>
+  <p>No other builds will be started on this master, and the master will
+  stop once all current builds are finished.</p>
+
+  {% if authz.needAuthForm('gracefulShutdown') %}
+    {{ auth() }}
+  {% endif %}
+
+  <input type="submit" value="Clean Shutdown" />
+ </form>
+{% endmacro %}
+
+{% macro cancel_clean_shutdown(cancel_shutdown_url, authz) %}
+  <form method="post" action="{{ cancel_shutdown_url }}" class='command cancel_clean_shutdown'>
+  <p>To cancel a previously initiated shutdown, push the 'Cancel Shutdown' button.</p>
+
+  {% if authz.needAuthForm('gracefulShutdown') %}
+    {{ auth() }}
+  {% endif %}
+
+  <input type="submit" value="Cancel Shutdown" />
+ </form>
+{% endmacro %}
+
 {% macro ping_builder(ping_url, authz) %}
   <form method="post" action="{{ ping_url }}" class='command ping_builder'>
     <p>To ping the buildslave(s), push the 'Ping' button</p>

buildbot/status/web/templates/root.html

@@ -1,4 +1,5 @@
 {% extends 'layout.html' %}
+{% import 'forms.html' as forms %}
 
 {% block content %}
 
@@ -44,6 +45,15 @@ <h1>Welcome to the Buildbot
   <li class="{{ item_class.next() }}"><a href="about">About</a> this Buildbot</li>
 </ul>
 
+{%- if authz.advertiseAction('cleanShutdown') -%}
+{%- if shutting_down -%}
+Master is shutting down<br/>
+{{ forms.cancel_clean_shutdown(cancel_shutdown_url, authz) }}
+{%- else -%}
+{{ forms.clean_shutdown(shutdown_url, authz) }}
+{%- endif -%}
+{%- endif -%}
+
 <p><i>This and other pages can be overridden and customized.</i></p>
 
 </div>

buildbot/test/unit/test_master_cleanshutdown.py

@@ -0,0 +1,143 @@
+# Test clean shutdown functionality of the master
+from mock import Mock, patch, patch_object
+from twisted.trial import unittest
+from twisted.internet import defer
+from buildbot.master import BotMaster
+
+class TestCleanShutdown(unittest.TestCase):
+    def setUp(self):
+        self.master = BotMaster()
+        self.master.reactor = Mock()
+        self.master.startService()
+
+    def test_shutdown_idle(self):
+        """Test that the master shuts down when it's idle"""
+        d = self.master.cleanShutdown()
+        def _check(ign):
+            self.assertEquals(self.master.reactor.stop.called, True)
+
+        d.addCallback(_check)
+        return d
+
+    def test_shutdown_busy(self):
+        """Test that the master shuts down after builds finish"""
+        # Fake some builds
+        builder = Mock()
+        build = Mock()
+        builder.builder_status.getCurrentBuilds.return_value = [build]
+
+        d_finished = defer.Deferred()
+        build.waitUntilFinished.return_value = d_finished
+
+        self.master.builders = Mock()
+        self.master.builders.values.return_value = [builder]
+
+        d_shutdown = self.master.cleanShutdown()
+
+        # Trigger the loop to get things going
+        self.master.loop.trigger()
+
+        # First we wait for it to quiet down again
+        d = self.master.loop.when_quiet()
+
+        # Next we check that we haven't stopped yet, since there's a running
+        # build
+        def _check1(ign):
+            self.assertEquals(self.master.reactor.stop.called, False)
+        d.addCallback(_check1)
+
+        # Now we cause the build to finish, then kick the loop again,
+        # empty out the list of running builds, and wait for the shutdown
+        # process to finish
+        def _finish_build(ign):
+            d_finished.callback(None)
+            self.master.loop.trigger()
+            self.master.builders.values.return_value = []
+            return d_shutdown
+        d.addCallback(_finish_build)
+
+        # And now we should be done
+        def _check2(ign):
+            self.assertEquals(self.master.reactor.stop.called, True)
+        d.addCallback(_check2)
+
+        return d
+
+    def test_shutdown_cancel(self):
+        """Test that we can cancel a shutdown"""
+        # Fake some builds
+        builder = Mock()
+        build = Mock()
+        builder.builder_status.getCurrentBuilds.return_value = [build]
+
+        d_finished = defer.Deferred()
+        build.waitUntilFinished.return_value = d_finished
+
+        self.master.builders = Mock()
+        self.master.builders.values.return_value = [builder]
+
+        d_shutdown = self.master.cleanShutdown()
+
+        # Trigger the loop to get things going
+        self.master.loop.trigger()
+
+        # First we wait for it to quiet down again
+        d = self.master.loop.when_quiet()
+
+        # Next we check that we haven't stopped yet, since there's a running
+        # build.
+        # We cancel the shutdown here too
+        def _check1(ign):
+            self.assertEquals(self.master.reactor.stop.called, False)
+            self.master.cancelCleanShutdown()
+        d.addCallback(_check1)
+
+        # Now we cause the build to finish, then kick the loop again,
+        # empty out the list of running builds, and wait for the shutdown
+        # process to finish
+        def _finish_build(ign):
+            d_finished.callback(None)
+            self.master.loop.trigger()
+            self.master.builders.values.return_value = []
+            return d_shutdown
+        d.addCallback(_finish_build)
+
+        # We should still be running!
+        def _check2(ign):
+            self.assertEquals(self.master.reactor.stop.called, False)
+        d.addCallback(_check2)
+
+        return d
+
+    def test_shutdown_no_new_builds(self):
+        """Test that no new builds get handed out when we're shutting down"""
+        # Fake some builds
+        builder = Mock()
+        build = Mock()
+        builder.builder_status.getCurrentBuilds.return_value = [build]
+
+        d_finished = defer.Deferred()
+        build.waitUntilFinished.return_value = d_finished
+
+        self.master.builders = Mock()
+        self.master.builders.values.return_value = [builder]
+
+        self.assertEquals(self.master._get_processors(), [builder.run])
+
+        d_shutdown = self.master.cleanShutdown()
+
+        # Trigger the loop to get things going
+        self.master.loop.trigger()
+
+        # First we wait for it to quiet down again
+        d = self.master.loop.when_quiet()
+
+        # Next we check that we haven't stopped yet, since there's a running
+        # build.
+        # Also check that we're not trying to hand out new builds!
+        def _check1(ign):
+            self.assertEquals(self.master.reactor.stop.called, False)
+            self.assertEquals(self.master._get_processors(), [])
+        d.addCallback(_check1)
+
+        return d