Merge branch 'master' into nine

Conflicts:
	master/buildbot/db/connector.py
	master/buildbot/db/schedulers.py
	master/buildbot/master.py
	master/buildbot/process/buildrequestdistributor.py
	master/buildbot/schedulers/base.py
	master/buildbot/schedulers/basic.py
	master/buildbot/schedulers/forcesched.py
	master/buildbot/test/fake/fakedb.py
	master/buildbot/test/integration/test_slave_comm.py
	master/buildbot/test/unit/test_process_builder.py
	master/buildbot/test/unit/test_process_buildrequestdistributor_BuildRequestDistributor.py

common/generate_buildbot_api_documentation.sh

@@ -19,4 +19,3 @@ tar -xzf buildbot-slave-0.8.7.tar.gz
 git clone https://github.com/buildbot/buildbot.git
 cd buildbot/apidocs
 PYTHONPATH="/tmp/epydoc-3.0.1:/tmp/buildbot-0.8.7:/tmp/buildbot-slave-0.8.7${PYTHONPATH:+:}${PYTHONPATH}" make
-

master/buildbot/buildslave/__init__.py

@@ -172,6 +172,39 @@ def _lockReleased(self):
             return # oh well..
         self.botmaster.maybeStartBuildsForSlave(self.slavename)
 
+    def _applySlaveInfo(self, info):
+        if not info:
+            return
+
+        self.slave_status.setAdmin(info.get("admin"))
+        self.slave_status.setHost(info.get("host"))
+        self.slave_status.setAccessURI(info.get("access_uri"))
+        self.slave_status.setVersion(info.get("version"))
+
+    def _saveSlaveInfoDict(self):
+        slaveinfo = {
+            'admin': self.slave_status.getAdmin(),
+            'host': self.slave_status.getHost(),
+            'access_uri': self.slave_status.getAccessURI(),
+            'version': self.slave_status.getVersion(),
+        }
+        return self.master.db.buildslaves.updateBuildslave(
+            name=self.slavename,
+            slaveinfo=slaveinfo,
+        )
+
+    def _getSlaveInfo(self):
+        d = self.master.db.buildslaves.getBuildslaveByName(self.slavename)
+
+        @d.addCallback
+        def applyInfo(buildslave):
+            if buildslave is None:
+                return
+
+            self._applySlaveInfo(buildslave.get('slaveinfo'))
+
+        return d
+
     def setServiceParent(self, parent):
         # botmaster needs to set before setServiceParent which calls startService
         self.botmaster = parent
@@ -181,7 +214,9 @@ def setServiceParent(self, parent):
     def startService(self):
         self.updateLocks()
         self.startMissingTimer()
-        return service.MultiService.startService(self)
+        d = self._getSlaveInfo()
+        d.addCallback(lambda _: service.MultiService.startService(self))
+        return d
 
     @defer.inlineCallbacks
     def reconfigService(self, new_config):
@@ -372,12 +407,14 @@ def attached(self, bot):
         self.slave_status.addGracefulWatcher(self._gracefulChanged)
 
         d = defer.succeed(None)
+
+        @d.addCallback
         def _log_attachment_on_slave(res):
             d1 = bot.callRemote("print", "attached")
             d1.addErrback(lambda why: None)
             return d1
-        d.addCallback(_log_attachment_on_slave)
 
+        @d.addCallback
         def _get_info(res):
             d1 = bot.callRemote("getSlaveInfo")
             def _got_info(info):
@@ -396,10 +433,11 @@ def _info_unavailable(why):
                 log.err(why)
             d1.addCallbacks(_got_info, _info_unavailable)
             return d1
-        d.addCallback(_get_info)
-        self.startKeepaliveTimer()
 
-        def _get_version(res):
+        d.addCallback(lambda _: self.startKeepaliveTimer())
+
+        @d.addCallback
+        def _get_version(_):
             d = bot.callRemote("getVersion")
             def _got_version(version):
                 state["version"] = version
@@ -409,9 +447,9 @@ def _version_unavailable(why):
                 state["version"] = '(unknown)'
             d.addCallbacks(_got_version, _version_unavailable)
             return d
-        d.addCallback(_get_version)
 
-        def _get_commands(res):
+        @d.addCallback
+        def _get_commands(_):
             d1 = bot.callRemote("getCommands")
             def _got_commands(commands):
                 state["slave_commands"] = commands
@@ -423,14 +461,13 @@ def _commands_unavailable(why):
                 log.err(why)
             d1.addCallbacks(_got_commands, _commands_unavailable)
             return d1
-        d.addCallback(_get_commands)
 
+        @d.addCallback
         def _accept_slave(res):
-            self.slave_status.setAdmin(state.get("admin"))
-            self.slave_status.setHost(state.get("host"))
-            self.slave_status.setAccessURI(state.get("access_uri"))
-            self.slave_status.setVersion(state.get("version"))
             self.slave_status.setConnected(True)
+
+            self._applySlaveInfo(state)
+
             self.slave_commands = state.get("slave_commands")
             self.slave_environ = state.get("slave_environ")
             self.slave_basedir = state.get("slave_basedir")
@@ -447,8 +484,10 @@ def _accept_slave(res):
             self.stopMissingTimer()
             self.master.status.slaveConnected(self.slavename)
 
-            return self.updateSlave()
-        d.addCallback(_accept_slave)
+        d.addCallback(lambda _: self._saveSlaveInfoDict())
+
+        d.addCallback(lambda _: self.updateSlave())
+
         d.addCallback(lambda _:
                 self.botmaster.maybeStartBuildsForSlave(self.slavename))
 
@@ -974,7 +1013,7 @@ def _soft_disconnect(self, fast=False):
 
     def disconnect(self):
         # This returns a Deferred but we don't use it
-        self._soft_disconnect() 
+        self._soft_disconnect()
         # this removes the slave from all builders.  It won't come back
         # without a restart (or maybe a sighup)
         self.botmaster.slaveLost(self)

master/buildbot/db/buildslaves.py

@@ -0,0 +1,84 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+from buildbot.db import base
+
+class BuildslavesConnectorComponent(base.DBConnectorComponent):
+    # Documentation is in developer/database.rst
+
+    def getBuildslaves(self):
+        def thd(conn):
+            tbl = self.db.model.buildslaves
+            rows = conn.execute(tbl.select()).fetchall()
+
+            dicts = []
+            if rows:
+                for row in rows:
+                    dicts.append({
+                        'slaveid': row.id, 
+                        'name': row.name
+                    })
+            return dicts
+        d = self.db.pool.do(thd)
+        return d
+
+    def getBuildslaveByName(self, name):
+        def thd(conn):
+            tbl = self.db.model.buildslaves
+            res = conn.execute(tbl.select(whereclause=(tbl.c.name == name)))
+            row = res.fetchone()
+
+            rv = None
+            if row:
+                rv = self._bdictFromRow(row)
+            res.close()
+            return rv
+        return self.db.pool.do(thd)
+
+    def updateBuildslave(self, name, slaveinfo, _race_hook=None):
+        def thd(conn):
+            transaction = conn.begin()
+
+            tbl = self.db.model.buildslaves
+
+            # first try update, then try insert
+            q = tbl.update(whereclause=(tbl.c.name == name))
+            res = conn.execute(q, info=slaveinfo)
+
+            if res.rowcount == 0:
+                _race_hook and _race_hook(conn)
+
+                # the update hit 0 rows, so try inserting a new one
+                try:
+                    q = tbl.insert()
+                    res = conn.execute(q,
+                            name=name,
+                            info=slaveinfo)
+                except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+                    # someone else beat us to the punch inserting this row;
+                    # let them win.
+                    transaction.rollback()
+                    return
+
+            transaction.commit()
+        return self.db.pool.do(thd)
+
+    def _bdictFromRow(self, row):
+        return {
+            'slaveid': row.id, 
+            'name': row.name,
+            'slaveinfo': row.info
+        }

master/buildbot/db/connector.py

@@ -21,7 +21,7 @@
 from buildbot.db import enginestrategy, exceptions
 from buildbot.db import pool, model, changes, schedulers, sourcestamps
 from buildbot.db import state, buildsets, buildrequests
-from buildbot.db import builds, users, masters, builders
+from buildbot.db import builds, buildslaves, users, masters, builders
 
 upgrade_message = textwrap.dedent("""\
 
@@ -67,6 +67,7 @@ def __init__(self, master, basedir):
         self.buildrequests = buildrequests.BuildRequestsConnectorComponent(self)
         self.state = state.StateConnectorComponent(self)
         self.builds = builds.BuildsConnectorComponent(self)
+        self.buildslaves = buildslaves.BuildslavesConnectorComponent(self)
         self.users = users.UsersConnectorComponent(self)
         self.masters = masters.MastersConnectorComponent(self)
         self.builders = builders.BuildersConnectorComponent(self)

master/buildbot/db/migrate/versions/024_add_buildslaves_table.py

@@ -0,0 +1,33 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+from buildbot.db.types.json import JsonObject
+
+def upgrade(migrate_engine):
+
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    buildslaves = sa.Table("buildslaves", metadata,
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("name", sa.String(256), nullable=False),
+        sa.Column("info", JsonObject, nullable=False),
+    )
+    buildslaves.create()
+
+    idx = sa.Index('buildslaves_name', buildslaves.c.name, unique=True)
+    idx.create()
+

master/buildbot/db/model.py

@@ -19,6 +19,7 @@
 import migrate.versioning.repository
 from twisted.python import util, log
 from buildbot.db import base
+from buildbot.db.types.json import JsonObject
 
 try:
     from migrate.versioning import exceptions
@@ -207,6 +208,13 @@ class Model(base.DBConnectorComponent):
             nullable=False),
     )
 
+    # buildslaves
+    buildslaves = sa.Table("buildslaves", metadata,
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("name", sa.String(256), nullable=False),
+        sa.Column("info", JsonObject, nullable=False),
+    )
+
     # changes
 
     # Files touched in changes
@@ -503,6 +511,7 @@ class Model(base.DBConnectorComponent):
     sa.Index('buildsets_submitted_at', buildsets.c.submitted_at)
     sa.Index('buildset_properties_buildsetid',
             buildset_properties.c.buildsetid)
+    sa.Index('buildslaves_name', buildslaves.c.name, unique=True)
     sa.Index('changes_branch', changes.c.branch)
     sa.Index('changes_revision', changes.c.revision)
     sa.Index('changes_author', changes.c.author)

master/buildbot/db/schedulers.py

@@ -71,9 +71,9 @@ def thd(conn):
             ch_tbl = self.db.model.changes
 
             wc = (sch_ch_tbl.c.schedulerid == schedulerid)
-                        
+
             # may need to filter further based on branch, etc
-            extra_wheres = [] 
+            extra_wheres = []
             if branch != -1:
                 extra_wheres.append(ch_tbl.c.branch == branch)
             if repository != -1:

master/buildbot/db/types/json.py

@@ -0,0 +1,35 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from buildbot.util import json
+from sqlalchemy.types import TypeDecorator, Text
+
+class JsonObject(TypeDecorator):
+    """Represents an immutable json-encoded string."""
+
+    impl = Text
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            value = json.dumps(value)
+
+        return value
+
+    def process_result_value(self, value, dialect):
+        if value is not None:
+            value = json.loads(value)
+        else:
+            value = {}
+        return value

master/buildbot/master.py

@@ -189,7 +189,7 @@ def startService(self, _reactor=reactor):
             try:
                 self.config = config.MasterConfig.loadConfig(self.basedir,
                                                         self.configFileName)
-                
+
             except config.ConfigErrors, e:
                 log.msg("Configuration Errors:")
                 for msg in e.errors:

master/buildbot/monkeypatches/__init__.py

@@ -67,15 +67,7 @@ def patch_gatherResults():
         from buildbot.monkeypatches import gatherResults
         gatherResults.patch()
 
-
 def patch_all(for_tests=False):
-    patch_bug4881()
-    patch_bug4520()
-    patch_bug5079()
-    patch_sqlalchemy2364()
-    patch_sqlalchemy2189()
-    patch_gatherResults()
-
     if for_tests:
         from buildbot.monkeypatches import servicechecks
         servicechecks.patch_servicechecks()
@@ -85,3 +77,10 @@ def patch_all(for_tests=False):
         decorators.patch()
         from buildbot.monkeypatches import testcase_synctest
         testcase_synctest.patch_testcase_synctest()
+
+    patch_bug4881()
+    patch_bug4520()
+    patch_bug5079()
+    patch_sqlalchemy2364()
+    patch_sqlalchemy2189()
+    patch_gatherResults()

master/buildbot/process/botmaster.py

@@ -334,7 +334,7 @@ def maybeStartBuildsForSlave(self, slave_name):
 
     def maybeStartBuildsForAllBuilders(self):
         """
-        Call this when something suggests that this would be a good time to 
+        Call this when something suggests that this would be a good time to
         start some builds, but nothing more specific.
         """
         self.brd.maybeStartBuildsOn(self.builderNames)