Merge branch 'BuildslaveDatabase' of git://github.com/jaredgrubb/buil…

…dbot

master/buildbot/buildslave/__init__.py

@@ -172,6 +172,39 @@ def _lockReleased(self):
             return # oh well..
         self.botmaster.maybeStartBuildsForSlave(self.slavename)
 
+    def _applySlaveInfo(self, info):
+        if not info:
+            return
+
+        self.slave_status.setAdmin(info.get("admin"))
+        self.slave_status.setHost(info.get("host"))
+        self.slave_status.setAccessURI(info.get("access_uri"))
+        self.slave_status.setVersion(info.get("version"))
+
+    def _saveSlaveInfoDict(self):
+        slaveinfo = {
+            'admin': self.slave_status.getAdmin(),
+            'host': self.slave_status.getHost(),
+            'access_uri': self.slave_status.getAccessURI(),
+            'version': self.slave_status.getVersion(),
+        }
+        return self.master.db.buildslaves.updateBuildslave(
+            name=self.slavename,
+            slaveinfo=slaveinfo,
+        )
+
+    def _getSlaveInfo(self):
+        d = self.master.db.buildslaves.getBuildslaveByName(self.slavename)
+
+        @d.addCallback
+        def applyInfo(buildslave):
+            if buildslave is None:
+                return
+
+            self._applySlaveInfo(buildslave.get('slaveinfo'))
+
+        return d
+
     def setServiceParent(self, parent):
         # botmaster needs to set before setServiceParent which calls startService
         self.botmaster = parent
@@ -181,7 +214,9 @@ def setServiceParent(self, parent):
     def startService(self):
         self.updateLocks()
         self.startMissingTimer()
-        return service.MultiService.startService(self)
+        d = self._getSlaveInfo()
+        d.addCallback(lambda _: service.MultiService.startService(self))
+        return d
 
     @defer.inlineCallbacks
     def reconfigService(self, new_config):
@@ -372,12 +407,14 @@ def attached(self, bot):
         self.slave_status.addGracefulWatcher(self._gracefulChanged)
 
         d = defer.succeed(None)
+
+        @d.addCallback
         def _log_attachment_on_slave(res):
             d1 = bot.callRemote("print", "attached")
             d1.addErrback(lambda why: None)
             return d1
-        d.addCallback(_log_attachment_on_slave)
 
+        @d.addCallback
         def _get_info(res):
             d1 = bot.callRemote("getSlaveInfo")
             def _got_info(info):
@@ -396,10 +433,11 @@ def _info_unavailable(why):
                 log.err(why)
             d1.addCallbacks(_got_info, _info_unavailable)
             return d1
-        d.addCallback(_get_info)
-        self.startKeepaliveTimer()
 
-        def _get_version(res):
+        d.addCallback(lambda _: self.startKeepaliveTimer())
+
+        @d.addCallback
+        def _get_version(_):
             d = bot.callRemote("getVersion")
             def _got_version(version):
                 state["version"] = version
@@ -409,9 +447,9 @@ def _version_unavailable(why):
                 state["version"] = '(unknown)'
             d.addCallbacks(_got_version, _version_unavailable)
             return d
-        d.addCallback(_get_version)
 
-        def _get_commands(res):
+        @d.addCallback
+        def _get_commands(_):
             d1 = bot.callRemote("getCommands")
             def _got_commands(commands):
                 state["slave_commands"] = commands
@@ -423,14 +461,13 @@ def _commands_unavailable(why):
                 log.err(why)
             d1.addCallbacks(_got_commands, _commands_unavailable)
             return d1
-        d.addCallback(_get_commands)
 
+        @d.addCallback
         def _accept_slave(res):
-            self.slave_status.setAdmin(state.get("admin"))
-            self.slave_status.setHost(state.get("host"))
-            self.slave_status.setAccessURI(state.get("access_uri"))
-            self.slave_status.setVersion(state.get("version"))
             self.slave_status.setConnected(True)
+
+            self._applySlaveInfo(state)
+
             self.slave_commands = state.get("slave_commands")
             self.slave_environ = state.get("slave_environ")
             self.slave_basedir = state.get("slave_basedir")
@@ -447,8 +484,10 @@ def _accept_slave(res):
             self.stopMissingTimer()
             self.master.status.slaveConnected(self.slavename)
 
-            return self.updateSlave()
-        d.addCallback(_accept_slave)
+        d.addCallback(lambda _: self._saveSlaveInfoDict())
+
+        d.addCallback(lambda _: self.updateSlave())
+
         d.addCallback(lambda _:
                 self.botmaster.maybeStartBuildsForSlave(self.slavename))
 

master/buildbot/db/buildslaves.py

@@ -0,0 +1,84 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+from buildbot.db import base
+
+class BuildslavesConnectorComponent(base.DBConnectorComponent):
+    # Documentation is in developer/database.rst
+
+    def getBuildslaves(self):
+        def thd(conn):
+            tbl = self.db.model.buildslaves
+            rows = conn.execute(tbl.select()).fetchall()
+
+            dicts = []
+            if rows:
+                for row in rows:
+                    dicts.append({
+                        'slaveid': row.id, 
+                        'name': row.name
+                    })
+            return dicts
+        d = self.db.pool.do(thd)
+        return d
+
+    def getBuildslaveByName(self, name):
+        def thd(conn):
+            tbl = self.db.model.buildslaves
+            res = conn.execute(tbl.select(whereclause=(tbl.c.name == name)))
+            row = res.fetchone()
+
+            rv = None
+            if row:
+                rv = self._bdictFromRow(row)
+            res.close()
+            return rv
+        return self.db.pool.do(thd)
+
+    def updateBuildslave(self, name, slaveinfo, _race_hook=None):
+        def thd(conn):
+            transaction = conn.begin()
+
+            tbl = self.db.model.buildslaves
+
+            # first try update, then try insert
+            q = tbl.update(whereclause=(tbl.c.name == name))
+            res = conn.execute(q, info=slaveinfo)
+
+            if res.rowcount == 0:
+                _race_hook and _race_hook(conn)
+
+                # the update hit 0 rows, so try inserting a new one
+                try:
+                    q = tbl.insert()
+                    res = conn.execute(q,
+                            name=name,
+                            info=slaveinfo)
+                except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+                    # someone else beat us to the punch inserting this row;
+                    # let them win.
+                    transaction.rollback()
+                    return
+
+            transaction.commit()
+        return self.db.pool.do(thd)
+
+    def _bdictFromRow(self, row):
+        return {
+            'slaveid': row.id, 
+            'name': row.name,
+            'slaveinfo': row.info
+        }

master/buildbot/db/connector.py

@@ -20,7 +20,7 @@
 from buildbot import config
 from buildbot.db import enginestrategy
 from buildbot.db import pool, model, changes, schedulers, sourcestamps, sourcestampsets
-from buildbot.db import state, buildsets, buildrequests, builds, users
+from buildbot.db import state, buildsets, buildrequests, builds, buildslaves, users
 
 class DatabaseNotReadyError(Exception):
     pass
@@ -70,6 +70,7 @@ def __init__(self, master, basedir):
         self.buildrequests = buildrequests.BuildRequestsConnectorComponent(self)
         self.state = state.StateConnectorComponent(self)
         self.builds = builds.BuildsConnectorComponent(self)
+        self.buildslaves = buildslaves.BuildslavesConnectorComponent(self)
         self.users = users.UsersConnectorComponent(self)
 
         self.cleanup_timer = internet.TimerService(self.CLEANUP_PERIOD,

master/buildbot/db/migrate/versions/024_add_buildslaves_table.py

@@ -0,0 +1,33 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+from buildbot.db.types.json import JsonObject
+
+def upgrade(migrate_engine):
+
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    buildslaves = sa.Table("buildslaves", metadata,
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("name", sa.String(256), nullable=False),
+        sa.Column("info", JsonObject, nullable=False),
+    )
+    buildslaves.create()
+
+    idx = sa.Index('buildslaves_name', buildslaves.c.name)
+    idx.create()
+

master/buildbot/db/model.py

@@ -19,6 +19,7 @@
 import migrate.versioning.repository
 from twisted.python import util, log
 from buildbot.db import base
+from buildbot.db.types.json import JsonObject
 
 try:
     from migrate.versioning import exceptions
@@ -135,6 +136,13 @@ class Model(base.DBConnectorComponent):
             sa.ForeignKey('sourcestampsets.id')),
     )
 
+    # buildslaves
+    buildslaves = sa.Table("buildslaves", metadata,
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("name", sa.String(256), nullable=False),
+        sa.Column("info", JsonObject, nullable=False),
+    )
+
     # changes
 
     # Files touched in changes
@@ -363,6 +371,7 @@ class Model(base.DBConnectorComponent):
     sa.Index('buildsets_submitted_at', buildsets.c.submitted_at)
     sa.Index('buildset_properties_buildsetid',
             buildset_properties.c.buildsetid)
+    sa.Index('buildslaves_name', buildslaves.c.name)
     sa.Index('changes_branch', changes.c.branch)
     sa.Index('changes_revision', changes.c.revision)
     sa.Index('changes_author', changes.c.author)

master/buildbot/db/types/json.py

@@ -0,0 +1,35 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from buildbot.util import json
+from sqlalchemy.types import TypeDecorator, Text
+
+class JsonObject(TypeDecorator):
+    """Represents an immutable json-encoded string."""
+
+    impl = Text
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            value = json.dumps(value)
+
+        return value
+
+    def process_result_value(self, value, dialect):
+        if value is not None:
+            value = json.loads(value)
+        else:
+            value = {}
+        return value

master/buildbot/status/slave.py

@@ -17,6 +17,7 @@
 from zope.interface import implements
 from buildbot import interfaces
 from buildbot.util.eventual import eventually
+from buildbot.util import ascii2unicode
 
 class SlaveStatus:
     implements(interfaces.ISlaveStatus)
@@ -59,9 +60,9 @@ def getConnectCount(self):
         return len([ t for t in self.connect_times if t > then ])
 
     def setAdmin(self, admin):
-        self.admin = admin
+        self.admin = ascii2unicode(admin)
     def setHost(self, host):
-        self.host = host
+        self.host = ascii2unicode(host)
     def setAccessURI(self, access_uri):
         self.access_uri = access_uri
     def setVersion(self, version):

master/buildbot/status/web/builder.py

@@ -323,7 +323,7 @@ def prop_match(oprops):
             s['name'] = slave.getName()
             c = s['connected'] = slave.isConnected()
             s['paused'] = slave.isPaused()
-            s['admin'] = unicode(slave.getAdmin() or '', 'utf-8')
+            s['admin'] = slave.getAdmin() or u''
             if c:
                 connected_slaves += 1
         cxt['connected_slaves'] = connected_slaves

master/buildbot/status/web/slaves.py

@@ -133,12 +133,12 @@ def content(self, request, ctx):
                         pause_url = pause_url,
                         authz = self.getAuthz(request),
                         this_url = "../../../" + path_to_slave(request, slave),
-                        access_uri = slave.getAccessURI()),
-                        admin = unicode(slave.getAdmin() or '', 'utf-8'),
-                        host = unicode(slave.getHost() or '', 'utf-8'),
+                        access_uri = slave.getAccessURI(),
+                        admin = slave.getAdmin() or u'',
+                        host = slave.getHost() or u'',
                         slave_version = slave.getVersion(),
                         show_builder_column = True,
-                        connect_count = connect_count)
+                        connect_count = connect_count))
         template = request.site.buildbot_service.templates.get_template("buildslave.html")
         data = template.render(**ctx)
         return data
@@ -184,7 +184,7 @@ def content(self, request, ctx):
             info['connectCount'] = slave.getConnectCount()
             info['paused'] = slave.isPaused()
 
-            info['admin'] = unicode(slave.getAdmin() or '', 'utf-8')
+            info['admin'] = slave.getAdmin() or u''
             last = slave.lastMessageReceived()
             if last:
                 info['last_heard_from_age'] = abbreviate_age(time.time() - last)