Merge pull request #1090 from tardyp/nine_demo_python

Nine ui python

master/buildbot/data/buildrequests.py

@@ -64,6 +64,11 @@ def get(self, resultSpec, kwargs):
             defer.returnValue((yield self.db2data(buildrequest)))
         defer.returnValue(None)
 
+    def startConsuming(self, callback, options, kwargs):
+        buildrequestid = kwargs.get('buildrequestid')
+        return self.master.mq.startConsuming(callback,
+                                             ('buildrequests', buildrequestid, None))
+
 
 class BuildRequestsEndpoint(Db2DataMixin, base.Endpoint):
 
@@ -133,6 +138,9 @@ class BuildRequest(base.ResourceType):
     plural = "buildrequests"
     endpoints = [BuildRequestEndpoint, BuildRequestsEndpoint]
     keyFields = ['buildsetid', 'builderid', 'buildrequestid']
+    eventPathPatterns = """
+        /buildrequests/:buildrequestid
+    """
 
     class EntityType(types.Entity):
         buildrequestid = types.Integer()
@@ -152,6 +160,13 @@ class EntityType(types.Entity):
         link = types.Link()
     entityType = EntityType(name)
 
+    @defer.inlineCallbacks
+    def generateEvent(self, brids, event):
+        for _id in brids:
+            # get the build and munge the result for the notification
+            br = yield self.master.data.get(('buildrequests', str(_id)))
+            self.produceEvent(br, event)
+
     @defer.inlineCallbacks
     def callDbBuildRequests(self, brids, db_callable, **kw):
         if not brids:
@@ -163,6 +178,7 @@ def callDbBuildRequests(self, brids, db_callable, **kw):
             # the db layer returned an AlreadyClaimedError exception, usually
             # because one of the buildrequests has already been claimed by another master
             defer.returnValue(False)
+        yield self.generateEvent(brids, "update")
         defer.returnValue(True)
 
     @base.updateMethod
@@ -183,6 +199,7 @@ def reclaimBuildRequests(self, brids, _reactor=reactor):
     def unclaimBuildRequests(self, brids):
         if brids:
             yield self.master.db.buildrequests.unclaimBuildRequests(brids)
+            yield self.generateEvent(brids, "update")
 
     @base.updateMethod
     @defer.inlineCallbacks
@@ -201,6 +218,7 @@ def completeBuildRequests(self, brids, results, complete_at=None,
             # the db layer returned a NotClaimedError exception, usually
             # because one of the buildrequests has been claimed by another master
             defer.returnValue(False)
+        yield self.generateEvent(brids, "update")
         defer.returnValue(True)
 
     @base.updateMethod

master/buildbot/data/buildsets.py

@@ -64,10 +64,11 @@ class BuildsetEndpoint(Db2DataMixin, base.Endpoint):
         /buildsets/n:bsid
     """
 
+    @defer.inlineCallbacks
     def get(self, resultSpec, kwargs):
-        d = self.master.db.buildsets.getBuildset(kwargs['bsid'])
-        d.addCallback(self.db2data)
-        return d
+        res = yield self.master.db.buildsets.getBuildset(kwargs['bsid'])
+        res = yield self.db2data(res)
+        defer.returnValue(res)
 
     def startConsuming(self, callback, options, kwargs):
         return self.master.mq.startConsuming(callback,

master/buildbot/data/connector.py

@@ -52,6 +52,7 @@ class DataConnector(service.AsyncService):
         'buildbot.data.schedulers',
         'buildbot.data.forceschedulers',
         'buildbot.data.root',
+        'buildbot.data.properties',
     ]
 
     def __init__(self, master):

master/buildbot/data/masters.py

@@ -140,6 +140,8 @@ def _masterDeactivated(self, masterid, name):
             masterid=masterid)
         yield self.master.data.rtypes.changesource._masterDeactivated(
             masterid=masterid)
+
+        yield self.master.doMasterHouseKeeping(masterid=masterid)
         self.produceEvent(
             dict(masterid=masterid, name=name, active=False),
             'stopped')

master/buildbot/data/properties.py

@@ -0,0 +1,51 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from buildbot.data import base
+from buildbot.data import types
+
+
+class BuildsetPropertiesEndpoint(base.Endpoint):
+
+    isCollection = False
+    pathPatterns = """
+        /buildsets/n:bsid/properties
+    """
+
+    def get(self, resultSpec, kwargs):
+        return self.master.db.buildsets.getBuildsetProperties(kwargs['bsid'])
+
+
+class BuildPropertiesEndpoint(base.Endpoint):
+
+    isCollection = False
+    pathPatterns = """
+        /builds/n:buildid/properties
+    """
+
+    def get(self, resultSpec, kwargs):
+        # its not really implemented db.getBuildProperties is TBD
+        # this code is kept here as a placeholder (so its not yet documented)
+        return self.master.db.builds.getBuildProperties(kwargs['buildid'])
+
+
+class Properties(base.ResourceType):
+
+    name = "property"
+    plural = "properties"
+    endpoints = [BuildsetPropertiesEndpoint, BuildPropertiesEndpoint]
+    keyFields = []
+
+    entityType = types.SourcedProperties()

master/buildbot/data/steps.py

@@ -121,7 +121,11 @@ class EntityType(types.Entity):
         complete_at = types.NoneOk(types.DateTime())
         results = types.NoneOk(types.Integer())
         state_strings = types.List(of=types.String())
-        urls = types.List(of=types.String())
+        urls = types.List(
+            of=types.Dict(
+                name=types.String(),
+                url=types.String()
+            ))
         link = types.Link()
     entityType = EntityType(name)
 
@@ -151,6 +155,13 @@ def setStepStateStrings(self, stepid, state_strings):
             stepid=stepid, state_strings=state_strings)
         yield self.generateEvent(stepid, 'updated')
 
+    @base.updateMethod
+    @defer.inlineCallbacks
+    def addStepURL(self, stepid, name, url):
+        yield self.master.db.steps.addURL(
+            stepid=stepid, name=name, url=url)
+        yield self.generateEvent(stepid, 'updated')
+
     @base.updateMethod
     @defer.inlineCallbacks
     def finishStep(self, stepid, results):

master/buildbot/db/__init__.py

@@ -12,3 +12,7 @@
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Copyright Buildbot Team Members
+
+# a NULL constant to use in sqlalchemy whereclauses e.g. (tbl.c.results == NULL)
+# so that pep8 is happy
+NULL = None

master/buildbot/db/buildrequests.py

@@ -16,6 +16,7 @@
 import itertools
 import sqlalchemy as sa
 
+from buildbot.db import NULL
 from buildbot.db import base
 from buildbot.util import datetime2epoch
 from buildbot.util import epoch2datetime
@@ -82,11 +83,11 @@ def thd(conn):
                 if isinstance(claimed, bool):
                     if not claimed:
                         q = q.where(
-                            (claims_tbl.c.claimed_at == None) &
+                            (claims_tbl.c.claimed_at == NULL) &
                             (reqs_tbl.c.complete == 0))
                     else:
                         q = q.where(
-                            (claims_tbl.c.claimed_at != None))
+                            (claims_tbl.c.claimed_at != NULL))
                 else:
                     q = q.where(
                         (claims_tbl.c.masterid == claimed))

master/buildbot/db/builds.py

@@ -15,6 +15,7 @@
 
 import sqlalchemy as sa
 
+from buildbot.db import NULL
 from buildbot.db import base
 from buildbot.util import epoch2datetime
 from buildbot.util import json
@@ -105,6 +106,18 @@ def thd(conn):
                          results=results)
         return self.db.pool.do(thd)
 
+    def finishBuildsFromMaster(self, masterid, results, _reactor=reactor):
+        def thd(conn):
+            tbl = self.db.model.builds
+            q = tbl.update()
+            q = q.where(tbl.c.masterid == masterid)
+            q = q.where(tbl.c.results == NULL)
+
+            conn.execute(q,
+                         complete_at=_reactor.seconds(),
+                         results=results)
+        return self.db.pool.do(thd)
+
     def _builddictFromRow(self, row):
         def mkdt(epoch):
             if epoch:

master/buildbot/db/buildsets.py

@@ -19,6 +19,7 @@
 
 import sqlalchemy as sa
 
+from buildbot.db import NULL
 from buildbot.db import base
 from buildbot.util import datetime2epoch
 from buildbot.util import epoch2datetime
@@ -127,7 +128,7 @@ def thd(conn):
 
             q = tbl.update(whereclause=(
                 (tbl.c.id == bsid) &
-                ((tbl.c.complete == None) | (tbl.c.complete != 1))))
+                ((tbl.c.complete == NULL) | (tbl.c.complete != 1))))
             res = conn.execute(q,
                                complete=1,
                                results=results,
@@ -157,7 +158,7 @@ def thd(conn):
                     q = q.where(bs_tbl.c.complete != 0)
                 else:
                     q = q.where((bs_tbl.c.complete == 0) |
-                                (bs_tbl.c.complete == None))
+                                (bs_tbl.c.complete == NULL))
             res = conn.execute(q)
             return [self._thd_row2dict(conn, row) for row in res.fetchall()]
         return self.db.pool.do(thd)
@@ -180,7 +181,7 @@ def thd(conn):
                     q = q.where(bs_tbl.c.complete != 0)
                 else:
                     q = q.where((bs_tbl.c.complete == 0) |
-                                (bs_tbl.c.complete == None))
+                                (bs_tbl.c.complete == NULL))
             if branch:
                 q = q.where(ss_tbl.c.branch == branch)
             if repository:

master/buildbot/db/changesources.py

@@ -15,6 +15,7 @@
 
 import sqlalchemy as sa
 
+from buildbot.db import NULL
 from buildbot.db import base
 from twisted.internet import defer
 
@@ -86,9 +87,9 @@ def thd(conn):
                 if masterid is not None:
                     wc = (cs_mst_tbl.c.masterid == masterid)
                 elif active:
-                    wc = (cs_mst_tbl.c.masterid != None)
+                    wc = (cs_mst_tbl.c.masterid != NULL)
                 elif active is not None:
-                    wc = (cs_mst_tbl.c.masterid == None)
+                    wc = (cs_mst_tbl.c.masterid == NULL)
 
             q = sa.select([cs_tbl.c.id, cs_tbl.c.name,
                            cs_mst_tbl.c.masterid],

master/buildbot/db/migrate/versions/011_add_buildrequest_claims.py

@@ -16,6 +16,7 @@
 import migrate
 import sqlalchemy as sa
 
+from buildbot.db import NULL
 from buildbot.util import sautils
 
 
@@ -32,7 +33,7 @@ def migrate_claims(migrate_engine, metadata, buildrequests, objects,
         buildrequests.c.claimed_by_name.label("name"),
         sa.literal_column("'BuildMaster'").label("class_name"),
     ],
-        whereclause=buildrequests.c.claimed_by_name != None,
+        whereclause=buildrequests.c.claimed_by_name != NULL,
         distinct=True)
 
     # this doesn't seem to work without str() -- verified in sqla 0.6.0 - 0.7.1
@@ -50,7 +51,7 @@ def migrate_claims(migrate_engine, metadata, buildrequests, objects,
         objects.c.id.label('objectid'),
         buildrequests.c.claimed_at,
     ], from_obj=[join],
-        whereclause=buildrequests.c.claimed_by_name != None)
+        whereclause=buildrequests.c.claimed_by_name != NULL)
     migrate_engine.execute(
         str(sautils.InsertFromSelect(buildrequest_claims, claims)))
 

master/buildbot/db/schedulers.py

@@ -16,6 +16,7 @@
 import sqlalchemy as sa
 import sqlalchemy.exc
 
+from buildbot.db import NULL
 from buildbot.db import base
 from twisted.internet import defer
 
@@ -160,9 +161,9 @@ def thd(conn):
                 if masterid is not None:
                     wc = (sch_mst_tbl.c.masterid == masterid)
                 elif active:
-                    wc = (sch_mst_tbl.c.masterid != None)
+                    wc = (sch_mst_tbl.c.masterid != NULL)
                 elif active is not None:
-                    wc = (sch_mst_tbl.c.masterid == None)
+                    wc = (sch_mst_tbl.c.masterid == NULL)
 
             q = sa.select([sch_tbl.c.id, sch_tbl.c.name,
                            sch_mst_tbl.c.masterid],

master/buildbot/db/steps.py

@@ -24,14 +24,15 @@
 
 class StepsConnectorComponent(base.DBConnectorComponent):
     # Documentation is in developer/db.rst
+    url_lock = None
 
     def getStep(self, stepid=None, buildid=None, number=None, name=None):
+        tbl = self.db.model.steps
         if stepid is not None:
-            wc = self.db.model.steps.c.id == stepid
+            wc = (tbl.c.id == stepid)
         else:
             if buildid is None:
                 return defer.fail(RuntimeError('must supply either stepid or buildid'))
-            tbl = self.db.model.steps
             if number is not None:
                 wc = (tbl.c.number == number)
             elif name is not None:
@@ -124,6 +125,36 @@ def thd(conn):
             conn.execute(q, state_strings_json=json.dumps(state_strings))
         return self.db.pool.do(thd)
 
+    def addURL(self, stepid, name, url, _racehook=None):
+        # This methods adds an URL to the db
+        # This is a read modify write and thus there is a possibility
+        # that several urls are added at the same time (e.g with a deferredlist
+        # at the end of a step)
+        # this race condition is only inside the same master, as only one master
+        # is supposed to add urls to a buildstep.
+        # so threading.lock is used, as we are in the thread pool
+        if self.url_lock is None:
+            # this runs in reactor thread, so no race here..
+            self.url_lock = defer.DeferredLock()
+
+        def thd(conn):
+
+            tbl = self.db.model.steps
+            wc = (tbl.c.id == stepid)
+            q = sa.select([tbl.c.urls_json],
+                          whereclause=wc)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if _racehook is not None:
+                _racehook()
+            urls = json.loads(row.urls_json)
+            urls.append(dict(name=name, url=url))
+
+            q = tbl.update(whereclause=wc)
+            conn.execute(q, urls_json=json.dumps(urls))
+
+        return self.url_lock.run(lambda: self.db.pool.do(thd))
+
     def finishStep(self, stepid, results, _reactor=reactor):
         def thd(conn):
             tbl = self.db.model.steps