Merge delanne/buildbot into nine:9/Use_DB_API_for_build_requests (PR #…

…1043)

+import reorder

master/buildbot/data/buildrequests.py

@@ -16,7 +16,6 @@
 from buildbot.data import base
 from buildbot.data import types
 from buildbot.db.buildrequests import AlreadyClaimedError
-from buildbot.util import datetime2epoch
 
 from buildbot.db.buildrequests import NotClaimedError
 from twisted.internet import defer
@@ -27,20 +26,21 @@ class Db2DataMixin(object):
 
     def db2data(self, dbdict):
         data = {
-            'buildrequestid': dbdict['brid'],
+            'buildrequestid': dbdict['buildrequestid'],
             'buildsetid': dbdict['buildsetid'],
             'buildset_link': base.Link(('buildset', str(dbdict['buildsetid']))),
             'builderid': dbdict['builderid'],
+            'buildername': dbdict['buildername'],
             'priority': dbdict['priority'],
             'claimed': dbdict['claimed'],
-            'claimed_at': datetime2epoch(dbdict['claimed_at']),
+            'claimed_at': dbdict['claimed_at'],
             'claimed_by_masterid': dbdict['claimed_by_masterid'],
             'complete': dbdict['complete'],
             'results': dbdict['results'],
-            'submitted_at': datetime2epoch(dbdict['submitted_at']),
-            'complete_at': datetime2epoch(dbdict['complete_at']),
+            'submitted_at': dbdict['submitted_at'],
+            'complete_at': dbdict['complete_at'],
             'waited_for': dbdict['waited_for'],
-            'link': base.Link(('buildrequest', str(dbdict['brid']))),
+            'link': base.Link(('buildrequest', str(dbdict['buildrequestid']))),
         }
         return defer.succeed(data)
 
@@ -90,6 +90,7 @@ def get(self, resultSpec, kwargs):
                 defer.returnValue([])
         else:
             buildername = None
+
         complete = resultSpec.popBooleanFilter('complete')
         claimed_by_masterid = resultSpec.popBooleanFilter('claimed_by_masterid')
         if claimed_by_masterid:
@@ -99,7 +100,8 @@ def get(self, resultSpec, kwargs):
             claimed = claimed_by_masterid
         else:
             claimed = resultSpec.popBooleanFilter('claimed')
-        bsid = resultSpec.popBooleanFilter('bsid')
+
+        bsid = resultSpec.popBooleanFilter('buildsetid')
         branch = resultSpec.popBooleanFilter('branch')
         repository = resultSpec.popBooleanFilter('repository')
         buildrequests = yield self.master.db.buildrequests.getBuildRequests(
@@ -137,14 +139,15 @@ class EntityType(types.Entity):
         buildsetid = types.Integer()
         buildset_link = types.Link()
         builderid = types.Integer()
+        buildername = types.Identifier(20)
         priority = types.Integer()
         claimed = types.Boolean()
-        claimed_at = types.NoneOk(types.Integer())
+        claimed_at = types.NoneOk(types.DateTime())
         claimed_by_masterid = types.NoneOk(types.Integer())
         complete = types.Boolean()
         results = types.NoneOk(types.Integer())
-        submitted_at = types.Integer()
-        complete_at = types.NoneOk(types.Integer())
+        submitted_at = types.DateTime()
+        complete_at = types.NoneOk(types.DateTime())
         waited_for = types.Boolean()
         link = types.Link()
     entityType = EntityType(name)

master/buildbot/data/builds.py

@@ -15,7 +15,6 @@
 
 from buildbot.data import base
 from buildbot.data import types
-from buildbot.util import datetime2epoch
 from twisted.internet import defer
 
 
@@ -34,8 +33,8 @@ def db2data(self, dbdict):
             'slave_link': base.Link(('slave', str(dbdict['buildslaveid']))),
             'masterid': dbdict['masterid'],
             'master_link': base.Link(('master', str(dbdict['masterid']))),
-            'started_at': datetime2epoch(dbdict['started_at']),
-            'complete_at': datetime2epoch(dbdict['complete_at']),
+            'started_at': dbdict['started_at'],
+            'complete_at': dbdict['complete_at'],
             'complete': dbdict['complete_at'] is not None,
             'state_strings': dbdict['state_strings'],
             'results': dbdict['results'],
@@ -131,9 +130,9 @@ class EntityType(types.Entity):
         slave_link = types.Link()
         masterid = types.Integer()
         master_link = types.Link()
-        started_at = types.Integer()
+        started_at = types.DateTime()
         complete = types.Boolean()
-        complete_at = types.NoneOk(types.Integer())
+        complete_at = types.NoneOk(types.DateTime())
         results = types.NoneOk(types.Integer())
         state_strings = types.List(of=types.String())
         link = types.Link()

master/buildbot/data/buildsets.py

@@ -216,8 +216,7 @@ def maybeBuildsetComplete(self, bsid, _reactor=reactor):
             return
 
         # mark it as completed in the database
-        complete_at_epoch = int(_reactor.seconds())
-        complete_at = epoch2datetime(complete_at_epoch)
+        complete_at = epoch2datetime(int(_reactor.seconds()))
         yield self.master.db.buildsets.completeBuildset(bsid,
                                                         cumulative_results, complete_at=complete_at)
 
@@ -238,9 +237,9 @@ def maybeBuildsetComplete(self, bsid, _reactor=reactor):
             external_idstring=bsdict['external_idstring'],
             reason=bsdict['reason'],
             sourcestamps=sourcestamps,
-            submitted_at=datetime2epoch(bsdict['submitted_at']),
+            submitted_at=bsdict['submitted_at'],
             complete=True,
-            complete_at=complete_at_epoch,
+            complete_at=complete_at,
             results=cumulative_results)
             # TODO: properties=properties)
         self.master.mq.produce(('buildset', str(bsid), 'complete'), msg)

master/buildbot/data/masters.py

@@ -15,7 +15,6 @@
 
 from buildbot.data import base
 from buildbot.data import types
-from buildbot.util import datetime2epoch
 from buildbot.util import epoch2datetime
 from twisted.internet import defer
 from twisted.internet import reactor
@@ -29,7 +28,7 @@ def _db2data(master):
     return dict(masterid=master['id'],
                 name=master['name'],
                 active=master['active'],
-                last_active=datetime2epoch(master['last_active']),
+                last_active=master['last_active'],
                 link=base.Link(('master', str(master['id']))))
 
 
@@ -95,7 +94,7 @@ class EntityType(types.Entity):
         masterid = types.Integer()
         name = types.String()
         active = types.Boolean()
-        last_active = types.Integer()
+        last_active = types.DateTime()
         link = types.Link()
     entityType = EntityType(name)
 

master/buildbot/data/sourcestamps.py

@@ -16,7 +16,6 @@
 from buildbot.data import base
 from buildbot.data import patches
 from buildbot.data import types
-from buildbot.util import datetime2epoch
 from twisted.internet import defer
 
 
@@ -28,7 +27,7 @@ def _db2data(ss):
         'project': ss['project'],
         'repository': ss['repository'],
         'codebase': ss['codebase'],
-        'created_at': datetime2epoch(ss['created_at']),
+        'created_at': ss['created_at'],
         'link': base.Link(('sourcestamp', str(ss['ssid']))),
         'patch': None,
     }
@@ -91,6 +90,6 @@ class EntityType(types.Entity):
         project = types.String()
         codebase = types.String()
         patch = types.NoneOk(patches.Patch.entityType)
-        created_at = types.Integer()
+        created_at = types.DateTime()
         link = types.Link()
     entityType = EntityType(name)

master/buildbot/data/steps.py

@@ -15,7 +15,6 @@
 
 from buildbot.data import base
 from buildbot.data import types
-from buildbot.util import datetime2epoch
 from twisted.internet import defer
 
 
@@ -28,9 +27,9 @@ def db2data(self, dbdict):
             'name': dbdict['name'],
             'buildid': dbdict['buildid'],
             'build_link': base.Link(('build', str(dbdict['buildid']))),
-            'started_at': datetime2epoch(dbdict['started_at']),
+            'started_at': dbdict['started_at'],
             'complete': dbdict['complete_at'] is not None,
-            'complete_at': datetime2epoch(dbdict['complete_at']),
+            'complete_at': dbdict['complete_at'],
             'state_strings': dbdict['state_strings'],
             'results': dbdict['results'],
             'urls': dbdict['urls'],
@@ -117,9 +116,9 @@ class EntityType(types.Entity):
         name = types.Identifier(50)
         buildid = types.Integer()
         build_link = types.Link()
-        started_at = types.NoneOk(types.Integer())
+        started_at = types.NoneOk(types.DateTime())
         complete = types.Boolean()
-        complete_at = types.NoneOk(types.Integer())
+        complete_at = types.NoneOk(types.DateTime())
         results = types.NoneOk(types.Integer())
         state_strings = types.List(of=types.String())
         urls = types.List(of=types.String())

master/buildbot/data/types.py

@@ -15,6 +15,7 @@
 
 # See "Type Validation" in master/docs/developer/tests.rst
 
+import datetime
 import re
 
 from buildbot import util
@@ -91,6 +92,11 @@ def valueFromString(self, arg):
         return int(arg)
 
 
+class DateTime(Instance):
+    name = "datetime"
+    types = (datetime.datetime)
+
+
 class String(Instance):
 
     name = "string"

master/buildbot/db/buildrequests.py

@@ -271,7 +271,7 @@ def mkdt(epoch):
         complete_at = mkdt(row.complete_at)
         claimed_at = mkdt(claimed_at)
 
-        return BrDict(brid=row.id, buildsetid=row.buildsetid,
+        return BrDict(buildrequestid=row.id, buildsetid=row.buildsetid,
                       buildername=row.buildername, priority=row.priority,
                       claimed=claimed, claimed_at=claimed_at,
                       claimed_by_masterid=claimed_by_masterid,

master/buildbot/process/builder.py

@@ -18,6 +18,7 @@
 
 from buildbot import config
 from buildbot import interfaces
+from buildbot.data import resultspec
 from buildbot.process import buildrequest
 from buildbot.process import slavebuilder
 from buildbot.process.build import Build
@@ -150,10 +151,8 @@ def getOldestRequestTime(self):
 
         @returns: datetime instance or None, via Deferred
         """
-        # TODO: use data API here
-        unclaimed = yield self.master.db.buildrequests.getBuildRequests(
-            buildername=self.name, claimed=False)
-
+        unclaimed = yield self.master.data.get(('builder', ascii2unicode(self.name), 'buildrequest'),
+                                               [resultspec.Filter('claimed', 'eq', [False])])
         if unclaimed:
             unclaimed = sorted([brd['submitted_at'] for brd in unclaimed])
             defer.returnValue(unclaimed[0])

master/buildbot/process/buildrequest.py

@@ -97,7 +97,7 @@ def fromBrdict(cls, master, brdict):
         @returns: L{BuildRequest}, via Deferred
         """
         cache = master.caches.get_cache("BuildRequests", cls._make_br)
-        return cache.get(brdict['brid'], brdict=brdict, master=master)
+        return cache.get(brdict['buildrequestid'], brdict=brdict, master=master)
 
     @classmethod
     @defer.inlineCallbacks

master/buildbot/process/buildrequestdistributor.py

@@ -14,6 +14,7 @@
 # Copyright Buildbot Team Members
 
 
+from buildbot.data import resultspec
 from buildbot.process import metrics
 from buildbot.process.buildrequest import BuildRequest
 from buildbot.util import ascii2unicode
@@ -89,8 +90,13 @@ def _fetchUnclaimedBrdicts(self):
         # the self.unclaimedBrdicts to None before calling."""
 
         if self.unclaimedBrdicts is None:
-            brdicts = yield self.master.db.buildrequests.getBuildRequests(
-                buildername=self.bldr.name, claimed=False)
+            # TODO: use order of the DATA API
+            brdicts = yield self.master.data.get(('builder',
+                                                  self.bldr.name,
+                                                  'buildrequest'),
+                                                 [resultspec.Filter('claimed',
+                                                                    'eq',
+                                                                    [False])])
             # sort by submitted_at, so the first is the oldest
             brdicts.sort(key=lambda brd: brd['submitted_at'])
             self.unclaimedBrdicts = brdicts
@@ -101,11 +107,11 @@ def _getBuildRequestForBrdict(self, brdict):
         # Turn a brdict into a BuildRequest into a brdict. This is useful
         # for API like 'nextBuild', which operate on BuildRequest objects.
 
-        breq = self.breqCache.get(brdict['brid'])
+        breq = self.breqCache.get(brdict['buildrequestid'])
         if not breq:
             breq = yield BuildRequest.fromBrdict(self.master, brdict)
             if breq:
-                self.breqCache[brdict['brid']] = breq
+                self.breqCache[brdict['buildrequestid']] = breq
         defer.returnValue(breq)
 
     def _getBrdictForBuildRequest(self, breq):
@@ -117,7 +123,7 @@ def _getBrdictForBuildRequest(self, breq):
 
         brid = breq.id
         for brdict in self.unclaimedBrdicts:
-            if brid == brdict['brid']:
+            if brid == brdict['buildrequestid']:
                 return brdict
         return None
 
@@ -524,10 +530,10 @@ def _maybeStartBuildsOnBuilder(self, bldr, _reactor=reactor):
                 brdict = yield self.master.db.buildrequests.getBuildRequest(brid)
                 key = ('buildset', str(brdict['buildsetid']),
                        'builder', str(-1),
-                       'buildrequest', str(brdict['brid']), 'claimed')
+                       'buildrequest', str(brdict['buildrequestid']), 'claimed')
                 msg = dict(
                     bsid=brdict['buildsetid'],
-                    brid=brdict['brid'],
+                    brid=brdict['buildrequestid'],
                     buildername=brdict['buildername'],
                     builderid=-1,
                     # TODO:
@@ -547,7 +553,7 @@ def _maybeStartBuildsOnBuilder(self, bldr, _reactor=reactor):
                     brid = breq.id
                     key = ('buildset', str(brdict['buildsetid']),
                            'builder', str(-1),
-                           'buildrequest', str(brdict['brid']), 'unclaimed')
+                           'buildrequest', str(brdict['buildrequestid']), 'unclaimed')
                     msg = dict(brid=brid, bsid=bsid, buildername=buildername,
                                builderid=-1)
                     self.master.mq.produce(key, msg)

master/buildbot/status/buildset.py

@@ -14,6 +14,7 @@
 # Copyright Buildbot Team Members
 
 from buildbot import interfaces
+from buildbot.data import resultspec
 from buildbot.status.buildrequest import BuildRequestStatus
 from zope.interface import implements
 
@@ -44,7 +45,8 @@ def isFinished(self):
     def getBuilderNamesAndBuildRequests(self):
         # returns a Deferred; undocumented method that may be removed
         # without warning
-        d = self.master.db.buildrequests.getBuildRequests(bsid=self.id)
+        d = self.master.data.get(('buildrequest', ),
+                                 filters=[resultspec.Filter('buildsetid', 'eq', [self.id])])
 
         def get_objects(brdicts):
             return dict([
@@ -55,7 +57,8 @@ def get_objects(brdicts):
         return d
 
     def getBuilderNames(self):
-        d = self.master.db.buildrequests.getBuildRequests(bsid=self.id)
+        d = self.master.data.get(('buildrequest', ),
+                                 filters=[resultspec.Filter('buildsetid', 'eq', [self.id])])
 
         def get_names(brdicts):
             return sorted([brd['buildername'] for brd in brdicts])

master/buildbot/status/mail.py

@@ -25,6 +25,8 @@
 from email.mime.text import MIMEText
 from email.utils import formatdate
 
+from buildbot.data import resultspec
+
 from twisted.internet import defer
 from twisted.internet import reactor
 from twisted.python import log as twlog
@@ -557,15 +559,16 @@ def _gotBuildRequests(self, breqs, buildset):
         for breq in breqs:
             buildername = breq['buildername']
             builder = self.master_status.getBuilder(buildername)
-            d = self.master.db.builds.getBuilds(buildrequestid=breq['brid'])
+            d = self.master.db.builds.getBuilds(buildrequestid=breq['buildrequestid'])
             d.addCallback(lambda builddictlist, builder=builder:
                           (builddictlist, builder))
             dl.append(d)
         d = defer.gatherResults(dl)
         d.addCallback(self._gotBuilds, buildset)
 
     def _gotBuildSet(self, buildset, bsid):
-        d = self.master.db.buildrequests.getBuildRequests(bsid=bsid)
+        d = self.master.data.get(('buildrequest', ),
+                                 filters=[resultspec.Filter('buildsetid', 'eq', [bsid])])
         d.addCallback(self._gotBuildRequests, buildset)
 
     def _buildset_complete_cb(self, key, msg):