add buildrequests DB connector

master/buildbot/db/buildrequests.py

@@ -0,0 +1,355 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+"""
+Support for buildsets in the database
+"""
+
+import sqlalchemy as sa
+from twisted.internet import reactor
+from buildbot.db import base
+from buildbot.util import epoch2datetime
+
+class AlreadyClaimedError(Exception):
+    pass
+
+class NotClaimedError(Exception):
+    pass
+
+class BuildRequestsConnectorComponent(base.DBConnectorComponent):
+    """
+    A DBConnectorComponent to handle buildrequests.  An instance is available
+    at C{master.db.buildrequests}.
+
+    Build Requests are represented as dictionaries with keys C{brid},
+    C{buildsetid}, C{buildername}, C{priority}, C{claimed} (boolean),
+    C{claimed_at}, C{mine} (boolean), C{complete}, C{results}, C{submitted_at},
+    and C{complete_at}.  The two time parameters (C{*_at}) are presented as
+    datetime objects.
+    """
+
+    def getBuildRequest(self, brid):
+        """
+        Get a single BuildRequest, in the format described above.  Returns
+        C{None} if there is no such buildrquest.
+
+        @param brid: build request id
+        @type brid: integer
+
+        @returns: Build request dictionary as above or None, via Deferred
+        """
+        def thd(conn):
+            tbl = self.db.model.buildrequests
+            res = conn.execute(tbl.select(whereclause=(tbl.c.id == brid)))
+            row = res.fetchone()
+
+            rv = None
+            if row:
+                rv = self._brdictFromRow(row)
+            res.close()
+            return rv
+        return self.db.pool.do(thd)
+
+    def getBuildRequests(self, buildername=None, complete=None, claimed=None):
+        """
+        Get a list of build requests matching the given characteristics.  Note
+        that C{unclaimed}, C{my_claimed}, and C{other_claimed} all default to
+        C{False}, so at least one must be provided or no results will be
+        returned.
+
+        The C{claimed} parameter can be C{None} (the default) to ignore the
+        claimed status of requests; C{True} to return only claimed builds,
+        C{False} to return only unclaimed builds, or C{"mine"} to return only
+        builds claimed by this master instance.  A request is considered
+        unclaimed if its C{claimed_at} column is either NULL or 0.
+
+        A build is considered completed if its C{complete} column is 1; the
+        C{complete_at} column is not consulted.
+
+        The resulting dictionaries may be cached internally, and should not be
+        modified directly.
+
+        @param buildername: limit results to buildrequests for this builder
+        @type buildername: string
+
+        @param complete: if true, limit to completed buildrequests; if false,
+        limit to incomplete buildrequests; if None, do not limit based on
+        completion.
+
+        @param claimed: see above
+
+        @returns: List of build request dictionaries as above, via Deferred
+        """
+        def thd(conn):
+            tbl = self.db.model.buildrequests
+            q = tbl.select()
+            if claimed is not None:
+                if not claimed:
+                    q = q.where(
+                        ((tbl.c.claimed_at == None) |
+                         (tbl.c.claimed_at == 0)) &
+                        (tbl.c.claimed_by_name == None) &
+                        (tbl.c.claimed_by_incarnation == None))
+                elif claimed == "mine":
+                    master_name = self.db.master.master_name
+                    master_incarnation = self.db.master.master_incarnation
+                    q = q.where(
+                        (tbl.c.claimed_at != None) &
+                        (tbl.c.claimed_by_name == master_name) &
+                        (tbl.c.claimed_by_incarnation == master_incarnation))
+                else:
+                    q = q.where(
+                        (tbl.c.claimed_at != None) &
+                        (tbl.c.claimed_at != 0) &
+                        (tbl.c.claimed_by_name != None) &
+                        (tbl.c.claimed_by_incarnation != None))
+            if buildername is not None:
+                q = q.where(tbl.c.buildername == buildername)
+            if complete is not None:
+                if complete:
+                    q = q.where(tbl.c.complete != 0)
+                else:
+                    q = q.where(tbl.c.complete == 0)
+            res = conn.execute(q)
+            return [ self._brdictFromRow(row) for row in res.fetchall() ]
+        return self.db.pool.do(thd)
+
+    def claimBuildRequests(self, brids, _reactor=reactor, _race_hook=None):
+        """
+        Try to "claim" the indicated build requests for this buildmaster
+        instance.  The resulting deferred will fire normally on success, or
+        fail with L{AleadyClaimedError} if I{any} of the build requests are
+        already claimed by another master instance, or don't exist.  In this
+        case, none of the claims will take effect.
+
+        This can be used to re-claim build requests, too.  That is, it will
+        succeed in claiming a build request that is already claimed by this
+        master instance, and will update its claimed_at date.
+
+        @param brid: ids of buildrequests to claim
+        @type brid: list
+
+        @param _reactor: reactor to use (for testing)
+        @param _race_hook: hook for testing
+
+        @returns: Deferred
+        """
+
+        # This function attempts to work reasonably well across a number of
+        # database engines with a variety of transactional isolation levels.
+        # Unlike older versions of Buildbot, this uses a qualified UPDATE
+        # statement that will only claim unclaimed builds, so no potential
+        # serialization of parallel UPDATE operations can result in both
+        # parties believing they have claimed a build request.  This technique
+        # will tend to work better in environments with higher isolation
+        # levels, and may result in an IntegrityError for SERIALIZABLE
+        # databases.
+        #
+        # We then perform a post-UPDATE check to ensure that we really have
+        # claimed all of the desired build requests.  This will be most
+        # effective in environments with lower transactional isolation levels,
+        # which may incorrectly serialize the conflicting UPDATES.
+
+        def alreadyClaimed(conn):
+            # helper function to un-claim already-claimed requests, if we can't
+            # claim all of them.  This may be redundant for the finer database
+            # engines, but won't hurt.
+            master_name = self.db.master.master_name
+            master_incarnation = self.db.master.master_incarnation
+            tbl = self.db.model.buildrequests
+
+            # only select *my builds* in this set of brids
+            q = tbl.update()
+            q = q.where((tbl.c.id.in_(brids)) &
+                ((tbl.c.claimed_at != None) &
+                 (tbl.c.claimed_by_name == master_name) &
+                 (tbl.c.claimed_by_incarnation == master_incarnation)))
+            # and unclaim them
+            conn.execute(q,
+                claimed_at=None,
+                claimed_by_name=None,
+                claimed_by_incarnation=None)
+
+        def thd(conn):
+            # update conditioned on the request being unclaimed, or claimed by
+            # this instance.  In either case, the claimed_at is set to the
+            # current time, so this will re-claim an already-claimed requeset.
+            master_name = self.db.master.master_name
+            master_incarnation = self.db.master.master_incarnation
+            tbl = self.db.model.buildrequests
+
+            try:
+                transaction = conn.begin()
+
+                q = tbl.update(whereclause=(tbl.c.id.in_(brids)))
+                q = q.where(
+                    # unclaimed
+                    (((tbl.c.claimed_at == None) | (tbl.c.claimed_at == 0)) &
+                    (tbl.c.claimed_by_name == None) &
+                    (tbl.c.claimed_by_incarnation == None)) |
+                    # .. or mine
+                    ((tbl.c.claimed_at != None) &
+                    (tbl.c.claimed_by_name == master_name) &
+                    (tbl.c.claimed_by_incarnation == master_incarnation)))
+                res = conn.execute(q,
+                    claimed_at=_reactor.seconds(),
+                    claimed_by_name=self.db.master.master_name,
+                    claimed_by_incarnation=self.db.master.master_incarnation)
+                updated_rows = res.rowcount
+                res.close()
+
+                transaction.commit()
+            except (sa.exc.ProgrammingError, sa.exc.IntegrityError):
+                alreadyClaimed(conn)
+                raise AlreadyClaimedError
+
+            # if no or too few rows were updated, then we failed
+            if updated_rows != len(brids):
+                alreadyClaimed(conn)
+                raise AlreadyClaimedError
+
+            # testing hook to simulate a race condition
+            if _race_hook:
+                _race_hook(conn)
+
+            # but double-check to be sure all of the desired build requests
+            # now belong to this master
+            q = sa.select([tbl.c.claimed_by_name,
+                           tbl.c.claimed_by_incarnation],
+                          whereclause=(tbl.c.id.in_(brids)))
+            res = conn.execute(q)
+            for row in res:
+                if row.claimed_by_name != master_name or \
+                        row.claimed_by_incarnation != master_incarnation:
+                    alreadyClaimed(conn)
+                    raise AlreadyClaimedError
+            res.close()
+
+        return self.db.pool.do(thd)
+
+    def completeBuildRequest(self, brid, results, _reactor=reactor):
+        """
+        Complete a build request that is owned by this master instance.  This
+        will fail with L{NotClaimedError} if the build request is not claimed
+        by this instance, is already completed, or does not exist.
+
+        @param brid: build request ID to claim
+        @type brid: integer
+
+        @param results: integer result code
+        @type results: integer
+
+        @param _reactor: reactor to use (for testing)
+
+        @returns: Deferred
+        """
+        def thd(conn):
+            # the update here is simple, but a number of conditions are
+            # attached to ensure that we do not update a row inappropriately
+            master_name = self.db.master.master_name
+            master_incarnation = self.db.master.master_incarnation
+            tbl = self.db.model.buildrequests
+
+            q = tbl.update(whereclause=(tbl.c.id == brid))
+            q = q.where(
+                (tbl.c.claimed_at != None) &
+                (tbl.c.claimed_by_name == master_name) &
+                (tbl.c.claimed_by_incarnation == master_incarnation) &
+                (tbl.c.complete == 0))
+            res = conn.execute(q,
+                complete=1,
+                results=results,
+                complete_at=_reactor.seconds())
+
+            # if no rows were updated, then we failed
+            if not res.rowcount:
+                raise NotClaimedError
+        return self.db.pool.do(thd)
+
+    def unclaimOldIncarnationRequests(self):
+        """
+        Find any incomplete build requests claimed by an old incarnation of
+        this master and mark them as unclaimed.
+
+        @returns: Deferred
+        """
+        def thd(conn):
+            tbl = self.db.model.buildrequests
+            master_name = self.db.master.master_name
+            master_incarnation = self.db.master.master_incarnation
+
+            q = tbl.update(whereclause=(
+                    (tbl.c.claimed_by_name == master_name) &
+                    (tbl.c.claimed_by_incarnation != master_incarnation) &
+                    (tbl.c.complete == 0)))
+            conn.execute(q,
+                claimed_at=0,
+                claimed_by_name=None,
+                claimed_by_incarnation=None)
+        return self.db.pool.do(thd)
+
+    def unclaimExpiredRequests(self, old, _reactor=reactor):
+        """
+        Find any incomplete claimed builds which are older than C{old} seconds,
+        and clear their claim information.
+
+        This is intended to catch builds that were claimed by a master which
+        has since disappeared.
+
+        @param old: number of seconds after which a claim is considered old
+        @type old: int
+
+        @param _reactor: for testing
+
+        @returns: Deferred
+        """
+        def thd(conn):
+            tbl = self.db.model.buildrequests
+            old_epoch = _reactor.seconds() - old
+
+            q = tbl.update(whereclause=(
+                    (tbl.c.claimed_at != 0) &
+                    (tbl.c.claimed_at < old_epoch) &
+                    (tbl.c.complete == 0)))
+            conn.execute(q,
+                claimed_at=0,
+                claimed_by_name=None,
+                claimed_by_incarnation=None)
+        return self.db.pool.do(thd)
+
+    def _brdictFromRow(self, row):
+        claimed = mine = False
+        if (row.claimed_at
+                and row.claimed_by_name is not None
+                and row.claimed_by_incarnation is not None):
+            claimed = True
+            master_name = self.db.master.master_name
+            master_incarnation = self.db.master.master_incarnation
+            if (row.claimed_by_name == master_name and
+                row.claimed_by_incarnation == master_incarnation):
+               mine = True
+
+        def mkdt(epoch):
+            if epoch:
+                return epoch2datetime(epoch)
+        submitted_at = mkdt(row.submitted_at)
+        claimed_at = mkdt(row.claimed_at)
+        complete_at = mkdt(row.complete_at)
+
+        return dict(brid=row.id, buildsetid=row.buildsetid,
+                buildername=row.buildername, priority=row.priority,
+                claimed=claimed, claimed_at=claimed_at, mine=mine,
+                complete=bool(row.complete), results=row.results,
+                submitted_at=submitted_at, complete_at=complete_at)

master/buildbot/db/connector.py

@@ -27,7 +27,7 @@
 from buildbot.util.eventual import eventually
 from buildbot.util import json
 from buildbot.db import pool, model, changes, schedulers, sourcestamps
-from buildbot.db import state, buildsets
+from buildbot.db import state, buildsets, buildrequests
 
 def _one_or_else(res, default=None, process_f=lambda x: x):
     if not res:
@@ -102,6 +102,9 @@ def __init__(self, master, db_url, basedir):
         self.buildsets = buildsets.BuildsetsConnectorComponent(self)
         "L{buildbot.db.sourcestamps.BuildsetsConnectorComponent} instance"
 
+        self.buildrequests = buildrequests.BuildRequestsConnectorComponent(self)
+        "L{buildbot.db.sourcestamps.BuildRequestsConnectorComponent} instance"
+
         self.state = state.StateConnectorComponent(self)
         "L{buildbot.db.state.StateConnectorComponent} instance"