remove db.get_active_buildset_ids

This means that the getBuildSets IStatus method returns a Deferred.

Also includes some further cleanup of the status refactor to allow for
easier inter-module imports

Also fixes an error where StatusClient's 'getBuildSets' method was
returning external_idstring's instead of bsid's.  This error would have
prevented try --wait from working properly in ssh mode in Buildbot
versions 0.8.0 through 0.8.3.

master/NEWS

@@ -30,6 +30,8 @@ configuration fell back to the global c['mergeRequests'] parameter's value.  To
 avoid this, remove `mergeRequests=True` from any BuilderConfig constructor
 invocations.
 
+*** The `Status.getBuildSets` method now returns its result via Deferred.
+
 ** Scheduler Improvements
 
 *** Nightly scheduler now accepts a change_filter argument

master/buildbot/db/connector.py

@@ -428,6 +428,7 @@ def _txn_claim_buildrequests(self, t, now, master_name, master_incarnation,
             qargs = [now, master_name, master_incarnation] + list(batch)
             t.execute(q, qargs)
 
+    # used by Builder._startBuildFor_2
     def build_started(self, brid, buildnumber):
         return self.runInteractionNow(self._txn_build_started, brid, buildnumber)
     def _txn_build_started(self, t, brid, buildnumber):
@@ -439,6 +440,7 @@ def _txn_build_started(self, t, brid, buildnumber):
         self.notify("add-build", bid)
         return bid
 
+    # used by Builder.buildFinished
     def builds_finished(self, bids):
         return self.runInteractionNow(self._txn_build_finished, bids)
     def _txn_build_finished(self, t, bids):
@@ -450,6 +452,7 @@ def _txn_build_finished(self, t, bids):
             qargs = [now] + list(batch)
             t.execute(q, qargs)
 
+    # used by Status
     def get_build_info(self, bid):
         return self.runInteractionNow(self._txn_get_build_info, bid)
     def _txn_get_build_info(self, t, bid):
@@ -463,13 +466,15 @@ def _txn_get_build_info(self, t, bid):
             return res[0]
         return (None,None,None)
 
+    # used by BuildRequestStatus.getBuilds
     def get_buildnums_for_brid(self, brid):
         return self.runInteractionNow(self._txn_get_buildnums_for_brid, brid)
     def _txn_get_buildnums_for_brid(self, t, brid):
         t.execute(self.quoteq("SELECT number FROM builds WHERE brid=?"),
                   (brid,))
         return [number for (number,) in t.fetchall()]
 
+    # used by Builder.buildFinished
     def resubmit_buildrequests(self, brids):
         return self.runInteraction(self._txn_resubmit_buildreqs, brids)
     def _txn_resubmit_buildreqs(self, t, brids):
@@ -484,6 +489,7 @@ def _txn_resubmit_buildreqs(self, t, brids):
             t.execute(q, batch)
         self.notify("add-buildrequest", *brids)
 
+    # used by Builder.buildFinished
     def retire_buildrequests(self, brids, results):
         return self.runInteractionNow(self._txn_retire_buildreqs, brids,results)
     def _txn_retire_buildreqs(self, t, brids, results):
@@ -510,6 +516,7 @@ def _txn_retire_buildreqs(self, t, brids, results):
         self.notify("retire-buildrequest", *brids)
         self.notify("modify-buildset", *bsids)
 
+    # used by BuildRequestControl.cancel and Builder.cancelBuildRequest
     def cancel_buildrequests(self, brids):
         return self.runInteractionNow(self._txn_cancel_buildrequest, brids)
     def _txn_cancel_buildrequest(self, t, brids):
@@ -573,6 +580,7 @@ def _check_buildset(self, t, bsid, now):
             # notify the master
             self.master.buildsetComplete(bsid, bs_results)
 
+    # used by BuildSetStatus
     def get_buildrequestids_for_buildset(self, bsid):
         return self.runInteractionNow(self._txn_get_buildrequestids_for_buildset,
                                       bsid)
@@ -582,6 +590,7 @@ def _txn_get_buildrequestids_for_buildset(self, t, bsid):
                   (bsid,))
         return dict(t.fetchall())
 
+    # use by Status.getBuildSets
     def examine_buildset(self, bsid):
         return self.runInteractionNow(self._txn_examine_buildset, bsid)
     def _txn_examine_buildset(self, t, bsid):
@@ -607,11 +616,7 @@ def _txn_examine_buildset(self, t, bsid):
             successful = True
         return (successful, finished)
 
-    def get_active_buildset_ids(self):
-        return self.runInteractionNow(self._txn_get_active_buildset_ids)
-    def _txn_get_active_buildset_ids(self, t):
-        t.execute("SELECT id FROM buildsets WHERE complete=0")
-        return [bsid for (bsid,) in t.fetchall()]
+    # used by BuildSetStatus.getReason, etc.
     def get_buildset_info(self, bsid):
         return self.runInteractionNow(self._txn_get_buildset_info, bsid)
     def _txn_get_buildset_info(self, t, bsid):
@@ -628,6 +633,7 @@ def _txn_get_buildset_info(self, t, bsid):
             return (external_idstring, reason, ssid, complete, results)
         return None # shouldn't happen
 
+    # used by BuilderStatus.getPendingBuilds
     def get_pending_brids_for_builder(self, buildername):
         return self.runInteractionNow(self._txn_get_pending_brids_for_builder,
                                       buildername)

master/buildbot/interfaces.py

@@ -151,7 +151,11 @@ def getSlave(name):
         """Return the ISlaveStatus object for a given named buildslave."""
 
     def getBuildSets():
-        """Return a list of active (non-finished) IBuildSetStatus objects."""
+        """
+        Return a list of un-completed build sets.
+
+        @returns: list of L{IBuildSetStatus} implementations, via Deferred.
+        """
 
     def generateFinishedBuilds(builders=[], branches=[],
                                num_builds=None, finished_before=None,

master/buildbot/master.py

@@ -78,6 +78,7 @@ def __init__(self, basedir, configFileName="master.cfg"):
         service.MultiService.__init__(self)
         self.setName("buildmaster")
         self.basedir = basedir
+        assert os.path.isdir(self.basedir)
         self.configFileName = configFileName
 
         self.pbmanager = buildbot.pbmanager.PBManager()
@@ -107,7 +108,7 @@ def __init__(self, basedir, configFileName="master.cfg"):
 
         self.debugClientRegistration = None
 
-        self.status = Status(self.botmaster, self.basedir)
+        self.status = Status(self)
         self.statusTargets = []
 
         self.db = None

master/buildbot/status/__init__.py

@@ -0,0 +1,19 @@
+import build, builder, buildstep, buildset, testresult, logfile
+import slave, master, buildrequest
+
+# styles.Versioned requires this, as it keys the version numbers on the fully
+# qualified class name; see master/buildbot/test/regressions/test_unpickling.py
+buildstep.BuildStepStatus.__module__ = 'buildbot.status.builder'
+build.BuildStatus.__module__ = 'buildbot.status.builder'
+
+# add all of these classes to builder; this is a form of late binding to allow
+# circular module references among the status modules
+builder.BuildStepStatus = buildstep.BuildStepStatus
+builder.BuildSetStatus = buildset.BuildSetStatus
+builder.TestResult = testresult.TestResult
+builder.LogFile = logfile.LogFile
+builder.HTMLLogFile = logfile.HTMLLogFile
+builder.SlaveStatus = slave.SlaveStatus
+builder.Status = master.Status
+builder.BuildStatus = build.BuildStatus
+builder.BuildRequestStatus = buildrequest.BuildRequestStatus

master/buildbot/status/builder.py

@@ -619,17 +619,3 @@ def asDict(self):
         result['pendingBuilds'] = len(self.getPendingBuilds())
         return result
 
-# old locations for these classes; these are still used in pickle files
-from buildbot.status.buildstep import BuildStepStatus
-from buildbot.status.buildset import BuildSetStatus
-from buildbot.status.testresult import TestResult
-from buildbot.status.logfile import LogFile, HTMLLogFile
-from buildbot.status.slave import SlaveStatus
-from buildbot.status.master import Status
-_hush_pyflakes = [ BuildStepStatus, BuildSetStatus, TestResult, LogFile,
-                   HTMLLogFile, SlaveStatus, Status, Event, BuildStatus ]
-
-# styles.Versioned requires this, as it keys the version numbers on the fully
-# qualified class name; see master/buildbot/test/regressions/test_unpickling.py
-BuildStepStatus.__module__ = 'buildbot.status.builder'
-BuildStatus.__module__ = 'buildbot.status.builder'

master/buildbot/status/buildset.py

@@ -41,6 +41,7 @@ def getResults(self):
         (external_idstring, reason, ssid, complete, results) = self._get_info()
         return results
     def getID(self):
+        # hah, fooled you - this returns the external_idstring!
         (external_idstring, reason, ssid, complete, results) = self._get_info()
         return external_idstring
 

master/buildbot/status/client.py

@@ -427,7 +427,11 @@ def perspective_unsubscribe(self):
     def perspective_getBuildSets(self):
         """This returns tuples of (buildset, bsid), because that is much more
         convenient for tryclient."""
-        return [(IRemote(s), s.getID()) for s in self.status.getBuildSets()]
+        d = self.status.getBuildSets()
+        def make_remotes(buildsets):
+            return [(IRemote(s), s.id) for s in buildsets]
+        d.addCallback(make_remotes)
+        return d
 
     def perspective_getBuilderNames(self):
         return self.status.getBuilderNames()

master/buildbot/status/master.py

@@ -22,36 +22,20 @@
 from buildbot import interfaces
 from buildbot.util import collections
 from buildbot.util.eventual import eventually
-from buildbot.status.buildset import BuildSetStatus
-from buildbot.status.builder import BuilderStatus
-from buildbot.status.buildrequest import BuildRequestStatus
+from buildbot.status import buildset, builder, buildrequest
 
 class Status:
     """
     I represent the status of the buildmaster.
     """
     implements(interfaces.IStatus)
 
-    def __init__(self, botmaster, basedir):
-        """
-        @type  botmaster: L{buildbot.process.botmaster.BotMaster}
-        @param botmaster: the Status object uses C{.botmaster} to get at
-                          both the L{buildbot.master.BuildMaster} (for
-                          various buildbot-wide parameters) and the
-                          actual Builders (to get at their L{BuilderStatus}
-                          objects). It is not allowed to change or influence
-                          anything through this reference.
-        @type  basedir: string
-        @param basedir: this provides a base directory in which saved status
-                        information (changes.pck, saved Build status
-                        pickles) can be stored
-        """
-        self.botmaster = botmaster
-        self.master = botmaster.parent # TODO: temporary; this should get set more formally
+    def __init__(self, master):
+        self.master = master
+        self.botmaster = master.botmaster
         self.db = None
-        self.basedir = basedir
+        self.basedir = master.basedir
         self.watchers = []
-        assert os.path.isdir(basedir)
         # compress logs bigger than 4k, a good default on linux
         self.logCompressionLimit = 4*1024
         self.logCompressionMethod = "bz2"
@@ -75,7 +59,7 @@ def cancelCleanShutdown(self):
         return self.botmaster.cancelCleanShutdown()
 
     def setDB(self, db):
-        self.db = db
+        # XXX not called anymore - what to do about this?
         self.db.subscribe_to("add-build", self._db_builds_changed)
         self.db.subscribe_to("add-buildset", self._db_buildset_added)
         self.db.subscribe_to("modify-buildset", self._db_buildsets_changed)
@@ -100,22 +84,22 @@ def getURLForThing(self, thing):
         if interfaces.ISchedulerStatus.providedBy(thing):
             pass
         if interfaces.IBuilderStatus.providedBy(thing):
-            builder = thing
+            bldr = thing
             return prefix + "builders/%s" % (
-                urllib.quote(builder.getName(), safe=''),
+                urllib.quote(bldr.getName(), safe=''),
                 )
         if interfaces.IBuildStatus.providedBy(thing):
             build = thing
-            builder = build.getBuilder()
+            bldr = build.getBuilder()
             return prefix + "builders/%s/builds/%d" % (
-                urllib.quote(builder.getName(), safe=''),
+                urllib.quote(bldr.getName(), safe=''),
                 build.getNumber())
         if interfaces.IBuildStepStatus.providedBy(thing):
             step = thing
             build = step.getBuild()
-            builder = build.getBuilder()
+            bldr = build.getBuilder()
             return prefix + "builders/%s/builds/%d/steps/%s" % (
-                urllib.quote(builder.getName(), safe=''),
+                urllib.quote(bldr.getName(), safe=''),
                 build.getNumber(),
                 urllib.quote(step.getName(), safe=''))
         # IBuildSetStatus
@@ -134,7 +118,7 @@ def getURLForThing(self, thing):
             loog = thing
             step = loog.getStep()
             build = step.getBuild()
-            builder = build.getBuilder()
+            bldr = build.getBuilder()
 
             logs = step.getLogs()
             for i in range(len(logs)):
@@ -143,7 +127,7 @@ def getURLForThing(self, thing):
             else:
                 return None
             return prefix + "builders/%s/builds/%d/steps/%s/logs/%s" % (
-                urllib.quote(builder.getName(), safe=''),
+                urllib.quote(bldr.getName(), safe=''),
                 build.getNumber(),
                 urllib.quote(step.getName(), safe=''),
                 urllib.quote(loog.getName()))
@@ -165,8 +149,8 @@ def getBuilderNames(self, categories=None):
         l = []
         # respect addition order
         for name in self.botmaster.builderNames:
-            builder = self.botmaster.builders[name]
-            if builder.builder_status.category in categories:
+            bldr = self.botmaster.builders[name]
+            if bldr.builder_status.category in categories:
                 l.append(name)
         return l
 
@@ -183,8 +167,13 @@ def getSlave(self, slavename):
         return self.botmaster.slaves[slavename].slave_status
 
     def getBuildSets(self):
-        return [BuildSetStatus(bsid, self, self.db)
-                for bsid in self.db.get_active_buildset_ids()]
+        d = self.master.db.buildsets.getBuildSets(complete=False)
+        def make_status_objects(bsdicts):
+            return [ buildset.BuildSetStatus(bsdict['bsid'], self,
+                                             self.master.db)
+                    for bsdict in bsdicts ]
+        d.addCallback(make_status_objects)
+        return d
 
     def generateFinishedBuilds(self, builders=[], branches=[],
                                num_builds=None, finished_before=None,
@@ -289,7 +278,7 @@ def builderAdded(self, name, basedir, category=None):
             log.msg("error follows:")
             log.err()
         if not builder_status:
-            builder_status = BuilderStatus(name, category)
+            builder_status = builder.BuilderStatus(name, category)
             builder_status.addPointEvent(["builder", "created"])
         log.msg("added builder %s in category %s" % (name, category))
         # an unpickled object might not have category set from before,
@@ -351,7 +340,7 @@ def buildreqs_retired(self, requests):
             pass
 
     def get_buildreq_for_id(self, brid):
-        return BuildRequestStatus(brid, self, self.db)
+        return buildrequest.BuildRequestStatus(brid, self, self.db)
 
     def _db_builds_changed(self, category, bid):
         brid,buildername,buildnum = self.db.get_build_info(bid)
@@ -368,7 +357,7 @@ def _buildrequest_unsubscribe(self, brid, observer):
         self._buildreq_observers.discard(brid, observer)
 
     def _db_buildset_added(self, category, bsid):
-        bss = BuildSetStatus(bsid, self, self.db)
+        bss = buildset.BuildSetStatus(bsid, self, self.db)
         for t in self.watchers:
             if hasattr(t, 'buildsetSubmitted'):
                 t.buildsetSubmitted(bss)
@@ -396,7 +385,7 @@ def _db_buildset_changed(self, bsid):
             and bsid not in self._buildset_finished_waiters):
             return
         successful,finished = self.db.examine_buildset(bsid)
-        bss = BuildSetStatus(bsid, self, self.db)
+        bss = buildset.BuildSetStatus(bsid, self, self.db)
         if successful is not None:
             for d in self._buildset_success_waiters.pop(bsid):
                 eventually(d.callback, bss)
@@ -419,7 +408,7 @@ def _handle_buildrequest_event(self, mode, brids):
         for brid in brids:
             buildername = self.db.get_buildername_for_brid(brid)
             if buildername in self._builder_observers:
-                brs = BuildRequestStatus(brid, self, self.db)
+                brs = buildrequest.BuildRequestStatus(brid, self, self.db)
                 for observer in self._builder_observers[buildername]:
                     if mode == "added":
                         if hasattr(observer, 'requestSubmitted'):