Feature/json pending builds

master/buildbot/process/build.py

@@ -424,6 +424,8 @@ def setupBuild(self, expectations):
         if len(self.requests) > 0:
             self.build_status.setSubmitted(self.requests[0].submittedAt)
             self.setProperty("submittedTime", self.requests[0].submittedAt, "buildrequest")
+            self.build_status.setBuildRequestID(self.requests[0].id)
+            self.setProperty("buildRequestID", self.requests[0].id, "buildrequest")
             self.build_status.setBuildChainID(self.requests[0].buildChainID)
             self.setProperty("buildChainID", self.requests[0].buildChainID, "buildrequest")
 

master/buildbot/process/builder.py

@@ -107,7 +107,7 @@ def reconfigService(self, new_config):
         self.builder_status.setCategory(builder_config.category)
         self.builder_status.setSlavenames(self.config.slavenames)
         self.builder_status.setStartSlavenames(self.config.startSlavenames)
-        self.builder_status.setCacheSize(new_config.caches['Builds'])
+        self.builder_status.setCacheSize(new_config.caches)
         self.builder_status.setProject(builder_config.project)
         self.builder_status.setFriendlyName(builder_config.friendly_name)
         self.builder_status.setTags(builder_config.tags)

master/buildbot/status/build.py

@@ -50,6 +50,7 @@ class BuildStatus(styles.Versioned, properties.PropertiesMixin):
     submitted = None
     owners = None
     buildChainID = None
+    buildRequestID = None
     currentStep = None
     text = []
     results = None
@@ -310,6 +311,9 @@ def setSubmitted(self, submitted):
     def setBuildChainID(self, buildChainID):
         self.buildChainID = buildChainID
 
+    def setBuildRequestID(self, buildRequestID):
+        self.buildRequestID = buildRequestID
+
     def setOwners(self, owners):
         self.owners = owners
 
@@ -613,6 +617,7 @@ def asBaseDict(self, request=None, include_current_step=False, include_artifacts
         result['reason'] = self.getReason()
         result['submittedTime'] = self.submitted
         result['owners'] = self.owners
+        result['buildRequestID'] = self.buildRequestID
         result['buildChainID'] = self.buildChainID
         result['blame'] = self.getResponsibleUsers()
         result['url'] = status.getURLForThing(self)

master/buildbot/status/builder.py

@@ -37,7 +37,39 @@
 _hush_pyflakes = [ SUCCESS, WARNINGS, FAILURE, SKIPPED,
                    EXCEPTION, RETRY, Results, worst_status ]
 
-LATEST_BUILD_FORMAT = "{0}={1};"
+CODEBASES_CACHE_KEY_FORMAT = "{0}={1};"
+
+
+def getCodebaseConfiguredBranch(codebase, key):
+    return codebase[key]['defaultbranch'] if 'defaultbranch' in codebase[key] \
+        else codebase[key]['branch'] if 'branch' in codebase[key] else ''
+
+
+def getCodebaseBranch(branch, codebases, key):
+    if key in codebases:
+        return codebases[key]
+
+    branch = branch if isinstance(branch, str) \
+        else branch[0] if (isinstance(branch, list) and len(branch) > 0) else ''
+
+    codebases[key] = branch
+
+    return branch
+
+
+def getCacheKey(project, codebases):
+    output = ""
+    if project and project.codebases:
+        project_codebases = project.codebases
+        cb_keys = sorted(project_codebases, key=lambda s: s.keys()[0])
+
+        for cb in cb_keys:
+            key = cb.keys()[0]
+            branch = getCodebaseConfiguredBranch(cb, key)
+            branch = getCodebaseBranch(branch, codebases, key)
+            output += CODEBASES_CACHE_KEY_FORMAT.format(key, branch)
+    return output
+
 
 class BuilderStatus(styles.Versioned):
     """I handle status information for a single process.build.Builder object.
@@ -102,7 +134,7 @@ def __init__(self, buildername, category, master, friendly_name=None, descriptio
 
     def setStatus(self, status):
         self.status = status
-        self.pendingBuildCache = PendingBuildsCache(self)
+        self.pendingBuildsCache = PendingBuildsCache(self)
 
         if not hasattr(self, 'latestBuildCache'):
             self.latestBuildCache = {}
@@ -131,8 +163,8 @@ def __getstate__(self):
         del d['master']
         self.deleteKey('loadingBuilds', d)
 
-        if 'pendingBuildCache' in d:
-            del d['pendingBuildCache']
+        if 'pendingBuildsCache' in d:
+            del d['pendingBuildsCache']
 
         self.deleteKey('latestBuildCache', d)
         return d
@@ -209,8 +241,10 @@ def saveYourself(self, skipBuilds=False):
 
     # build cache management
 
-    def setCacheSize(self, size):
-        self.buildCache.set_max_size(size)
+    def setCacheSize(self, caches):
+        self.buildCache.set_max_size(caches['Builds'])
+        if caches and 'BuilderBuildRequestStatus' in caches:
+            self.pendingBuildsCache.buildRequestStatusCache.set_max_size(caches['BuilderBuildRequestStatus'])
 
     def makeBuildFilename(self, number):
         return os.path.join(self.basedir, "%d" % number)
@@ -370,15 +404,13 @@ def getAllSlaves(self):
 
     @defer.inlineCallbacks
     def getPendingBuildRequestStatuses(self, codebases={}):
-        sourcestamps = [{'b_codebase': key, 'b_branch': value} for key, value in codebases.iteritems()]
-
-        brdicts = yield self.master.db.buildrequests.getBuildRequestInQueue(buildername=self.name,
-                                                                            sourcestamps=sourcestamps,
-                                                                            sorted=True)
-
-        result = [BuildRequestStatus(self.name, brdict['brid'], self.status) for brdict in brdicts]
+        pendingBuilds = yield self.pendingBuildsCache.getPendingBuilds(codebases=codebases)
+        defer.returnValue(pendingBuilds)
 
-        defer.returnValue(result)
+    @defer.inlineCallbacks
+    def getPendingBuildRequestStatusesDicts(self, codebases={}):
+        pendingBuildsDict = yield self.pendingBuildsCache.getPendingBuildsDicts(codebases=codebases)
+        defer.returnValue(pendingBuildsDict)
 
     def foundCodebasesInBuild(self, build, codebases):
         if len(codebases) > 0:
@@ -565,7 +597,7 @@ def generateFinishedBuildsAsync(self, branches=[], codebases={},
         finishedBuilds = []
         branches = set(branches)
 
-        key = self.getLatestBuildKey(codebases)
+        key = self.getCodebasesCacheKey(codebases)
 
         if self.shouldUseLatestBuildCache(useCache, num_builds, key):
             build = yield self.getLatestBuildCacheAsync(key)
@@ -608,7 +640,7 @@ def generateFinishedBuilds(self, branches=[], codebases={},
                                filter_fn=None,
                                useCache=False):
 
-        key = self.getLatestBuildKey(codebases)
+        key = self.getCodebasesCacheKey(codebases)
         if self.shouldUseLatestBuildCache(useCache, num_builds, key):
             build = self.getLatestBuildCache(key)
             if build:
@@ -865,39 +897,16 @@ def _buildFinished(self, s):
         self.saveLatestBuild(s)
         yield threads.deferToThread(self.prune) # conserve disk
 
-    def getCodebaseBranch(self, branch, codebases, key):
-        if key in codebases:
-            return codebases[key]
-
-        branch = branch if isinstance(branch, str) \
-            else branch[0] if (isinstance(branch, list) and len(branch) > 0) else ''
-
-        codebases[key] = branch
-
-        return branch
-
-    def getCodebaseConfiguredBranch(self, cb, key):
-        return cb[key]['defaultbranch'] if 'defaultbranch' in cb[key] \
-            else cb[key]['branch'] if 'branch' in cb[key] else ''
-
-    def getLatestBuildKey(self, codebases):
-        output = ""
+    def getCodebasesCacheKey(self, codebases={}):
+        codebase_key = ""
 
         if not codebases:
-            return output
+            return codebase_key
 
         project = self.master.getProject(self.getProject())
-        if project and project.codebases:
-            project_codebases = project.codebases
-            cb_keys = sorted(project_codebases, key=lambda s: s.keys()[0])
+        codebase_key = getCacheKey(project, codebases)
 
-            for cb in cb_keys:
-                key = cb.keys()[0]
-                branch = self.getCodebaseConfiguredBranch(cb, key)
-                branch = self.getCodebaseBranch(branch, codebases, key)
-                output += LATEST_BUILD_FORMAT.format(key, branch)
-
-        return output
+        return codebase_key
 
     def updateLatestBuildCache(self, cache, k):
         def nonEmptyCacheUpdateToEmptyBuild():
@@ -933,7 +942,7 @@ def saveLatestBuild(self, build, key=None):
                     codebases[s.codebase] = s.branch
 
             # We save it in the same way as we access it
-            key = self.getLatestBuildKey(codebases)
+            key = self.getCodebasesCacheKey(codebases)
 
         self.updateLatestBuildCache(cache, key)
 
@@ -978,29 +987,17 @@ def asSlaveDict(self):
 
     @defer.inlineCallbacks
     def asDict_async(self, codebases={}, request=None, base_build_dict=False, include_build_steps=True,
-               include_build_props=True):
+                     include_build_props=True, include_pending_builds=False):
         """Just like L{asDict}, but with a nonzero pendingBuilds."""
         result = self.asDict(codebases, request, base_build_dict, include_build_steps=include_build_steps,
                              include_build_props=include_build_props)
-        builds = self.pendingBuildCache.getPendingBuilds()
 
-        #Remove builds not within this codebase
-        count = 0
-        defers = []
-        if len(codebases) > 0:
-            for b in builds:
-                de = self.foundCodebasesInBuildRequest(b, codebases)
-                defers.append(de)
-
-            #Allow the defers to run async
-            for d in defers:
-                in_codebase = yield d
-                if in_codebase:
-                    count += 1
+        if include_pending_builds:
+            pendingBuildsDict = yield self.getPendingBuildRequestStatusesDicts(codebases=codebases)
+            result['pendingBuilds'] = pendingBuildsDict
         else:
-            count = len(builds)
+            result['pendingBuilds'] = yield self.pendingBuildsCache.getTotal(codebases=codebases)
 
-        result['pendingBuilds'] = count
         defer.returnValue(result)
 
     def getMetrics(self):
@@ -1018,18 +1015,70 @@ class PendingBuildsCache():
     """
     def __init__(self, builder):
         self.builder = builder
-        self.cache = []
+        self.buildRequestStatusCodebasesCache = {}
+        self.buildRequestStatusCodebasesDictsCache = {}
+        self.buildRequestStatusCache = LRUCache(BuildRequestStatus.createBuildRequestStatus, 50)
         self.cache_now()
         self.builder.subscribe(self)
 
+    @defer.inlineCallbacks
+    def fetchPendingBuildRequestStatuses(self, codebases={}):
+        sourcestamps = [{'b_codebase': key, 'b_branch': value} for key, value in codebases.iteritems()]
+
+        brdicts = yield self.builder.master.db.buildrequests.getBuildRequestInQueue(
+                buildername=self.builder.name,
+                sourcestamps=sourcestamps,
+                sorted=True
+        )
+
+        result = []
+        for brdict in brdicts:
+            brs = self.buildRequestStatusCache.get(
+                    key=brdict['brid'],
+                    buildername=self.builder.name,
+                    status=self.builder.status
+            )
+
+            brs.update(brdict)
+            result.append(brs)
+
+        defer.returnValue(result)
+
     @defer.inlineCallbacks
     def cache_now(self):
+        self.buildRequestStatusCodebasesCache = {}
+        self.buildRequestStatusCodebasesDictsCache = {}
         if hasattr(self.builder, "status"):
-            self.cache = yield self.builder.getPendingBuildRequestStatuses()
-            defer.returnValue(self.cache)
+            key = self.builder.getCodebasesCacheKey()
+            self.buildRequestStatusCodebasesCache[key] = yield self.fetchPendingBuildRequestStatuses()
+            defer.returnValue(self.buildRequestStatusCodebasesCache[key])
+
+    @defer.inlineCallbacks
+    def getPendingBuilds(self, codebases={}):
+        key = self.builder.getCodebasesCacheKey(codebases)
+
+        if not self.buildRequestStatusCodebasesCache or key not in self.buildRequestStatusCodebasesCache:
+            self.buildRequestStatusCodebasesCache[key] = \
+                yield self.fetchPendingBuildRequestStatuses(codebases=codebases)
+
+        defer.returnValue(self.buildRequestStatusCodebasesCache[key])
 
-    def getPendingBuilds(self):
-        return self.cache
+    @defer.inlineCallbacks
+    def getPendingBuildsDicts(self, codebases={}):
+        key = self.builder.getCodebasesCacheKey(codebases)
+
+        if not self.buildRequestStatusCodebasesDictsCache or key not in self.buildRequestStatusCodebasesDictsCache:
+            pendingBuilds = yield self.getPendingBuilds(codebases)
+            self.buildRequestStatusCodebasesDictsCache[key] = [
+                (yield brs.asDict_async(codebases=codebases)) for brs in pendingBuilds
+                ]
+
+        defer.returnValue(self.buildRequestStatusCodebasesDictsCache[key])
+
+    @defer.inlineCallbacks
+    def getTotal(self, codebases={}):
+        pendingBuilds = yield self.getPendingBuilds(codebases)
+        defer.returnValue(len(pendingBuilds))
 
     def buildStarted(self, builderName, state):
         self.cache_now()

master/buildbot/status/buildrequest.py

@@ -27,10 +27,13 @@ def __init__(self, buildername, brid, status):
         self.brid = brid
         self.status = status
         self.master = status.master
-
         self._buildrequest = None
         self._buildrequest_lock = defer.DeferredLock()
 
+    @classmethod
+    def createBuildRequestStatus(cls, brid, buildername, status):
+        return BuildRequestStatus(buildername, brid, status)
+
     @defer.inlineCallbacks
     def _getBuildRequest(self):
         """
@@ -172,7 +175,7 @@ def asDict(self):
         return result
 
     @defer.inlineCallbacks
-    def asDict_async(self, request=None):
+    def asDict_async(self, codebases={}):
         result = {}
         builder = self.status.getBuilder(self.getBuilderName())
         if not builder:
@@ -191,14 +194,16 @@ def asDict_async(self, request=None):
         result['reason'] = yield self.getReason()
         result['slaves'] =  self.getSlaves()
         result['submittedAt'] = yield self.getSubmitTime()
-        result['results'] = yield self.getResults()
 
         result['builderFriendlyName'] = builder.getFriendlyName()
         result['builderURL'] = self.status.getURLForThing(builder)
 
-        if request is not None:
-            from buildbot.status.web.base import getCodebasesArg
-            result['builderURL'] += getCodebasesArg(request)
+        if codebases:
+            from buildbot.status.web.base import getCodeBasesURLParam
+            result['builderURL'] += getCodeBasesURLParam(codebases=codebases)
+
+        # this data changes
+        result['results'] = yield self.getResults() # is already on the object at the time we fetch
 
         builds = yield self.getBuilds()
         all_builds = [build.asDict() for build in builds]
@@ -208,3 +213,7 @@ def asDict_async(self, request=None):
                                                                   and 'number' in sorted_builds[0] else None
 
         defer.returnValue(result)
+
+    def update(self, brdict):
+        if self._buildrequest and brdict and 'results' in brdict and self._buildrequest.results != brdict['results']:
+            self._buildrequest.results = brdict['results']