Merge branch 'treeStableTimer' of git://github.com/jaredgrubb/buildbot

master/buildbot/db/schedulers.py

@@ -58,16 +58,33 @@ def thd(conn):
         return self.db.pool.do(thd)
 
     class Thunk: pass
-    def getChangeClassifications(self, objectid, branch=Thunk):
+    def getChangeClassifications(self, objectid, branch=Thunk,
+                                 repository=Thunk, project=Thunk,
+                                 codebase=Thunk):
         def thd(conn):
             sch_ch_tbl = self.db.model.scheduler_changes
             ch_tbl = self.db.model.changes
 
             wc = (sch_ch_tbl.c.objectid == objectid)
+
+            # may need to filter further based on branch, etc
+            extra_wheres = [] 
             if branch is not self.Thunk:
-                wc = wc & (
-                    (sch_ch_tbl.c.changeid == ch_tbl.c.changeid) &
-                    (ch_tbl.c.branch == branch))
+                extra_wheres.append(ch_tbl.c.branch == branch)
+            if repository is not self.Thunk:
+                extra_wheres.append(ch_tbl.c.repository == repository)
+            if project is not self.Thunk:
+                extra_wheres.append(ch_tbl.c.project == project)
+            if codebase is not self.Thunk:
+                extra_wheres.append(ch_tbl.c.codebase == codebase)
+
+            # if we need to filter further append those, as well as a join
+            # on changeid (but just once for that one)
+            if extra_wheres:
+                wc &= (sch_ch_tbl.c.changeid == ch_tbl.c.changeid)
+                for w in extra_wheres:
+                    wc &= w
+
             q = sa.select(
                 [ sch_ch_tbl.c.changeid, sch_ch_tbl.c.important ],
                 whereclause=wc)

master/buildbot/schedulers/basic.py

@@ -243,12 +243,14 @@ def getChangeFilter(self, branch, branches, change_filter, categories):
                 categories=categories)
 
     def getTimerNameForChange(self, change):
-        return change.branch
+        # Py2.6+: could be a namedtuple
+        return (change.codebase, change.project, change.repository, change.branch)
 
     def getChangeClassificationsForTimer(self, objectid, timer_name):
-        branch = timer_name # set in getTimerNameForChange
+        codebase, project, repository, branch = timer_name # set in getTimerNameForChange
         return self.master.db.schedulers.getChangeClassifications(
-                self.objectid, branch=branch)
+                self.objectid, branch=branch, repository=repository,
+                codebase=codebase, project=project)
 
 # now at buildbot.schedulers.dependent, but keep the old name alive
 Dependent = dependent.Dependent

master/buildbot/test/fake/fakedb.py

@@ -440,6 +440,7 @@ def fakeAddChangeInstance(self, change):
             revlink=change.revlink,
             properties=change.properties,
             repository=change.repository,
+            codebase=change.codebase,
             project=change.project)
         self.changes[changeid] = row
 
@@ -471,16 +472,37 @@ def flushChangeClassifications(self, objectid, less_than=None):
             self.classifications[objectid] = {}
         return defer.succeed(None)
 
-    def getChangeClassifications(self, objectid, branch=-1):
+    def getChangeClassifications(self, objectid, branch=-1, repository=-1,
+                                 project=-1, codebase=-1):
         classifications = self.classifications.setdefault(objectid, {})
-        if branch is not -1:
+
+        sentinel = dict(branch=object(), repository=object(),
+                        project=object(), codebase=object())
+
+        if branch != -1:
+            # filter out the classifications for the requested branch
+            classifications = dict(
+                    (k,v) for (k,v) in classifications.iteritems()
+                    if self.db.changes.changes.get(k, sentinel)['branch'] == branch )
+
+        if repository != -1:
             # filter out the classifications for the requested branch
-            change_branches = dict(
-                    (id, c['branch'])
-                    for id, c in self.db.changes.changes.iteritems() )
             classifications = dict(
                     (k,v) for (k,v) in classifications.iteritems()
-                    if k in change_branches and change_branches[k] == branch )
+                    if self.db.changes.changes.get(k, sentinel)['repository'] == repository )
+
+        if project != -1:
+            # filter out the classifications for the requested branch
+            classifications = dict(
+                    (k,v) for (k,v) in classifications.iteritems()
+                    if self.db.changes.changes.get(k, sentinel)['project'] == project )
+
+        if codebase != -1:
+            # filter out the classifications for the requested branch
+            classifications = dict(
+                    (k,v) for (k,v) in classifications.iteritems()
+                    if self.db.changes.changes.get(k, sentinel)['codebase'] == codebase )
+
         return defer.succeed(classifications)
 
     # fake methods

master/buildbot/test/unit/test_schedulers_basic.py

@@ -314,7 +314,7 @@ def test_constructor_branch_forbidden(self):
                 lambda : basic.SingleBranchScheduler(name="tsched", treeStableTimer=60, branch='x'))
 
     def test_gotChange_treeStableTimer_multiple_branches(self):
-        # check that two changes with different branches have different treeStableTimers
+        """Two changes with different branches get different treeStableTimers"""
         sched = self.makeScheduler(basic.AnyBranchScheduler,
                             treeStableTimer=10, branches=['master', 'devel', 'boring'])
 
@@ -343,11 +343,110 @@ def mkch(**kwargs):
         d.addCallback(lambda _ :
                 sched.gotChange(mkch(branch='devel', number=16), True))
         d.addCallback(lambda _ :
-                self.assertEqual(sched.getPendingBuildTimes(), [11,15]))
+                self.assertEqual(sorted(sched.getPendingBuildTimes()), [11,15]))
         d.addCallback(lambda _ :
                 self.clock.pump([1]*10)) # time is now 15
         def check(_):
             self.assertEqual(self.events, [ 'B[13,14]@11', 'B[16]@15' ])
         d.addCallback(check)
 
         d.addCallback(lambda _ : sched.stopService())
+
+    def test_gotChange_treeStableTimer_multiple_repositories(self):
+        """Two repositories, even with the same branch name, have different treeStableTimers"""
+        sched = self.makeScheduler(basic.AnyBranchScheduler,
+                            treeStableTimer=10, branches=['master'])
+
+        sched.startService()
+
+        def mkch(**kwargs):
+            ch = self.makeFakeChange(**kwargs)
+            self.db.changes.fakeAddChangeInstance(ch)
+            return ch
+
+        d = defer.succeed(None)
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', repository="repo", number=13), True))
+        d.addCallback(lambda _ :
+                self.clock.advance(1)) # time is now 1
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', repository="repo", number=14), False))
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', repository="other_repo", number=15), False))
+        d.addCallback(lambda _ :
+                self.clock.pump([1]*4)) # time is now 5
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', repository="other_repo", number=17), True))
+        d.addCallback(lambda _ :
+                self.clock.pump([1]*10)) # time is now 15
+        def check(_):
+            self.assertEqual(self.events, [ 'B[13,14]@11', 'B[15,17]@15' ])
+        d.addCallback(check)
+
+        d.addCallback(lambda _ : sched.stopService())
+
+    def test_gotChange_treeStableTimer_multiple_projects(self):
+        """Two projects, even with the same branch name, have different treeStableTimers"""
+        sched = self.makeScheduler(basic.AnyBranchScheduler,
+                            treeStableTimer=10, branches=['master'])
+
+        sched.startService()
+
+        def mkch(**kwargs):
+            ch = self.makeFakeChange(**kwargs)
+            self.db.changes.fakeAddChangeInstance(ch)
+            return ch
+
+        d = defer.succeed(None)
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', project="proj", number=13), True))
+        d.addCallback(lambda _ :
+                self.clock.advance(1)) # time is now 1
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', project="proj", number=14), False))
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', project="other_proj", number=15), False))
+        d.addCallback(lambda _ :
+                self.clock.pump([1]*4)) # time is now 5
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', project="other_proj", number=17), True))
+        d.addCallback(lambda _ :
+                self.clock.pump([1]*10)) # time is now 15
+        def check(_):
+            self.assertEqual(self.events, [ 'B[13,14]@11', 'B[15,17]@15' ])
+        d.addCallback(check)
+
+        d.addCallback(lambda _ : sched.stopService())
+
+    def test_gotChange_treeStableTimer_multiple_codebases(self):
+        """Two codebases, even with the same branch name, have different treeStableTimers"""
+        sched = self.makeScheduler(basic.AnyBranchScheduler,
+                            treeStableTimer=10, branches=['master'])
+
+        sched.startService()
+
+        def mkch(**kwargs):
+            ch = self.makeFakeChange(**kwargs)
+            self.db.changes.fakeAddChangeInstance(ch)
+            return ch
+
+        d = defer.succeed(None)
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', codebase="base", number=13), True))
+        d.addCallback(lambda _ :
+                self.clock.advance(1)) # time is now 1
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', codebase="base", number=14), False))
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', codebase="other_base", number=15), False))
+        d.addCallback(lambda _ :
+                self.clock.pump([1]*4)) # time is now 5
+        d.addCallback(lambda _ :
+                sched.gotChange(mkch(branch='master', codebase="other_base", number=17), True))
+        d.addCallback(lambda _ :
+                self.clock.pump([1]*10)) # time is now 15
+        def check(_):
+            self.assertEqual(self.events, [ 'B[13,14]@11', 'B[15,17]@15' ])
+        d.addCallback(check)
+
+        d.addCallback(lambda _ : sched.stopService())

master/docs/relnotes/index.rst

@@ -36,6 +36,9 @@ Features
 
 * The latent buildslave support is less buggy, thanks to :bb:pull:`646`.
 
+* The ``treeStableTimer`` for ``AnyBranchScheduler`` now maintains separate timers for separate branches,
+  codebases, projects, and repositories.
+
 Deprecations, Removals, and Non-Compatible Changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~