Timed scheduler accepts codebases and creates a buildset

with sourcestamps for each codebase

master/buildbot/schedulers/base.py

@@ -35,7 +35,6 @@ class BaseScheduler(service.MultiService, ComparableMixin):
     implements(interfaces.IScheduler)
 
     compare_attrs = ('name', 'builderNames', 'properties', 'codebases')
-    allowed_codebase_attrs = set(['repository', 'branch', 'revision'])
 
     def __init__(self, name, builderNames, properties, codebases = None):
         """
@@ -92,14 +91,14 @@ def __init__(self, name, builderNames, properties, codebases = None):
         # Set the other codebases that are necessary to process the changes
         # These codebases will always result in a sourcestamp with or without changes
         if codebases:
-            try:
-                for codebase, values in codebases.iteritems():
-                    codebase_attrs = set(values.keys())
-                    # (A,B,D) & (A,B,C) = (A,B) != (A,B,C)
-                    if not codebase_attrs.issubset(self.allowed_codebase_attrs):
-                        raise ValueError, "codebase %s has invalid values %s" % (codebase,codebase_attrs)
-            except Exception as ex:
-                raise ValueError, "Codebases does not have the correct dictionary struct: %s" % ex
+            if not isinstance(codebases, dict):
+                raise ValueError, "Codebases must be a dict of dicts"
+            for codebase, codebase_attrs in codebases.iteritems():
+                if not isinstance(codebase_attrs, dict):
+                    raise ValueError, "Codebases must be a dict of dicts"
+                if 'repository' not in codebase_attrs:
+                    raise ValueError, "The key 'repository' is mandatory in codebases"
+
         self.codebases = codebases
 
         # internal variables
@@ -195,7 +194,10 @@ def getBranch(self, codebase):
             return None
 
     def getRevision(self, codebase):
-        return self.codebases[codebase]['revision']
+        if 'revision' in self.codebases[codebase]:
+            return self.codebases[codebase]['revision']
+        else:
+            return None
 
     ## change handling
 
@@ -317,9 +319,37 @@ def addBuildsetForLatest(self, reason='', external_idstring=None,
         yield wfd
         setid = wfd.getResult()
 
-        wfd = defer.waitForDeferred(self.master.db.sourcestamps.addSourceStamp(
-                branch=branch, revision=None, repository=repository,
-                project=project, sourcestampsetid=setid))
+        if self.codebases:
+            # add a sourcestamp for each codebase
+            dl = []
+            for codebase, cb_info in self.codebases.iteritems():
+                if 'repository' in cb_info:
+                    ss_repository = cb_info['repository']
+                else: 
+                    ss_repository = repository
+                if 'branch' in cb_info:
+                    ss_branch = cb_info['branch']
+                else:
+                    ss_branch = branch
+                if 'revision' in cb_info:
+                    ss_revision = cb_info['revision']
+                else:
+                    ss_revision = None
+                dl.append(  self.master.db.sourcestamps.addSourceStamp(
+                            codebase=codebase,
+                            repository=ss_repository,
+                            branch=ss_branch,
+                            revision=ss_revision,
+                            project=project,
+                            changeids=set(),
+                            sourcestampsetid=setid))
+            d = defer.gatherResults(dl)
+            wfd = defer.waitForDeferred(d)
+        else:
+            wfd = defer.waitForDeferred(self.master.db.sourcestamps.addSourceStamp(
+                    branch=branch, revision=None, repository=repository,
+                    project=project, sourcestampsetid=setid))
+
         yield wfd
         wfd.getResult()
 
@@ -356,13 +386,7 @@ def get_changeids_from_codebase(codebase):
             return [c.number for c in chDict[codebase]]
 
         def get_last_change_for_codebase(codebase):
-            lastChange = None
-            maxNr = -1
-            for c in chDict[codebase]:
-                if c.number > maxNr:
-                    lastChange = c
-                    maxNr = c.number
-            return lastChange
+            return max(chDict[codebase],key = lambda change: change.number)
 
         def create_sourcestamp(changeids, change, setid = None):
             return self.master.db.sourcestamps.addSourceStamp(
@@ -423,7 +447,7 @@ def create_sourcestamp_without_changes(setid, codebase):
             lastChange = get_last_change_for_codebase(codebase)
             # pass the last change to fill the sourcestamp 
             dcall = create_sourcestamp( changeids = ids, change=lastChange, 
-                                    setid = setid)
+                                        setid = setid)
             dl.append(dcall)
         d = defer.gatherResults(dl)
         wfd = defer.waitForDeferred(d)

master/buildbot/schedulers/timed.py

@@ -30,8 +30,9 @@ class Timed(base.BaseScheduler):
 
     compare_attrs = base.BaseScheduler.compare_attrs
 
-    def __init__(self, name, builderNames, properties={}):
-        base.BaseScheduler.__init__(self, name, builderNames, properties)
+    def __init__(self, name, builderNames, properties={}, codebases = None):
+        base.BaseScheduler.__init__(self, name, builderNames, properties, 
+                                    codebases = codebases)
 
         # tracking for when to start the next build
         self.lastActuated = None
@@ -237,18 +238,38 @@ class NoBranch: pass
     def __init__(self, name, builderNames, minute=0, hour='*',
                  dayOfMonth='*', month='*', dayOfWeek='*',
                  branch=NoBranch, fileIsImportant=None, onlyIfChanged=False,
-                 properties={}, change_filter=None, onlyImportant=False):
-        Timed.__init__(self, name=name, builderNames=builderNames, properties=properties)
+                 properties={}, change_filter=None, onlyImportant=False,
+                 codebases = None):
+        Timed.__init__(self, name=name, builderNames=builderNames, properties=properties,
+                       codebases = codebases)
 
         # If True, only important changes will be added to the buildset.
         self.onlyImportant = onlyImportant
 
         if fileIsImportant and not callable(fileIsImportant):
             config.error(
                 "fileIsImportant must be a callable")
-        if branch is Nightly.NoBranch:
+
+        # one of both parameters must be set
+        if not self.codebases and branch is Nightly.NoBranch:
+            config.error(
+                "Nightly parameter 'branch' is required when codebases is not set")
+
+        # not both
+        if self.codebases and branch is not Nightly.NoBranch:
             config.error(
-                "Nightly parameter 'branch' is required")
+                "Nightly parameter 'branch' or 'codebases' must be set not both")
+
+        if self.codebases:
+            # set branch to valid value
+            self.branch = None
+            # check if branch is set for all codebases
+            for k,v in codebases.iteritems():
+                if 'branch' not in v:
+                    config.error(
+                        "key 'branch' is required in codebases if nightly " + 
+                        "parameter 'branch' is not set")
+                    break
 
         self.minute = minute
         self.hour = hour
@@ -275,7 +296,10 @@ def gotChange(self, change, important):
         # we will include all such changes in any buildsets we start.  Note
         # that we must check the branch here because it is not included in the
         # change filter
-        if change.branch != self.branch:
+        if self.codebases:
+            if change.codebase not in self.codebases:
+                return defer.succeed(None) # don't care about this change
+        elif change.branch != self.branch:
             return defer.succeed(None) # don't care about this change
         return self.master.db.schedulers.classifyChanges(
                 self.objectid, { change.number : important })

master/buildbot/test/unit/test_schedulers_base.py

@@ -57,7 +57,8 @@ def test_constructor_codebases_valid(self):
         self.makeScheduler(codebases = codebases)
 
     def test_constructor_codebases_invalid(self):
-        codebases = {"codebase1": {"repository":"", "that":"", "failed":""}}
+        # scheduler only accepts codebases with at least repository set
+        codebases = {"codebase1": {"dictionary":"", "that":"", "fails":""}}
         self.assertRaises(ValueError,
                             lambda : self.makeScheduler(codebases = codebases))