Merge branch 'master' of github.com:buildbot/buildbot

* 'master' of github.com:buildbot/buildbot:
  Don't be ambivalent about not waiting for the build to complete.
  NightlyTriggerable: Recover gracefully from a bad lastTrigger in db.
  Add references to 'codebases' parameter of schedulers.
  Cleanup commments in NightlyTriggerable.
  Update NightlyTriggerable to match codebase-aware Triggerable.
  Fix passing hour to NightlyBase constructor.
  Fix NightlyTriggerable tests for sourcestamp sets.
  Fix compare_attrs of NightlyBase and Nightly scheduelers.
  Make Trigger step recognize NightlyTriggerable.
  Add documentation for NightlyTriggerable.
  Add tests for properties to NightlyTriggerable.
  Make NightlyTriggerable persist last trigger information.
  Add utility function to recreate a Properties object from the result of .asDict().
  Add tests for NightlyTriggerable.
  Factor out NightlyBase tests into a separate file.
  Add initial implementation of NightlyTriggerable.
  Refactor getNextBuildTime into base class.

master/buildbot/interfaces.py

@@ -1201,6 +1201,15 @@ def render(value):
 class IScheduler(Interface):
     pass
 
+class ITriggerableScheduler(Interface):
+    """
+    A scheduler that can be triggered by buildsteps.
+    """
+
+    def trigger(sourcestamps, set_props=None):
+        """Trigger a build with the given source stamp and properties.
+        """
+
 class IBuildStepFactory(Interface):
     def buildStep():
         """

master/buildbot/process/buildrequest.py

@@ -110,10 +110,7 @@ def _make_br(cls, brid, brdict, master):
         # fetch the buildset properties, and convert to Properties
         buildset_properties = yield master.db.buildsets.getBuildsetProperties(brdict['buildsetid'])
 
-        pr = properties.Properties()
-        for name, (value, source) in buildset_properties.iteritems():
-            pr.setProperty(name, value, source)
-        buildrequest.properties = pr
+        buildrequest.properties = properties.Properties.fromDict(buildset_properties)
 
         # fetch the sourcestamp dictionary
         sslist = yield  master.db.sourcestamps.getSourceStamps(buildset['sourcestampsetid'])

master/buildbot/process/properties.py

@@ -54,6 +54,13 @@ def __init__(self, **kwargs):
         self.build = None # will be set by the Build when starting
         if kwargs: self.update(kwargs, "TEST")
 
+    @classmethod
+    def fromDict(cls, propDict):
+        properties = cls()
+        for name, (value, source) in propDict.iteritems():
+            properties.setProperty(name, value, source)
+        return properties
+
     def __getstate__(self):
         d = self.__dict__.copy()
         d['build'] = None

master/buildbot/schedulers/base.py

@@ -361,6 +361,50 @@ def addBuildSetForSourceStampDetails(self, reason='', external_idstring=None,
         defer.returnValue(rv)
 
 
+    @defer.inlineCallbacks
+    def addBuildsetForSourceStampSetDetails(self, reason, sourcestamps, properties):
+        if sourcestamps is None:
+            sourcestamps = {}
+
+        # Define new setid for this set of sourcestamps
+        new_setid = yield self.master.db.sourcestampsets.addSourceStampSet()
+
+        # Merge codebases with the passed list of sourcestamps
+        # This results in a new sourcestamp for each codebase
+        for codebase in self.codebases:
+            ss = self.codebases[codebase].copy()
+             # apply info from passed sourcestamps onto the configured default
+             # sourcestamp attributes for this codebase.
+            ss.update(sourcestamps.get(codebase,{}))
+
+            # at least repository must be set, this is normaly forced except when
+            # codebases is not explicitly set in configuration file.
+            ss_repository = ss.get('repository')
+            if not ss_repository:
+                config.error("The codebases argument is not set but still receiving " +
+                             "non empty codebase values")
+
+            # add sourcestamp to the new setid
+            yield self.master.db.sourcestamps.addSourceStamp(
+                        codebase=codebase,
+                        repository=ss_repository,
+                        branch=ss.get('branch', None),
+                        revision=ss.get('revision', None),
+                        project=ss.get('project', ''),
+                        changeids=[c['number'] for c in getattr(ss, 'changes', [])],
+                        patch_body=ss.get('patch_body', None),
+                        patch_level=ss.get('patch_level', None),
+                        patch_author=ss.get('patch_author', None),
+                        patch_comment=ss.get('patch_comment', None),
+                        sourcestampsetid=new_setid)
+
+        rv = yield self.addBuildsetForSourceStamp(
+                                setid=new_setid, reason=reason,
+                                properties=properties)
+
+        defer.returnValue(rv)
+
+
     @defer.inlineCallbacks
     def addBuildsetForChanges(self, reason='', external_idstring=None,
             changeids=[], builderNames=None, properties=None):

master/buildbot/schedulers/timed.py

@@ -13,7 +13,11 @@
 #
 # Copyright Buildbot Team Members
 
+from zope.interface import implements
+
 from buildbot import util
+from buildbot.interfaces import ITriggerableScheduler
+from buildbot.process import buildstep, properties
 from buildbot.schedulers import base
 from twisted.internet import defer, reactor
 from twisted.python import log
@@ -220,20 +224,64 @@ def getNextBuildTime(self, lastActuated):
     def startBuild(self):
         return self.addBuildsetForLatest(reason=self.reason, branch=self.branch)
 
-class Nightly(Timed):
+class NightlyBase(Timed):
     compare_attrs = (Timed.compare_attrs
-            + ('minute', 'hour', 'dayOfMonth', 'month',
-               'dayOfWeek', 'onlyIfChanged', 'fileIsImportant',
-               'change_filter', 'onlyImportant', 'branch'))
+            + ('minute', 'hour', 'dayOfMonth', 'month', 'dayOfWeek'))
+
+    def __init__(self, name, builderNames, minute=0, hour='*',
+                 dayOfMonth='*', month='*', dayOfWeek='*',
+                 properties={}, codebases=base.BaseScheduler.DefaultCodebases):
+        Timed.__init__(self, name=name, builderNames=builderNames,
+                properties=properties, codebases=codebases)
+
+        self.minute = minute
+        self.hour = hour
+        self.dayOfMonth = dayOfMonth
+        self.month = month
+        self.dayOfWeek = dayOfWeek
+
+    def _timeToCron(self, time, isDayOfWeek = False):
+        if isinstance(time, int):
+            if isDayOfWeek:
+                time = (time + 1) % 7 # Convert from Mon = 0 format to Sun = 0 format for use in croniter
+            return time
+
+        if isinstance(time, basestring):
+            return time
+
+        if isDayOfWeek:
+            time = [ (t + 1) % 7 for t in time ] # Conversion for croniter (see above)
+
+        return ','.join([ str(s) for s in time ]) # Convert the list to a string
+
+    def getNextBuildTime(self, lastActuated):
+        # deferred import in case python-dateutil is not present
+        from buildbot.util import croniter
+
+        dateTime = lastActuated or self.now()
+        sched =  '%s %s %s %s %s' % (self._timeToCron(self.minute),
+                                     self._timeToCron(self.hour),
+                                     self._timeToCron(self.dayOfMonth),
+                                     self._timeToCron(self.month),
+                                     self._timeToCron(self.dayOfWeek, True))
+        cron = croniter.croniter(sched, dateTime)
+        nextdate = cron.get_next(float)
+        return defer.succeed(nextdate)
+
+class Nightly(NightlyBase):
+    compare_attrs = (NightlyBase.compare_attrs
+            + ('branch', 'onlyIfChanged', 'fileIsImportant',
+               'change_filter', 'onlyImportant',))
 
     class NoBranch: pass
     def __init__(self, name, builderNames, minute=0, hour='*',
                  dayOfMonth='*', month='*', dayOfWeek='*',
                  branch=NoBranch, fileIsImportant=None, onlyIfChanged=False,
                  properties={}, change_filter=None, onlyImportant=False,
                  codebases = base.BaseScheduler.DefaultCodebases):
-        Timed.__init__(self, name=name, builderNames=builderNames, properties=properties,
-                       codebases = codebases)
+        NightlyBase.__init__(self, name=name, builderNames=builderNames,
+                minute=minute, hour=hour, dayOfWeek=dayOfWeek, dayOfMonth=dayOfMonth,
+                properties=properties, codebases=codebases)
 
         # If True, only important changes will be added to the buildset.
         self.onlyImportant = onlyImportant
@@ -246,11 +294,6 @@ def __init__(self, name, builderNames, minute=0, hour='*',
             config.error(
                 "Nightly parameter 'branch' is required")
 
-        self.minute = minute
-        self.hour = hour
-        self.dayOfMonth = dayOfMonth
-        self.month = month
-        self.dayOfWeek = dayOfWeek
         self.branch = branch
         self.onlyIfChanged = onlyIfChanged
         self.fileIsImportant = fileIsImportant
@@ -276,34 +319,6 @@ def gotChange(self, change, important):
         return self.master.db.schedulers.classifyChanges(
                 self.objectid, { change.number : important })
 
-    def _timeToCron(self, time, isDayOfWeek = False):
-        if isinstance(time, int):
-            if isDayOfWeek:
-                time = (time + 1) % 7 # Convert from Mon = 0 format to Sun = 0 format for use in croniter
-            return time
-
-        if isinstance(time, basestring):
-            return time
-
-        if isDayOfWeek:
-            time = [ (t + 1) % 7 for t in time ] # Conversion for croniter (see above)
-
-        return ','.join([ str(s) for s in time ]) # Convert the list to a string
-
-    def getNextBuildTime(self, lastActuated):
-        # deferred import in case python-dateutil is not present
-        from buildbot.util import croniter
-
-        dateTime = lastActuated or self.now()
-        sched =  '%s %s %s %s %s' % (self._timeToCron(self.minute),
-                                     self._timeToCron(self.hour),
-                                     self._timeToCron(self.dayOfMonth),
-                                     self._timeToCron(self.month),
-                                     self._timeToCron(self.dayOfWeek, True))
-        cron = croniter.croniter(sched, dateTime)
-        nextdate = cron.get_next(float)
-        return defer.succeed(nextdate)
-
     @defer.inlineCallbacks
     def startBuild(self):
         scheds = self.master.db.schedulers
@@ -333,3 +348,67 @@ def startBuild(self):
             # start a build of the latest revision, whatever that is
             yield self.addBuildsetForLatest(reason=self.reason,
                                             branch=self.branch)
+
+class NightlyTriggerable(NightlyBase):
+    implements(ITriggerableScheduler)
+    def __init__(self, name, builderNames, minute=0, hour='*',
+                 dayOfMonth='*', month='*', dayOfWeek='*',
+                 properties={}, codebases=base.BaseScheduler.DefaultCodebases):
+        NightlyBase.__init__(self, name=name, builderNames=builderNames, minute=minute, hour=hour,
+                dayOfWeek=dayOfWeek, dayOfMonth=dayOfMonth, properties=properties, codebases=codebases)
+
+        self._lastTrigger = None
+        self.reason = "The NightlyTriggerable scheduler named '%s' triggered this build" % self.name
+
+    def startService(self):
+        NightlyBase.startService(self)
+
+        # get the scheduler's lastTrigger time (note: only done at startup)
+        d = self.getState('lastTrigger', None)
+        def setLast(lastTrigger):
+            try:
+                if lastTrigger:
+                    assert isinstance(lastTrigger[0], dict)
+                    self._lastTrigger = (lastTrigger[0], properties.Properties.fromDict(lastTrigger[1]))
+            except:
+                # If the lastTrigger isn't of the right format, ignore it
+                log.msg("NightlyTriggerable Scheduler <%s>: bad lastTrigger: %r" % (self.name, lastTrigger))
+        d.addCallback(setLast)
+
+    def trigger(self, sourcestamps, set_props=None):
+        """Trigger this scheduler with the given sourcestamp ID. Returns a
+        deferred that will fire when the buildset is finished."""
+        self._lastTrigger = (sourcestamps, set_props)
+
+        # record the trigger in the db
+        if set_props:
+            propsDict = set_props.asDict()
+        else:
+            propsDict = {}
+        d = self.setState('lastTrigger',
+                (sourcestamps, propsDict))
+
+        ## Trigger expects a callback with the success of the triggered
+        ## build, if waitForFinish is True.
+        ## Just return SUCCESS, to indicate that the trigger was succesful,
+        ## don't want for the nightly to run.
+        return d.addCallback(lambda _: buildstep.SUCCESS)
+
+    @defer.inlineCallbacks
+    def startBuild(self):
+        if self._lastTrigger is None:
+            defer.returnValue(None)
+
+        (sourcestamps, set_props) = self._lastTrigger
+        self._lastTrigger = None
+        yield self.setState('lastTrigger', None)
+
+        # properties for this buildset are composed of our own properties,
+        # potentially overridden by anything from the triggering build
+        props = properties.Properties()
+        props.updateFromProperties(self.properties)
+        if set_props:
+            props.updateFromProperties(set_props)
+
+        yield self.addBuildsetForSourceStampSetDetails(reason=self.reason, sourcestamps=sourcestamps,
+                properties=props)

master/buildbot/schedulers/triggerable.py

@@ -13,13 +13,16 @@
 #
 # Copyright Buildbot Team Members
 
+from zope.interface import implements
+
 from twisted.python import failure
 from twisted.internet import defer
+from buildbot.interfaces import ITriggerableScheduler
 from buildbot.schedulers import base
 from buildbot.process.properties import Properties
-from buildbot import config
 
 class Triggerable(base.BaseScheduler):
+    implements(ITriggerableScheduler)
 
     compare_attrs = base.BaseScheduler.compare_attrs
 
@@ -43,7 +46,7 @@ def trigger(self, sourcestamps = None, set_props=None):
         # note that this does not use the buildset subscriptions mechanism, as
         # the duration of interest to the caller is bounded by the lifetime of
         # this process.
-        d = self._addBuildsetForTrigger(self.reason, sourcestamps, props)
+        d = self.addBuildsetForSourceStampDetails(self.reason, sourcestamps, props)
         def setup_waiter((bsid,brids)):
             d = defer.Deferred()
             self._waiters[bsid] = (d, brids)
@@ -67,49 +70,6 @@ def stopService(self):
 
         return base.BaseScheduler.stopService(self)
 
-    @defer.inlineCallbacks
-    def _addBuildsetForTrigger(self, reason, sourcestamps,  properties):
-        if sourcestamps is None:
-            sourcestamps = {}
-
-        # Define new setid for this set of triggering sourcestamps
-        new_setid = yield self.master.db.sourcestampsets.addSourceStampSet()
-
-        # Merge codebases with the passed list of sourcestamps
-        # This results in a new sourcestamp for each codebase
-        for codebase in self.codebases:
-            ss = self.codebases[codebase].copy()
-             # apply info from passed sourcestamps onto the configured default
-             # sourcestamp attributes for this codebase.
-            ss.update(sourcestamps.get(codebase,{}))
-
-            # at least repository must be set, this is normaly forced except when 
-            # codebases is not explicitly set in configuration file.
-            ss_repository = ss.get('repository')
-            if not ss_repository:
-                config.error("The codebases argument is not set but still receiving " +
-                             "non empty codebase values")
-
-            # add sourcestamp to the new setid
-            yield self.master.db.sourcestamps.addSourceStamp(
-                        codebase=codebase,
-                        repository=ss_repository,
-                        branch=ss.get('branch', None),
-                        revision=ss.get('revision', None),
-                        project=ss.get('project', ''),
-                        changeids=[c['number'] for c in getattr(ss, 'changes', [])],
-                        patch_body=ss.get('patch_body', None),
-                        patch_level=ss.get('patch_level', None),
-                        patch_author=ss.get('patch_author', None),
-                        patch_comment=ss.get('patch_comment', None),
-                        sourcestampsetid=new_setid)
-
-        bsid,brids = yield self.addBuildsetForSourceStamp(
-                                setid=new_setid, reason=reason,
-                                properties=properties)
-
-        defer.returnValue((bsid,brids))
-
 
     def _updateWaiters(self):
         if self._waiters and not self._bsc_subscription:

master/buildbot/steps/trigger.py

@@ -13,9 +13,9 @@
 #
 # Copyright Buildbot Team Members
 
+from buildbot.interfaces import ITriggerableScheduler
 from buildbot.process.buildstep import LoggingBuildStep, SUCCESS, FAILURE, EXCEPTION
 from buildbot.process.properties import Properties
-from buildbot.schedulers.triggerable import Triggerable
 from twisted.python import log
 from twisted.internet import defer
 from buildbot import config
@@ -98,7 +98,7 @@ def getSchedulers(self):
             scheduler = scheduler
             if all_schedulers.has_key(scheduler):
                 sch = all_schedulers[scheduler]
-                if isinstance(sch, Triggerable):
+                if ITriggerableScheduler.providedBy(sch):
                     triggered_schedulers.append(sch)
                 else:
                     invalid_schedulers.append(scheduler)