Merge pull request #1636 from jaredgrubb/jgrubb-backport-dyn-scheduler

Backport the "dynamic trigger" from nine

master/buildbot/steps/trigger.py

@@ -69,6 +69,9 @@ def __init__(self, schedulerNames=[], sourceStamp=None, sourceStamps=None,
         self.ended = False
         LoggingBuildStep.__init__(self, **kwargs)
 
+        # Scheduler name cache
+        self._all_schedulers = None
+
     def interrupt(self, reason):
         if self.running and not self.ended:
             self.step_status.setText(["interrupted"])
@@ -80,33 +83,27 @@ def end(self, result):
             return self.finished(result)
 
     # Create the properties that are used for the trigger
-    def createTriggerProperties(self):
+    def createTriggerProperties(self, properties):
         # make a new properties object from a dict rendered by the old
         # properties object
         trigger_properties = Properties()
-        trigger_properties.update(self.set_properties, "Trigger")
+        trigger_properties.update(properties, "Trigger")
         return trigger_properties
 
-    # Get all scheduler instances that were configured
-    # A tuple of (triggerables, invalidnames) is returned
-    def getSchedulers(self):
-        all_schedulers = self.build.builder.botmaster.parent.allSchedulers()
-        all_schedulers = dict([(sch.name, sch) for sch in all_schedulers])
-        invalid_schedulers = []
-        triggered_schedulers = []
-        # don't fire any schedulers if we discover an unknown one
-        for scheduler in self.schedulerNames:
-            scheduler = scheduler
-            if scheduler in all_schedulers:
-                sch = all_schedulers[scheduler]
-                if ITriggerableScheduler.providedBy(sch):
-                    triggered_schedulers.append(sch)
-                else:
-                    invalid_schedulers.append(scheduler)
-            else:
-                invalid_schedulers.append(scheduler)
-
-        return (triggered_schedulers, invalid_schedulers)
+    def getSchedulerByName(self, name):
+        # Use a quick cache to avoid generating this dict every time.
+        all_schedulers = self._all_schedulers
+        if all_schedulers is None:
+            all_schedulers = self.build.builder.botmaster.parent.allSchedulers()
+            all_schedulers = dict([(sch.name, sch) for sch in all_schedulers])
+            self._all_schedulers = all_schedulers
+
+        sch = all_schedulers.get(name)
+        if sch is not None:
+            if ITriggerableScheduler.providedBy(sch):
+                return sch
+
+        return None
 
     def prepareSourcestampListForTrigger(self):
         if self.sourceStamps:
@@ -135,24 +132,37 @@ def prepareSourcestampListForTrigger(self):
 
         return ss_for_trigger
 
+    def getSchedulersAndProperties(self):
+        return [(sched, self.set_properties) for sched in self.schedulerNames]
+
     @defer.inlineCallbacks
     def start(self):
+        schedulerNames_and_props = yield self.getSchedulersAndProperties()
+
         # Get all triggerable schedulers and check if there are invalid schedules
-        (triggered_schedulers, invalid_schedulers) = self.getSchedulers()
+        invalid_schedulers = []
+        schedulers_and_props = []
+        for name, props_to_set in schedulerNames_and_props:
+            sch = self.getSchedulerByName(name)
+            if sch is None:
+                invalid_schedulers.append(name)
+                continue
+
+            props_to_set = self.createTriggerProperties(props_to_set)
+            schedulers_and_props.append((sch, props_to_set))
+
         if invalid_schedulers:
             self.step_status.setText(['not valid scheduler:'] + invalid_schedulers)
             self.end(FAILURE)
             return
 
         self.running = True
 
-        props_to_set = self.createTriggerProperties()
-
         ss_for_trigger = self.prepareSourcestampListForTrigger()
 
         dl = []
         triggered_names = []
-        for sch in triggered_schedulers:
+        for sch, props_to_set in schedulers_and_props:
             dl.append(sch.trigger(ss_for_trigger, set_props=props_to_set))
             triggered_names.append(sch.name)
         self.step_status.setText(['triggered'] + triggered_names)

master/docs/manual/cfg-buildsteps.rst

@@ -2714,6 +2714,11 @@ For example::
 Triggering Schedulers
 ---------------------
 
+ .. py:class:: Trigger(schedulerNames=[], sourceStamp=None, sourceStamps=None,
+                  updateSourceStamp=None, alwaysUseLatest=False,
+                  waitForFinish=False, set_properties={},
+                  copy_properties=[])
+
 The counterpart to the Triggerable described in section :bb:Sched:`Triggerable` is the :bb:step:`Trigger` build step::
 
     from buildbot.plugins import steps
@@ -2723,32 +2728,63 @@ The counterpart to the Triggerable described in section :bb:Sched:`Triggerable`
                             updateSourceStamp=True,
                             set_properties={'quick' : False}))
 
-The ``schedulerNames=`` argument lists the :bb:sched:`Triggerable` schedulers that should be triggered when this step is executed.
-Note that it is possible, but not advisable, to create a cycle where a build continually triggers itself, because the schedulers are specified by name.
+The SourceStamps to use for the triggered build are controlled by the arguments ``updateSourceStamp``, ``alwaysUseLatest``, and ``sourceStamps``.
 
-If ``waitForFinish`` is ``True``, then the step will not finish until all of the builds from the triggered schedulers have finished.
-Hyperlinks are added to the waterfall and the build detail web pages for each triggered build.
-If this argument is ``False`` (the default) or not given, then the buildstep succeeds immediately after triggering the schedulers.
+Hyperlinks are added to the build detail web pages for each triggered build.
 
-The SourceStamps to use for the triggered build are controlled by the arguments ``updateSourceStamp``, ``alwaysUseLatest``, and ``sourceStamps``.
-If ``updateSourceStamp`` is ``True`` (the default), then step updates the source stamps given to the :bb:sched:`Triggerable` schedulers to include ``got_revision`` (the revision actually used in this build) as ``revision`` (the revision to use in the triggered builds).
-This is useful to ensure that all of the builds use exactly the same source stamps, even if other :class:`Change`\s have occurred while the build was running.
-If ``updateSourceStamp`` is False (and neither of the other arguments are specified), then the exact same SourceStamps are used.
-If ``alwaysUseLatest`` is True, then no SourceStamps are given, corresponding to using the latest revisions of the repositories specified in the Source steps.
-This is useful if the triggered builds use to a different source repository.
-The argument ``sourceStamps`` accepts a list of dictionaries containing the keys ``branch``, ``revision``, ``repository``, ``project``, and optionally ``patch_level``, ``patch_body``, ``patch_subdir``, ``patch_author`` and ``patch_comment`` and creates the corresponding SourceStamps.
-If only one sourceStamp has to be specified then the argument ``sourceStamp`` can be used for a dictionary containing the keys mentioned above.
-The arguments ``updateSourceStamp``, ``alwaysUseLatest``, and ``sourceStamp`` can be specified using properties.
-
-The ``set_properties`` parameter allows control of the properties that are passed to the triggered scheduler.
-The parameter takes a dictionary mapping property names to values.
-You may use :ref:`Interpolate` here to dynamically construct new property values.
-For the simple case of copying a property, this might look like::
-
-    set_properties={"my_prop1" : Property("my_prop1")}
+:param schedulerNames:
+    lists the :bb:sched:`Triggerable` schedulers that should be triggered when this step is executed.
+
+    .. note::
+
+        It is possible, but not advisable, to create a cycle where a build continually triggers itself, because the schedulers are specified by name.
+:param waitForFinish:
+    * If ``True``, the step will not finish until all of the builds from the triggered schedulers have finished.
+    * If ``False`` (the default) or not given, then the buildstep succeeds immediately after triggering the schedulers.
+
+:param updateSourceStamp:
+    * If ``True`` (the default), then step updates the source stamps given to the :bb:sched:`Triggerable` schedulers to include ``got_revision`` (the revision actually used in this build) as ``revision`` (the revision to use in the triggered builds).
+        This is useful to ensure that all of the builds use exactly the same source stamps, even if other :class:`Change`\s have occurred while the build was running.
+    * If ``False`` (and neither of the other arguments are specified), then the exact same SourceStamps are used.
+
+:param alwaysUseLatest:
+    If ``True``, then no SourceStamps are given, corresponding to using the latest revisions of the repositories specified in the Source steps.
+    This is useful if the triggered builds use to a different source repository.
+
+:param sourceStamps:
+    Accepts a list of dictionaries containing the keys ``branch``, ``revision``, ``repository``, ``project``, and optionally ``patch_level``, ``patch_body``, ``patch_subdir``, ``patch_author`` and ``patch_comment`` and creates the corresponding SourceStamps.
+    If only one sourceStamp has to be specified then the argument ``sourceStamp`` can be used for a dictionary containing the keys mentioned above.
+    The arguments ``updateSourceStamp``, ``alwaysUseLatest``, and ``sourceStamp`` can be specified using properties.
+
+:param set_properties:
+    allows control of the properties that are passed to the triggered scheduler.
+    The parameter takes a dictionary mapping property names to values.
+    You may use :ref:`Interpolate` here to dynamically construct new property values.
+    For the simple case of copying a property, this might look like::
+
+        set_properties={"my_prop1" : Property("my_prop1")}
 
 The ``copy_properties`` parameter, given a list of properties to copy into the new build request, has been deprecated in favor of explicit use of ``set_properties``.
 
+Dynamic Trigger
++++++++++++++++
+
+Sometimes it is desirable to select which scheduler to trigger, and which properties to set dynamically, at the time of the build.
+For this purpose, Trigger step support a method that you can customize in order to override statically defined ``schedulernames``, and ``set_properties``.
+
+.. py:method:: getSchedulersAndProperties()
+
+    :returns: list of tuples (schedulerName, propertiesDict) optionally via deferred
+
+    This methods returns a list of tuples describing what scheduler to trigger, with which properties.
+    The properties should already be rendered (ie, concrete value, not objects wrapped by ``Interpolate`` or
+    ``Property``). Since this function happens at build-time, the property values are available from the
+    step and can be used to decide what schedulers or properties to use.
+
+    With this method, you can also trigger the same scheduler multiple times with different set of properties.
+    The sourcestamp configuration is however the same for each triggered build request.
+
+
 RPM-Related Steps
 -----------------
 

master/docs/relnotes/index.rst

@@ -91,6 +91,9 @@ Features
 
 * GitHub change hook now supports pull_request events.
 
+* :class:`~buildbot.process.buildstep.Trigger`: the ``getSchedulersAndProperties`` customization method has been backported from Nine.
+  This provides a way to dynamically specify which schedulers (and the properties for that scheduler) to trigger at runtime.
+
 Fixes
 ~~~~~
 
@@ -110,6 +113,10 @@ Deprecations, Removals, and Non-Compatible Changes
 Changes for Developers
 ~~~~~~~~~~~~~~~~~~~~~~
 
+* :class:`~buildbot.process.buildstep.Trigger`: ``createTriggerProperties`` now takes one argument (the properties to generate).
+
+* :class:`~buildbot.process.buildstep.Trigger`: ``getSchedulers`` method is no longer used and was removed.
+
 Slave
 -----