support for non important trigger schedulers

master/buildbot/steps/trigger.py

@@ -47,9 +47,12 @@ class Trigger(BuildStep):
     def __init__(self, schedulerNames=None, sourceStamp=None, sourceStamps=None,
                  updateSourceStamp=None, alwaysUseLatest=False,
                  waitForFinish=False, set_properties=None,
-                 copy_properties=None, parent_relationship="Triggered from", **kwargs):
+                 copy_properties=None, parent_relationship="Triggered from",
+                 unimportantSchedulerNames=None, **kwargs):
         if schedulerNames is None:
             schedulerNames = []
+        if unimportantSchedulerNames is None:
+            unimportantSchedulerNames = []
         if not schedulerNames:
             config.error(
                 "You must specify a scheduler to trigger")
@@ -63,7 +66,13 @@ def __init__(self, schedulerNames=None, sourceStamp=None, sourceStamps=None,
             config.error(
                 "You can't specify both alwaysUseLatest and updateSourceStamp"
             )
+        if not set(schedulerNames).issuperset(set(unimportantSchedulerNames)):
+            config.error(
+                "unimportantSchedulerNames must be a subset of schedulerNames"
+            )
+
         self.schedulerNames = schedulerNames
+        self.unimportantSchedulerNames = unimportantSchedulerNames
         self.sourceStamps = sourceStamps or []
         if sourceStamp:
             self.sourceStamps.append(sourceStamp)
@@ -129,7 +138,11 @@ def getSchedulerByName(self, name):
     # This customization enpoint allows users to dynamically select which
     # scheduler and properties to trigger
     def getSchedulersAndProperties(self):
-        return [(sched, self.set_properties) for sched in self.schedulerNames]
+        return [{
+            'sched_name': sched,
+            'props_to_set': self.set_properties,
+            'unimportant': sched in self.unimportantSchedulerNames}
+            for sched in self.schedulerNames]
 
     def prepareSourcestampListForTrigger(self):
         if self.sourceStamps:
@@ -167,14 +180,20 @@ def getAllGotRevisions(self):
         return all_got_revisions
 
     @defer.inlineCallbacks
-    def worstStatus(self, overall_results, rclist):
+    def worstStatus(self, overall_results, rclist, unimportant_brids):
         for was_cb, results in rclist:
             if isinstance(results, tuple):
-                results, _ = results
+                results, brids_dict = results
 
             if not was_cb:
                 yield self.addLogWithFailure(results)
                 results = EXCEPTION
+
+            # brids_dict.values() rapresents the list of brids kicked by a certain scheduler.
+            # We want to ignore the result of ANY brid that was kicked off
+            # by an UNimportant scheduler.
+            if set(unimportant_brids).issuperset(set(brids_dict.values())):
+                continue
             overall_results = worst_status(overall_results, results)
         defer.returnValue(overall_results)
 
@@ -199,12 +218,30 @@ def addBuildUrls(self, rclist):
     def run(self):
         schedulers_and_props = yield self.getSchedulersAndProperties()
 
+        schedulers_and_props_list = []
+
+        # To be back compatible we need to differ between old and new style
+        # schedulers_and_props can either consist of 2 elements tuple or dictionary
+        for element in schedulers_and_props:
+            if isinstance(element, dict):
+                schedulers_and_props_list = schedulers_and_props
+                break
+            else:
+                # Old-style back compatibility: Convert tuple to dict and make it important
+                d = {
+                    'sched_name': element[0],
+                    'props_to_set': element[1],
+                    'unimportant': False
+                }
+                schedulers_and_props_list.append(d)
+
         # post process the schedulernames, and raw properties
         # we do this out of the loop, as this can result in errors
         schedulers_and_props = [(
-            self.getSchedulerByName(sch),
-            self.createTriggerProperties(props_to_set))
-            for sch, props_to_set in schedulers_and_props]
+            self.getSchedulerByName(entry_dict['sched_name']),
+            self.createTriggerProperties(entry_dict['props_to_set']),
+            entry_dict['unimportant'])
+            for entry_dict in schedulers_and_props_list]
 
         ss_for_trigger = self.prepareSourcestampListForTrigger()
 
@@ -213,7 +250,9 @@ def run(self):
         results = SUCCESS
         self.running = True
 
-        for sch, props_to_set in schedulers_and_props:
+        unimportant_brids = []
+
+        for sch, props_to_set, unimportant in schedulers_and_props:
             idsDeferred, resultsDeferred = sch.trigger(
                 waited_for=self.waitForFinish, sourcestamps=ss_for_trigger,
                 set_props=props_to_set,
@@ -228,7 +267,8 @@ def run(self):
             except Exception as e:
                 yield self.addLogWithException(e)
                 results = EXCEPTION
-
+            if unimportant:
+                unimportant_brids.extend(itervalues(brids))
             self.brids.extend(itervalues(brids))
             for brid in brids.values():
                 # put the url to the brids, so that we can have the status from
@@ -247,7 +287,7 @@ def run(self):
             if self.ended:
                 defer.returnValue(CANCELLED)
             yield self.addBuildUrls(rclist)
-            results = yield self.worstStatus(results, rclist)
+            results = yield self.worstStatus(results, rclist, unimportant_brids)
         else:
             # do something to handle errors
             for d in dl:

master/buildbot/test/unit/test_steps_trigger.py

@@ -112,10 +112,12 @@ def setupStep(self, step, sourcestampsInBuild=None, gotRevisionsInBuild=None, *a
 
         self.scheduler_a = a = FakeTriggerable(name='a')
         self.scheduler_b = b = FakeTriggerable(name='b')
-        m.scheduler_manager.namedServices = dict(a=a, b=b)
+        self.scheduler_c = c = FakeTriggerable(name='c')
+        m.scheduler_manager.namedServices = dict(a=a, b=b, c=c)
 
         a.brids = {77: 11}
         b.brids = {78: 22}
+        c.brids = {79: 33, 80: 44}
 
         make_fake_br = lambda brid, builderid: fakedb.BuildRequest(
             id=brid, buildsetid=BRID_TO_BSID(brid), builderid=builderid)
@@ -127,14 +129,21 @@ def setupStep(self, step, sourcestampsInBuild=None, gotRevisionsInBuild=None, *a
         m.db.insertTestData([
             fakedb.Builder(id=77, name='A'),
             fakedb.Builder(id=78, name='B'),
+            fakedb.Builder(id=79, name='C1'),
+            fakedb.Builder(id=80, name='C2'),
             fakedb.Master(id=9),
             fakedb.Buildset(id=2022),
             fakedb.Buildset(id=2011),
+            fakedb.Buildset(id=2033),
             fakedb.Worker(id=13, name="some:worker"),
             make_fake_br(11, 77),
             make_fake_br(22, 78),
+            fakedb.BuildRequest(id=33, buildsetid=2033, builderid=79),
+            fakedb.BuildRequest(id=44, buildsetid=2033, builderid=80),
             make_fake_build(11),
             make_fake_build(22),
+            make_fake_build(33),
+            make_fake_build(44),
         ])
 
         def getAllSourceStamps():
@@ -148,6 +157,7 @@ def getAllGotRevisions():
         self.exp_add_sourcestamp = None
         self.exp_a_trigger = None
         self.exp_b_trigger = None
+        self.exp_c_trigger = None
         self.exp_added_urls = []
 
     def runStep(self):
@@ -188,13 +198,16 @@ def wait(_):
 
         return d
 
-    def expectTriggeredWith(self, a=None, b=None):
+    def expectTriggeredWith(self, a=None, b=None, c=None):
         self.exp_a_trigger = a
         if a is not None:
             self.expectTriggeredLinks('a_br')
         self.exp_b_trigger = b
         if b is not None:
             self.expectTriggeredLinks('b_br')
+        self.exp_c_trigger = c
+        if c is not None:
+            self.expectTriggeredLinks('c_br')
 
     def expectAddedSourceStamp(self, **kwargs):
         self.exp_add_sourcestamp = kwargs
@@ -206,6 +219,11 @@ def expectTriggeredLinks(self, *args):
         if 'b_br' in args:
             self.exp_added_urls.append(
                 ('b #22', 'baseurl/#buildrequests/22'))
+        if 'c_br' in args:
+            self.exp_added_urls.append(
+                ('c #33', 'baseurl/#buildrequests/33'))
+            self.exp_added_urls.append(
+                ('c #44', 'baseurl/#buildrequests/44'))
         if 'a' in args:
             self.exp_added_urls.append(
                 ('success: A #4011', 'baseurl/#builders/77/builds/4011'))
@@ -221,6 +239,10 @@ def test_no_schedulerNames(self):
         self.assertRaises(config.ConfigErrors, lambda:
                           trigger.Trigger())
 
+    def test_unimportantSchedulerNames_not_in_schedulerNames(self):
+        self.assertRaises(config.ConfigErrors, lambda:
+                          trigger.Trigger(schedulerNames=['a'], unimportantsShedulerNames=['b']))
+
     def test_sourceStamp_and_updateSourceStamp(self):
         self.assertRaises(config.ConfigErrors, lambda:
                           trigger.Trigger(schedulerNames=['c'],
@@ -542,3 +564,31 @@ def test_waitForFinish_interrupt(self):
         self.step.interrupt(failure.Failure(RuntimeError('oh noes')))
 
         return d
+
+    def test_getSchedulersAndProperties_back_comp(self):
+        class DynamicTrigger(trigger.Trigger):
+            def getSchedulersAndProperties(self):
+                return [("a", {}, False), ("b", {}, True)]
+
+        self.setupStep(DynamicTrigger(schedulerNames=['a', 'b']))
+        self.scheduler_a.result = SUCCESS
+        self.scheduler_b.result = FAILURE
+        self.expectOutcome(result=SUCCESS, state_string='triggered a, b')
+        self.expectTriggeredWith(a=(False, [], {}), b=(False, [], {}))
+        return self.runStep()
+
+    def test_unimportantsShedulerNames(self):
+        self.setupStep(trigger.Trigger(schedulerNames=['a', 'b'], unimportantSchedulerNames=['b']))
+        self.scheduler_a.result = SUCCESS
+        self.scheduler_b.result = FAILURE
+        self.expectOutcome(result=SUCCESS, state_string='triggered a, b')
+        self.expectTriggeredWith(a=(False, [], {}), b=(False, [], {}))
+        return self.runStep()
+
+    def test_unimportantsShedulerNames_with_more_brids_for_bsid(self):
+        self.setupStep(trigger.Trigger(schedulerNames=['a', 'c'], unimportantSchedulerNames=['c']))
+        self.scheduler_a.result = SUCCESS
+        self.scheduler_c.result = FAILURE
+        self.expectOutcome(result=SUCCESS, state_string='triggered a, c')
+        self.expectTriggeredWith(a=(False, [], {}), c=(False, [], {}))
+        return self.runStep()

master/docs/manual/cfg-buildsteps.rst

@@ -2443,6 +2443,10 @@ Hyperlinks are added to the build detail web pages for each triggered build.
 
         It is possible, but not advisable, to create a cycle where a build continually triggers itself, because the schedulers are specified by name.
 
+``unimportantSchedulerNames``
+    When ``waitForFinish`` is ``True``, all schedulers in this list will not cause the trigger step to fail. unimportantSchedulerNames must be a subset of schedulerNames
+    If ``waitForFinish`` is ``False``, unimportantSchedulerNames will simply be ignored.
+
 ``waitForFinish``
     If ``True``, the step will not finish until all of the builds from the triggered schedulers have finished.
 
@@ -2479,13 +2483,14 @@ Dynamic Trigger
 +++++++++++++++
 
 Sometimes it is desirable to select which scheduler to trigger, and which properties to set dynamically, at the time of the build.
-For this purpose, Trigger step supports a method that you can customize in order to override statically defined ``schedulernames``, and ``set_properties``.
+For this purpose, Trigger step supports a method that you can customize in order to override statically defined ``schedulernames``, ``set_properties`` and optionally ``unimportant``.
 
 .. py:method:: getSchedulersAndProperties()
 
-    :returns: list of tuples (schedulerName, propertiesDict) optionally via deferred
+    :returns: list of dictionaries containing the keys 'sched_name', 'props_to_set' and 'unimportant' optionally via deferred
 
-    This methods returns a list of tuples describing what scheduler to trigger, with which properties.
+    This method returns a list of dictionaries describing what scheduler to trigger, with which properties and if the scheduler is unimportant.
+    Old style list of tuples is still supported, in which case unimportant is considered ``False``.
     The properties should already be rendered (ie, concrete value, not objects wrapped by ``Interpolate`` or
     ``Property``). Since this function happens at build-time, the property values are available from the
     step and can be used to decide what schedulers or properties to use.

master/docs/spelling_wordlist.txt

@@ -691,6 +691,8 @@ sautils
 sched
 schedulerid
 schedulerName
+schedulerNames
+unimportantSchedulerNames
 schemas
 scp
 searcheable