Merge branch 'master' of git@github.com:djmitche/buildbot

* 'master' of git@github.com:djmitche/buildbot:
  Fixing recursion problems in loop processing

buildbot/broken_test/runutils.py

@@ -107,6 +107,12 @@ def tearDown(self):
         d.addCallback(lambda ign: self.master.stopService())
         d.addCallback(self.wait_until_idle)
         d.addCallback(lambda ign: log.msg("tearDown done"))
+
+        # Unset our master instance so that future invocations of this TestCase
+        # will work as expected (e.g. when doing trial -u)
+        def _done(ign):
+            self.master = None
+        d.addCallback(_done)
         return d
 
     def connectOneSlave(self, slavename, opts={}):

buildbot/broken_test/unit/test_config.py

@@ -23,6 +23,7 @@
 from buildbot.status import base
 from buildbot.steps import dummy, maxq, python, python_twisted, shell, \
      source, transfer
+from buildbot.util import eventual
 words = None
 try:
     from buildbot.status import words
@@ -789,6 +790,7 @@ def _check1(ign):
         d.addCallback(_check1)
 
         d.addCallback(lambda ign: master.loadConfig(buildersCfg))
+        d.addCallback(eventual.flushEventualQueue)
         def _check2(ign):
             self.failUnlessEqual(bm.builderNames, ["builder1"])
             self.failUnlessEqual(bm.builders.keys(), ["builder1"])
@@ -812,6 +814,7 @@ def _check2(ign):
         d.addCallback(_check2)
         # make sure a reload of the same data doesn't interrupt the Builder
         d.addCallback(lambda ign: master.loadConfig(buildersCfg))
+        d.addCallback(eventual.flushEventualQueue)
         def _check3(ign):
             self.failUnlessEqual(bm.builderNames, ["builder1"])
             self.failUnlessEqual(bm.builders.keys(), ["builder1"])
@@ -824,6 +827,7 @@ def _check3(ign):
 
         # but changing something should result in a new Builder
         d.addCallback(lambda ign: master.loadConfig(buildersCfg2))
+        d.addCallback(eventual.flushEventualQueue)
         def _check4(ign):
             self.failUnlessEqual(bm.builderNames, ["builder1"])
             self.failUnlessEqual(bm.builders.keys(), ["builder1"])
@@ -836,6 +840,7 @@ def _check4(ign):
 
         # adding new builder
         d.addCallback(lambda ign: master.loadConfig(buildersCfg3))
+        d.addCallback(eventual.flushEventualQueue)
         def _check5(ign):
             self.failUnlessEqual(bm.builderNames, ["builder1", "builder2"])
             self.failUnlessListsEquivalent(bm.builders.keys(),
@@ -846,6 +851,7 @@ def _check5(ign):
 
         # changing first builder should leave it at the same place in the list
         d.addCallback(lambda ign: master.loadConfig(buildersCfg4))
+        d.addCallback(eventual.flushEventualQueue)
         def _check6(ign):
             self.failUnlessEqual(bm.builderNames, ["builder1", "builder2"])
             self.failUnlessListsEquivalent(bm.builders.keys(),
@@ -855,6 +861,7 @@ def _check6(ign):
         d.addCallback(_check6)
 
         d.addCallback(lambda ign: master.loadConfig(buildersCfg5))
+        d.addCallback(eventual.flushEventualQueue)
         def _check6a(ign):
             self.failUnlessEqual(bm.builderNames, ["builder1", "builder2"])
             self.failUnlessListsEquivalent(bm.builders.keys(),
@@ -863,6 +870,7 @@ def _check6a(ign):
 
         # and removing it should make the Builder go away
         d.addCallback(lambda ign: master.loadConfig(emptyCfg))
+        d.addCallback(eventual.flushEventualQueue)
         def _check7(ign):
             self.failUnlessEqual(bm.builderNames, [])
             self.failUnlessEqual(bm.builders, {})
@@ -1063,19 +1071,22 @@ def testLocks(self):
 
         # create a Builder that uses Locks
         d.addCallback(lambda ign: master.loadConfig(lockCfg1a))
+        d.addCallback(eventual.flushEventualQueue)
         def _check1(ign):
             self.b1 = botmaster.builders["builder1"]
             self.failUnlessEqual(len(self.b1.locks), 2)
         d.addCallback(_check1)
 
         # reloading the same config should not change the Builder
         d.addCallback(lambda ign: master.loadConfig(lockCfg1a))
+        d.addCallback(eventual.flushEventualQueue)
         def _check2(ign):
             self.failUnlessIdentical(self.b1, botmaster.builders["builder1"])
         d.addCallback(_check2)
 
         # but changing the set of locks used should change it
         d.addCallback(lambda ign: master.loadConfig(lockCfg1b))
+        d.addCallback(eventual.flushEventualQueue)
         def _check3(ign):
             self.failIfIdentical(self.b1, botmaster.builders["builder1"])
             self.b1 = botmaster.builders["builder1"]
@@ -1084,25 +1095,29 @@ def _check3(ign):
 
         # similar test with step-scoped locks
         d.addCallback(lambda ign: master.loadConfig(lockCfg2a))
+        d.addCallback(eventual.flushEventualQueue)
 
         def _check4(ign):
             self.b1 = botmaster.builders["builder1"]
         d.addCallback(_check4)
 
         # reloading the same config should not change the Builder
         d.addCallback(lambda ign: master.loadConfig(lockCfg2a))
+        d.addCallback(eventual.flushEventualQueue)
         def _check5(ign):
             self.failUnlessIdentical(self.b1, botmaster.builders["builder1"])
 
         # but changing the set of locks used should change it
         d.addCallback(lambda ign: master.loadConfig(lockCfg2b))
+        d.addCallback(eventual.flushEventualQueue)
         def _check6(ign):
             self.failIfIdentical(self.b1, botmaster.builders["builder1"])
             self.b1 = botmaster.builders["builder1"]
         d.addCallback(_check6)
 
         # remove the locks entirely
         d.addCallback(lambda ign: master.loadConfig(lockCfg2c))
+        d.addCallback(eventual.flushEventualQueue)
         def _check7(ign):
             self.failIfIdentical(self.b1, master.botmaster.builders["builder1"])
         d.addCallback(_check7)

buildbot/test/unit/test_util_loop.py

@@ -42,7 +42,7 @@
 
 from buildbot.test.state import State
 
-from buildbot.util import loop
+from buildbot.util import loop, eventual
 
 class TestLoopMixin(object):
     def setUpTestLoop(self, skipConstructor=False):
@@ -52,6 +52,7 @@ def setUpTestLoop(self, skipConstructor=False):
         self.results = []
 
     def tearDownTestLoop(self):
+        eventual._setReactor(None)
         assert self.loop.is_quiet()
         return self.loop.stopService()
 
@@ -100,6 +101,15 @@ def check(res):
             self.assertEqual(sorted(res), ['x', 'y', 'z'])
         return self.whenQuiet(check)
 
+    def test_big_multi_processor(self):
+        for i in range(300):
+            self.loop.add(self.make_cb(i))
+        def check(res):
+            self.assertEqual(sorted(res), range(300))
+        self.loop.trigger()
+        d = self.whenQuiet(check)
+        return d
+
     def test_simple_selfTrigger(self):
         self.loop.add(self.make_cb('c', selfTrigger=3))
         self.loop.trigger()
@@ -137,6 +147,7 @@ def check(res):
 
     def test_sleep(self):
         reactor = self.loop._reactor = task.Clock()
+        eventual._setReactor(reactor)
         state = State(count=5)
         def proc():
             self.results.append(reactor.seconds())
@@ -145,8 +156,12 @@ def proc():
                 return defer.succeed(reactor.seconds() + 10.0)
         self.loop.add(proc)
         self.loop.trigger()
-        reactor.pump((0,) + (1,)*50) # run for 50 fake seconds
-        self.assertEqual(self.results, [ 0.0, 10.0, 20.0, 30.0, 40.0 ])
+        def check(ign):
+            reactor.pump((0,) + (1,)*50) # run for 50 fake seconds
+            self.assertEqual(self.results, [ 0.0, 10.0, 20.0, 30.0, 40.0 ])
+        d = eventual.flushEventualQueue()
+        d.addCallback(check)
+        return d
 
     def test_loop_done(self):
         # monkey-patch the instance to have a 'loop_done' method

buildbot/util/eventual.py

@@ -5,6 +5,9 @@
 from twisted.python import log
 
 class _SimpleCallQueue(object):
+
+    _reactor = reactor
+
     def __init__(self):
         self._events = []
         self._flushObservers = []
@@ -14,7 +17,7 @@ def __init__(self):
     def append(self, cb, args, kwargs):
         self._events.append((cb, args, kwargs))
         if not self._timer:
-            self._timer = reactor.callLater(0, self._turn)
+            self._timer = self._reactor.callLater(0, self._turn)
 
     def _turn(self):
         self._timer = None
@@ -30,7 +33,7 @@ def _turn(self):
                 log.err()
         self._in_turn = False
         if self._events and not self._timer:
-            self._timer = reactor.callLater(0, self._turn)
+            self._timer = self._reactor.callLater(0, self._turn)
         if not self._events:
             observers, self._flushObservers = self._flushObservers, []
             for o in observers:
@@ -79,3 +82,13 @@ def flushEventualQueue(_ignored=None):
     test method.
     """
     return _theSimpleQueue.flush()
+
+def _setReactor(r=None):
+    """This sets the reactor used to schedule future events to r.  If r is None
+    (the default), the reactor is reset to its default value.
+
+    This should only be used for unit tests.
+    """
+    if r is None:
+        r = reactor
+    _theSimpleQueue._reactor = r

buildbot/util/loop.py

@@ -84,6 +84,8 @@
 from twisted.application import service
 from twisted.python import log
 
+from buildbot.util.eventual import eventually
+
 class LoopBase(service.MultiService):
     OCD_MINIMUM_DELAY = 5.0
 
@@ -157,8 +159,9 @@ def _loop_next(self):
         d.addErrback(log.err)
         d.addBoth(self._one_done)
         return None # no long Deferred chains
+
     def _one_done(self, ignored):
-        self._loop_next()
+        eventually(self._loop_next)
 
     def _loop_done(self):
         if self._everything_needs_to_run: