Try to be slightly more accomodating of temporary thread leaks

master/buildbot/test/unit/test_db_connector.py

@@ -1,24 +1,22 @@
-import threading
-
 from twisted.trial import unittest
 
 from buildbot.db import dbspec, connector
+from buildbot.test.util import threads
 
-class DBConnector_Basic(unittest.TestCase):
+class DBConnector_Basic(threads.ThreadLeakMixin, unittest.TestCase):
     """
     Basic tests of the DBConnector class - all start with an empty DB
     """
 
     def setUp(self):
+        self.setUpThreadLeak()
         # use an in-memory sqlite database to test
         self.dbc = connector.DBConnector(dbspec.DBSpec.from_url("sqlite://"))
         self.dbc.start()
-        self.start_thdcount = len(threading.enumerate())
 
     def tearDown(self):
         self.dbc.stop()
-        # double-check we haven't left a ThreadPool open
-        assert len(threading.enumerate()) - self.start_thdcount < 1
+        self.tearDownThreadLeak()
 
     def test_quoteq_format(self):
         self.dbc.paramstyle = "format" # override default

master/buildbot/test/unit/test_db_dbspec.py

@@ -1,9 +1,9 @@
 import os
-import threading
 
 from twisted.trial import unittest
 
 from buildbot.db import dbspec
+from buildbot.test.util import threads
 
 class DBSpec(unittest.TestCase):
     # a dburl of "sqlite:///.." can use either the third-party sqlite3
@@ -126,19 +126,18 @@ def test_fromURL_mysqlAuthNoPassPortArgs(self):
                 connkw=dict(host='somehost.com', db='dbname', user="user",
                             port=8000, foo="moo", use_unicode=True, charset='utf8'))
 
-class DBSpec_methods(unittest.TestCase):
+class DBSpec_methods(threads.ThreadLeakMixin, unittest.TestCase):
 
     def setUp(self):
+        self.setUpThreadLeak()
         self.spec = dbspec.DBSpec.from_url("sqlite://")
         self.pools = []
-        self.start_thdcount = len(threading.enumerate())
 
     def tearDown(self):
         # be careful to stop all pools
         for pool in self.pools:
             pool.close()
-        # double-check we haven't left a ThreadPool open
-        assert len(threading.enumerate()) - self.start_thdcount < 1
+        self.tearDownThreadLeak()
 
     # track a pool that must be closed
     def trackPool(self, pool):

master/buildbot/test/util/threads.py

@@ -0,0 +1,22 @@
+import threading
+import time
+
+from twisted.python import log
+
+class ThreadLeakMixin(object):
+    """
+    Monitor for leaking thread pools. Just call the setUp and tearDown methods!
+    """
+    def setUpThreadLeak(self):
+        self.start_thdcount = len(threading.enumerate())
+
+    def tearDownThreadLeak(self):
+        # double-check we haven't left a ThreadPool open.  Sometimes, threads
+        # take a while to go away, so this will wait up to 5s for that to occur
+        for time in xrange(5):
+            cur_thdcount = len(threading.enumerate())
+            if cur_thdcount - self.start_thdcount < 1:
+                return
+            log.msg("threadcount: %d (start) %d (now)" % (self.start_thdcount, cur_thdcount))
+            time.sleep(1)
+        self.fail("leaked %d threads" % (cur_thdcount - self.start_thdcount))