Make synchronous connections obey max_idle as well

buildbot/db/connector.py

@@ -93,6 +93,8 @@ def __init__(self, spec):
         # this is for synchronous calls: runQueryNow, runInteractionNow
         self._dbapi = spec.get_dbapi()
         self._nonpool = None
+        self._nonpool_lastused = None
+        self._nonpool_max_idle = spec.get_maxidle()
 
         # pass queries in with "?" placeholders. If the backend uses a
         # different style, we'll replace them.
@@ -117,7 +119,7 @@ def __init__(self, spec):
 
     def _getCurrentTime(self):
         # this is a seam for use in testing
-        return time.time()
+        return util.now()
 
     def start(self):
         # this only *needs* to be called in reactorless environments (which
@@ -205,23 +207,38 @@ def runInteractionNow(self, interaction, *args, **kwargs):
             self._end_operation(t)
             self._add_query_time(start)
 
-    def _runInteractionNow(self, interaction, *args, **kwargs):
+    def get_sync_connection(self):
+        # This is a wrapper around spec.get_sync_connection that maintains a
+        # single connection to the database for synchronous usage.  It will get
+        # a new connection if the existing one has been idle for more than
+        # max_idle seconds.
+        if self._nonpool_max_idle is not None:
+            now = util.now()
+            if self._nonpool_lastused and self._nonpool_lastused + self._nonpool_max_idle < now:
+                self._nonpool = None
+
         if not self._nonpool:
             self._nonpool = self._spec.get_sync_connection()
-        c = self._nonpool.cursor()
+
+        self._nonpool_lastused = util.now()
+        return self._nonpool
+
+    def _runInteractionNow(self, interaction, *args, **kwargs):
+        conn = self.get_sync_connection()
+        c = conn.cursor()
         try:
             result = interaction(c, *args, **kwargs)
             c.close()
-            self._nonpool.commit()
+            conn.commit()
             return result
         except:
             excType, excValue, excTraceback = sys.exc_info()
             try:
-                self._nonpool.rollback()
-                c2 = self._nonpool.cursor()
+                conn.rollback()
+                c2 = conn.cursor()
                 c2.execute(self._pool.good_sql)
                 c2.close()
-                self._nonpool.commit()
+                conn.commit()
             except:
                 log.msg("rollback failed, will reconnect next query")
                 log.err()

buildbot/db/dbspec.py

@@ -45,6 +45,7 @@
 
 from buildbot.db.connector import DBConnector
 from buildbot.db.exceptions import *
+from buildbot import util
 
 class ExpiringConnectionPool(adbapi.ConnectionPool):
     """
@@ -66,7 +67,7 @@ def __init__(self, dbapiName, max_idle=60, *args, **kwargs):
 
     def connect(self):
         tid = self.threadID()
-        now = time.time()
+        now = util.now()
         lastused = self.connection_lastused.get(tid)
         if lastused and lastused + self.max_idle < now:
             conn = self.connections.get(tid)
@@ -178,15 +179,15 @@ def _get_sqlite_dbapi_name(self):
 
     def get_dbapi(self):
         """
-        Get the dbapi module used for this connection (for things like exceptions
-        and module-global attributes
+        Get the dbapi module used for this connection (for things like
+        exceptions and module-global attributes
         """
         return reflect.namedModule(self.dbapiName)
 
     def get_sync_connection(self):
         """
-        Get a synchronous connection to the specified database.  This returns a simple
-        DBAPI connection object.
+        Get a synchronous connection to the specified database.  This returns
+        a simple DBAPI connection object.
         """
         dbapi = self.get_dbapi()
         connkw = self.connkw.copy()
@@ -198,7 +199,8 @@ def get_sync_connection(self):
 
     def get_async_connection_pool(self):
         """
-        Get an asynchronous (adbapi) connection pool for the specified database.
+        Get an asynchronous (adbapi) connection pool for the specified
+        database.
         """
 
         # add some connection keywords
@@ -220,3 +222,6 @@ def get_async_connection_pool(self):
             return ExpiringConnectionPool(self.dbapiName, *self.connargs, **connkw)
         else:
             return adbapi.ConnectionPool(self.dbapiName, *self.connargs, **connkw)
+
+    def get_maxidle(self):
+        return self.connkw.get("max_idle")