[#3332] Properly close database connections in background workers

RQ background workers exit via `os._exit`, which (by design) omits all
of Python's usual clean up routines. As a result, the SQLAlchemy session
and its associated database connections have to be cleaned up manually.
This hadn't been done before, leading to sporadic exceptions due to SSL
connections being unexpectedly closed.

This commit fixes that situation by properly closing each worker's
SQLAlchemy session and its associated database connections.

ckan/config/environment.py

@@ -284,6 +284,8 @@ def update_config():
     except sqlalchemy.exc.InternalError:
         # The database is not initialised.  Travis hits this
         pass
-    # if an extension or our code does not finish
-    # transaction properly db cli commands can fail
+
+    # Close current session and open database connections to ensure a clean
+    # clean environment even if an error occurs later on
     model.Session.remove()
+    model.Session.bind.dispose()

ckan/lib/jobs.py

@@ -29,6 +29,7 @@
 from ckan.lib.redis import connect_to_redis
 from ckan.common import config
 from ckan.config.environment import load_environment
+from ckan.model import meta
 
 
 log = logging.getLogger(__name__)
@@ -256,3 +257,18 @@ def main_work_horse(self, job, queue):
         # after forking.
         load_environment(config[u'global_conf'], config)
         return super(Worker, self).main_work_horse(job, queue)
+
+    def perform_job(self, *args, **kwargs):
+        result = super(Worker, self).perform_job(*args, **kwargs)
+        # rq.Worker.main_work_horse does a hard exit via os._exit directly
+        # after its call to perform_job returns. Hence here is the correct
+        # location to clean up.
+        try:
+            meta.Session.remove()
+        except Exception:
+            log.exception(u'Error while closing database session')
+        try:
+            meta.engine.dispose()
+        except Exception:
+            log.exception(u'Error while disposing database engine')
+        return result