Merge branch 'master' into 3196-common-requests-code

ckan/config/environment.py

@@ -104,6 +104,11 @@ def find_controller(self, controller):
 
     app_globals.reset()
 
+    # issue #3260: remove idle transaction
+    # Session that was used for getting all config params nor committed,
+    # neither removed and we have idle connection as result
+    model.Session.commit()
+
     # Build JavaScript translations. Must be done after plugins have
     # been loaded.
     build_js_translations()
@@ -284,6 +289,8 @@ def update_config():
     except sqlalchemy.exc.InternalError:
         # The database is not initialised.  Travis hits this
         pass
-    # if an extension or our code does not finish
-    # transaction properly db cli commands can fail
+
+    # Close current session and open database connections to ensure a clean
+    # clean environment even if an error occurs later on
     model.Session.remove()
+    model.Session.bind.dispose()

ckan/config/routing.py

@@ -93,6 +93,10 @@ def make_map():
     map.minimization = False
     map.explicit = True
 
+    # CUSTOM ROUTES HERE
+    for plugin in p.PluginImplementations(p.IRoutes):
+        map = plugin.before_map(map)
+
     # The ErrorController route (handles 404/500 error pages); it should
     # likely stay at the top, ensuring it can always be resolved.
     map.connect('/error/{action}', controller='error', ckan_core=True)
@@ -101,10 +105,6 @@ def make_map():
     map.connect('*url', controller='home', action='cors_options',
                 conditions=OPTIONS, ckan_core=True)
 
-    # CUSTOM ROUTES HERE
-    for plugin in p.PluginImplementations(p.IRoutes):
-        map = plugin.before_map(map)
-
     # Mark all routes added from extensions on the `before_map` extension point
     # as non-core
     for route in map.matchlist:

ckan/lib/i18n.py

@@ -381,10 +381,20 @@ def build_js_translations():
 
     # Build translations for each language
     for lang in sorted(langs):
-        po_files = [os.path.join(i18n_dir, lang, u'LC_MESSAGES',
-                                 domain + u'.po')
-                    for i18n_dir, domain in i18n_dirs.iteritems()]
-        po_files = [fn for fn in po_files if os.path.isfile(fn)]
+        po_files = [
+            fn for fn in (
+                os.path.join(
+                    i18n_dir,
+                    lang,
+                    u'LC_MESSAGES',
+                    domain + u'.po'
+                )
+                for i18n_dir, domain in i18n_dirs.iteritems()
+            ) if os.path.isfile(fn)
+        ]
+        if not po_files:
+            continue
+
         latest = max(os.path.getmtime(fn) for fn in po_files)
         dest_file = os.path.join(_JS_TRANSLATIONS_DIR, lang + u'.js')
         if os.path.isfile(dest_file) and os.path.getmtime(dest_file) > latest:

ckan/lib/jobs.py

@@ -29,6 +29,7 @@
 from ckan.lib.redis import connect_to_redis
 from ckan.common import config
 from ckan.config.environment import load_environment
+from ckan.model import meta
 
 
 log = logging.getLogger(__name__)
@@ -256,3 +257,18 @@ def main_work_horse(self, job, queue):
         # after forking.
         load_environment(config[u'global_conf'], config)
         return super(Worker, self).main_work_horse(job, queue)
+
+    def perform_job(self, *args, **kwargs):
+        result = super(Worker, self).perform_job(*args, **kwargs)
+        # rq.Worker.main_work_horse does a hard exit via os._exit directly
+        # after its call to perform_job returns. Hence here is the correct
+        # location to clean up.
+        try:
+            meta.Session.remove()
+        except Exception:
+            log.exception(u'Error while closing database session')
+        try:
+            meta.engine.dispose()
+        except Exception:
+            log.exception(u'Error while disposing database engine')
+        return result

ckan/logic/schema.py

@@ -323,7 +323,6 @@ def group_form_schema():
         "capacity": [ignore_missing],
         "__extras": [ignore]
     }
-    schema['display_name'] = [ignore_missing]
     return schema
 
 

ckan/public/base/i18n/.gitignore

@@ -0,0 +1,6 @@
+// This directory contains CKAN's automatically generated JavaScript
+// translation files, which are ignored by git. This file makes sure that the
+// directory itself is tracked.
+
+*.js
+