Merge pull request #3163 from ckan/2842-common-config

[#2842] New CKAN config object, decoupled from Pylons

ckan/authz.py

@@ -4,7 +4,7 @@
 import re
 from logging import getLogger
 
-from pylons import config
+from ckan.common import config
 from paste.deploy.converters import asbool
 
 import ckan.plugins as p

ckan/ckan_nose_plugin.py

@@ -8,7 +8,7 @@
 import re
 import pkg_resources
 from paste.deploy import loadapp
-from pylons import config
+from ckan.common import config
 import unittest
 import time
 

ckan/common.py

@@ -8,6 +8,12 @@
 # NOTE:  This file is specificaly created for
 # from ckan.common import x, y, z to be allowed
 
+from collections import MutableMapping
+
+import flask
+import pylons
+
+from werkzeug.local import Local
 
 from pylons.i18n import _, ungettext
 from pylons import g, c, request, session, response
@@ -17,3 +23,78 @@
     from collections import OrderedDict  # from python 2.7
 except ImportError:
     from sqlalchemy.util import OrderedDict
+
+
+class CKANConfig(MutableMapping):
+    u'''Main CKAN configuration object
+
+    This is a dict-like object that also proxies any changes to the
+    Flask and Pylons configuration objects.
+
+    The actual `config` instance in this module is initialized in the
+    `load_environment` method with the values of the ini file or env vars.
+
+    '''
+
+    def __init__(self, *args, **kwargs):
+        self.store = dict()
+        self.update(dict(*args, **kwargs))
+
+    def __getitem__(self, key):
+        return self.store[key]
+
+    def __iter__(self):
+        return iter(self.store)
+
+    def __len__(self):
+        return len(self.store)
+
+    def __repr__(self):
+        return self.store.__repr__()
+
+    def copy(self):
+        return self.store.copy()
+
+    def clear(self):
+        self.store.clear()
+
+        try:
+            flask.current_app.config.clear()
+        except RuntimeError:
+            pass
+        try:
+            pylons.config.clear()
+            # Pylons set this default itself
+            pylons.config[u'lang'] = None
+        except TypeError:
+            pass
+
+    def __setitem__(self, key, value):
+        self.store[key] = value
+        try:
+            flask.current_app.config[key] = value
+        except RuntimeError:
+            pass
+        try:
+            pylons.config[key] = value
+        except TypeError:
+            pass
+
+    def __delitem__(self, key):
+        del self.store[key]
+        try:
+            del flask.current_app.config[key]
+        except RuntimeError:
+            pass
+        try:
+            del pylons.config[key]
+        except TypeError:
+            pass
+
+local = Local()
+
+# This a proxy to the bounded config object
+local(u'config')
+
+# Thread-local safe objects
+config = local.config = CKANConfig()

ckan/config/environment.py

@@ -1,14 +1,14 @@
 # encoding: utf-8
 
-"""Pylons environment configuration"""
+'''CKAN environment configuration'''
 import os
 import logging
 import warnings
 from urlparse import urlparse
 import pytz
 
 import sqlalchemy
-from pylons import config
+from pylons import config as pylons_config
 import formencode
 
 import ckan.config.routing as routing
@@ -22,7 +22,7 @@
 import ckan.authz as authz
 import ckan.lib.jinja_extensions as jinja_extensions
 
-from ckan.common import _, ungettext
+from ckan.common import _, ungettext, config
 from ckan.exceptions import CkanConfigurationException
 
 log = logging.getLogger(__name__)
@@ -72,8 +72,17 @@ def find_controller(self, controller):
                  static_files=os.path.join(root, 'public'),
                  templates=[])
 
-    # Initialize config with the basic options
-    config.init_app(global_conf, app_conf, package='ckan', paths=paths)
+    # Initialize main CKAN config object
+    config.update(global_conf)
+    config.update(app_conf)
+
+    # Initialize Pylons own config object
+    pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths)
+
+    # Update the main CKAN config object with the Pylons specific stuff, as it
+    # quite hard to keep them separated. This should be removed once Pylons
+    # support is dropped
+    config.update(pylons_config)
 
     # Setup the SQLAlchemy database engine
     # Suppress a couple of sqlalchemy warnings
@@ -85,8 +94,9 @@ def find_controller(self, controller):
         warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
 
     # load all CKAN plugins
-    p.load_all(config)
+    p.load_all()
 
+    app_globals.reset()
 
 # A mapping of config settings that can be overridden by env vars.
 # Note: Do not remove the following lines, they are used in the docs
@@ -184,10 +194,11 @@ def update_config():
     # The RoutesMiddleware needs its mapper updating if it exists
     if 'routes.middleware' in config:
         config['routes.middleware'].mapper = routes_map
+    # routes.named_routes is a CKAN thing
     config['routes.named_routes'] = routing.named_routes
     config['pylons.app_globals'] = app_globals.app_globals
     # initialise the globals
-    config['pylons.app_globals']._init()
+    app_globals.app_globals._init()
 
     helpers.load_plugin_helpers()
     config['pylons.h'] = helpers.helper_functions

ckan/config/middleware/__init__.py

@@ -6,6 +6,7 @@
 
 from werkzeug.test import create_environ, run_wsgi_app
 
+from ckan.config.environment import load_environment
 from ckan.config.middleware.flask_app import make_flask_stack
 from ckan.config.middleware.pylons_app import make_pylons_stack
 
@@ -41,8 +42,11 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
     middleware.
     '''
 
-    pylons_app = make_pylons_stack(conf, full_stack, static_files, **app_conf)
-    flask_app = make_flask_stack(conf)
+    load_environment(conf, app_conf)
+
+    pylons_app = make_pylons_stack(conf, full_stack, static_files,
+                                   **app_conf)
+    flask_app = make_flask_stack(conf, **app_conf)
 
     app = AskAppDispatcherMiddleware({'pylons_app': pylons_app,
                                       'flask_app': flask_app})
@@ -119,4 +123,13 @@ def __call__(self, environ, start_response):
 
         log.debug('Serving request via {0} app'.format(app_name))
         environ['ckan.app'] = app_name
-        return self.apps[app_name](environ, start_response)
+        if app_name == 'flask_app':
+            return self.apps[app_name](environ, start_response)
+        else:
+            # Although this request will be served by Pylons we still
+            # need an application context in order for the Flask URL
+            # builder to work and to be able to access the Flask config
+            flask_app = self.apps['flask_app']._wsgi_app
+
+            with flask_app.app_context():
+                return self.apps[app_name](environ, start_response)

ckan/config/middleware/flask_app.py

@@ -3,17 +3,27 @@
 from flask import Flask
 from werkzeug.exceptions import HTTPException
 
+from ckan.common import config
+
 
 import logging
 log = logging.getLogger(__name__)
 
 
-def make_flask_stack(conf):
+def make_flask_stack(conf, **app_conf):
     """ This has to pass the flask app through all the same middleware that
     Pylons used """
 
     app = flask_app = CKANFlask(__name__)
 
+    # Update Flask config with the CKAN values. We use the common config
+    # object as values might have been modified on `load_environment`
+    if config:
+        app.config.update(config)
+    else:
+        app.config.update(conf)
+        app.config.update(app_conf)
+
     @app.route('/hello', methods=['GET'])
     def hello_world():
         return 'Hello World, this is served by Flask'

ckan/config/middleware/pylons_app.py

@@ -2,7 +2,6 @@
 
 import os
 
-from pylons import config
 from pylons.wsgiapp import PylonsApp
 
 from beaker.middleware import CacheMiddleware, SessionMiddleware
@@ -19,15 +18,15 @@
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IMiddleware
 import ckan.lib.uploader as uploader
-from ckan.config.environment import load_environment
-import ckan.lib.app_globals as app_globals
 from ckan.config.middleware import common_middleware
+from ckan.common import config
 
 import logging
 log = logging.getLogger(__name__)
 
 
-def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf):
+def make_pylons_stack(conf, full_stack=True, static_files=True,
+                      **app_conf):
     """Create a Pylons WSGI application and return it
 
     ``conf``
@@ -50,15 +49,9 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf):
         defaults to main).
 
     """
-    # Configure the Pylons environment
-    load_environment(conf, app_conf)
-
     # The Pylons WSGI app
     app = pylons_app = CKANPylonsApp()
 
-    # set pylons globals
-    app_globals.reset()
-
     for plugin in PluginImplementations(IMiddleware):
         app = plugin.make_middleware(app, config)
 
@@ -72,7 +65,8 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf):
 
     # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
     # app = QueueLogMiddleware(app)
-    if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)):
+    if asbool(config.get('ckan.use_pylons_response_cleanup_middleware',
+                         True)):
         app = execute_on_completion(app, config,
                                     cleanup_pylons_response_string)
 
@@ -138,11 +132,13 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf):
 
     if asbool(static_files):
         # Serve static files
-        static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \
+        static_max_age = None if not asbool(
+            config.get('ckan.cache_enabled')) \
             else int(config.get('ckan.static_max_age', 3600))
 
-        static_app = StaticURLParser(config['pylons.paths']['static_files'],
-                                     cache_max_age=static_max_age)
+        static_app = StaticURLParser(
+            config['pylons.paths']['static_files'],
+            cache_max_age=static_max_age)
         static_parsers = [static_app, app]
 
         storage_directory = uploader.get_storage_path()
@@ -160,7 +156,8 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf):
 
         # Configurable extra static file paths
         extra_static_parsers = []
-        for public_path in config.get('extra_public_paths', '').split(','):
+        for public_path in config.get(
+                'extra_public_paths', '').split(','):
             if public_path.strip():
                 extra_static_parsers.append(
                     StaticURLParser(public_path.strip(),

ckan/controllers/admin.py

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-from pylons import config
+from ckan.common import config
 
 import ckan.lib.base as base
 import ckan.lib.helpers as h

ckan/controllers/feed.py

@@ -25,7 +25,7 @@
 import urlparse
 
 import webhelpers.feedgenerator
-from pylons import config
+from ckan.common import config
 
 import ckan.model as model
 import ckan.lib.base as base

ckan/controllers/home.py

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-from pylons import config, cache
+from pylons import cache
 import sqlalchemy.exc
 
 import ckan.logic as logic

ckan/controllers/package.py

@@ -6,7 +6,7 @@
 import mimetypes
 import cgi
 
-from pylons import config
+from ckan.common import config
 from paste.deploy.converters import asbool
 import paste.fileapp
 

ckan/controllers/tag.py

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-from pylons import config
+from ckan.common import config
 from paste.deploy.converters import asbool
 
 import ckan.logic as logic

ckan/controllers/user.py

@@ -3,7 +3,7 @@
 import logging
 from urllib import quote
 
-from pylons import config
+from ckan.common import config
 from paste.deploy.converters import asbool
 
 import ckan.lib.base as base

ckan/lib/activity_streams_session_extension.py

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-from pylons import config
+from ckan.common import config
 from sqlalchemy.orm.session import SessionExtension
 from paste.deploy.converters import asbool
 import logging