[#547] Rework the plugins and config stuff

ckan/config/environment.py

@@ -19,6 +19,9 @@
 import ckan.lib.helpers as h
 import ckan.lib.app_globals as app_globals
 import ckan.lib.render as render
+import ckan.lib.search as search
+import ckan.logic as logic
+import ckan.new_authz as new_authz
 
 log = logging.getLogger(__name__)
 
@@ -34,6 +37,11 @@ class _Helpers(object):
     templates have helper functions provided by extensions that have
     not been enabled. '''
     def __init__(self, helpers):
+        self.helpers = helpers
+        self._setup()
+
+    def _setup(self):
+        helpers = self.helpers
         functions = {}
         allowed = helpers.__allowed_functions__[:]
         # list of functions due to be deprecated
@@ -92,7 +100,7 @@ def __getattr__(self, name):
 
 def load_environment(global_conf, app_conf):
     """Configure the Pylons environment via the ``pylons.config``
-    object
+    object.  This code should only need to be run once.
     """
 
     ######  Pylons monkey-patch
@@ -127,93 +135,8 @@ def find_controller(self, controller):
                  templates=[])
 
     # Initialize config with the basic options
-
     config.init_app(global_conf, app_conf, package='ckan', paths=paths)
 
-    # load all CKAN plugins
-    p.load_all(config)
-
-    # Load the synchronous search plugin, unless already loaded or
-    # explicitly disabled
-    if not 'synchronous_search' in config.get('ckan.plugins',[]) and \
-            asbool(config.get('ckan.search.automatic_indexing', True)):
-        log.debug('Loading the synchronous search plugin')
-        p.load('synchronous_search')
-
-    for plugin in p.PluginImplementations(p.IConfigurer):
-        # must do update in place as this does not work:
-        # config = plugin.update_config(config)
-        plugin.update_config(config)
-
-    # This is set up before globals are initialized
-    site_id = os.environ.get('CKAN_SITE_ID')
-    if site_id:
-        config['ckan.site_id'] = site_id
-
-    site_url = config.get('ckan.site_url', '')
-    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
-    if config.get('ckan.site_id') is None:
-        if ':' in ckan_host:
-            ckan_host, port = ckan_host.split(':')
-        assert ckan_host, 'You need to configure ckan.site_url or ' \
-                          'ckan.site_id for SOLR search-index rebuild to work.'
-        config['ckan.site_id'] = ckan_host
-
-    # ensure that a favicon has been set
-    favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
-    config['ckan.favicon'] = favicon
-
-    # Init SOLR settings and check if the schema is compatible
-    #from ckan.lib.search import SolrSettings, check_solr_schema_version
-
-    # lib.search is imported here as we need the config enabled and parsed
-    import ckan.lib.search as search
-    search.SolrSettings.init(config.get('solr_url'),
-                             config.get('solr_user'),
-                             config.get('solr_password'))
-    search.check_solr_schema_version()
-
-    config['routes.map'] = routing.make_map()
-    config['routes.named_routes'] = routing.named_routes
-    config['pylons.app_globals'] = app_globals.app_globals
-    # initialise the globals
-    config['pylons.app_globals']._init()
-
-    # add helper functions
-    helpers = _Helpers(h)
-    config['pylons.h'] = helpers
-
-    ## redo template setup to use genshi.search_path
-    ## (so remove std template setup)
-    legacy_templates_path = os.path.join(root, 'templates_legacy')
-    jinja2_templates_path = os.path.join(root, 'templates')
-    if asbool(config.get('ckan.legacy_templates', 'no')):
-        # We want the new template path for extra snippets like the
-        # dataviewer and also for some testing stuff
-        template_paths = [legacy_templates_path, jinja2_templates_path]
-    else:
-        template_paths = [jinja2_templates_path, legacy_templates_path]
-
-    extra_template_paths = config.get('extra_template_paths', '')
-    if extra_template_paths:
-        # must be first for them to override defaults
-        template_paths = extra_template_paths.split(',') + template_paths
-    config['pylons.app_globals'].template_paths = template_paths
-
-    # Translator (i18n)
-    translator = Translator(pylons.translator)
-
-    def template_loaded(template):
-        translator.setup(template)
-
-    # Markdown ignores the logger config, so to get rid of excessive
-    # markdown debug messages in the log, set it to the level of the
-    # root logger.
-    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
-
-    # Create the Genshi TemplateLoader
-    config['pylons.app_globals'].genshi_loader = TemplateLoader(
-        template_paths, auto_reload=True, callback=template_loaded)
 
     #################################################################
     #                                                               #
@@ -296,6 +219,103 @@ def genshi_lookup_attr(cls, obj, key):
     #                                                               #
     #################################################################
 
+    # Setup the SQLAlchemy database engine
+    # Suppress a couple of sqlalchemy warnings
+    msgs = ['^Unicode type received non-unicode bind param value',
+            "^Did not recognize type 'BIGINT' of column 'size'",
+            "^Did not recognize type 'tsvector' of column 'search_vector'"
+            ]
+    for msg in msgs:
+        warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
+
+    # load all CKAN plugins
+    p.load_all(config)
+    update_config()
+
+
+def update_config():
+    ''' This code needs to be run when the config is changed to take those
+    changes into account. '''
+
+    for plugin in p.PluginImplementations(p.IConfigurer):
+        # must do update in place as this does not work:
+        # config = plugin.update_config(config)
+        plugin.update_config(config)
+
+    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    # This is set up before globals are initialized
+    site_id = os.environ.get('CKAN_SITE_ID')
+    if site_id:
+        config['ckan.site_id'] = site_id
+
+    site_url = config.get('ckan.site_url', '')
+    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
+    if config.get('ckan.site_id') is None:
+        if ':' in ckan_host:
+            ckan_host, port = ckan_host.split(':')
+        assert ckan_host, 'You need to configure ckan.site_url or ' \
+                          'ckan.site_id for SOLR search-index rebuild to work.'
+        config['ckan.site_id'] = ckan_host
+
+    # ensure that a favicon has been set
+    favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
+    config['ckan.favicon'] = favicon
+
+    # Init SOLR settings and check if the schema is compatible
+    #from ckan.lib.search import SolrSettings, check_solr_schema_version
+
+    # lib.search is imported here as we need the config enabled and parsed
+    search.SolrSettings.init(config.get('solr_url'),
+                             config.get('solr_user'),
+                             config.get('solr_password'))
+    search.check_solr_schema_version()
+
+    config['routes.map'] = routing.make_map()
+    config['routes.named_routes'] = routing.named_routes
+    config['pylons.app_globals'] = app_globals.app_globals
+    # initialise the globals
+    config['pylons.app_globals']._init()
+
+    # add helper functions
+    helpers = _Helpers(h)
+    config['pylons.h'] = helpers
+
+    helpers = config['pylons.h']
+    if helpers:
+        helpers._setup()
+
+    ## redo template setup to use genshi.search_path
+    ## (so remove std template setup)
+    legacy_templates_path = os.path.join(root, 'templates_legacy')
+    jinja2_templates_path = os.path.join(root, 'templates')
+    if asbool(config.get('ckan.legacy_templates', 'no')):
+        # We want the new template path for extra snippets like the
+        # dataviewer and also for some testing stuff
+        template_paths = [legacy_templates_path, jinja2_templates_path]
+    else:
+        template_paths = [jinja2_templates_path, legacy_templates_path]
+
+    extra_template_paths = config.get('extra_template_paths', '')
+    if extra_template_paths:
+        # must be first for them to override defaults
+        template_paths = extra_template_paths.split(',') + template_paths
+    config['pylons.app_globals'].template_paths = template_paths
+
+    # Translator (i18n)
+    translator = Translator(pylons.translator)
+
+    def template_loaded(template):
+        translator.setup(template)
+
+    # Markdown ignores the logger config, so to get rid of excessive
+    # markdown debug messages in the log, set it to the level of the
+    # root logger.
+    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
+
+    # Create the Genshi TemplateLoader
+    config['pylons.app_globals'].genshi_loader = TemplateLoader(
+        template_paths, auto_reload=True, callback=template_loaded)
+
 
     # Create Jinja2 environment
     env = lib.jinja_extensions.Environment(
@@ -319,17 +339,7 @@ def genshi_lookup_attr(cls, obj, key):
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)
 
-    # Setup the SQLAlchemy database engine
-    # Suppress a couple of sqlalchemy warnings
-    msgs = ['^Unicode type received non-unicode bind param value',
-            "^Did not recognize type 'BIGINT' of column 'size'",
-            "^Did not recognize type 'tsvector' of column 'search_vector'"
-            ]
-    for msg in msgs:
-        warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
-
     ckan_db = os.environ.get('CKAN_DB')
-
     if ckan_db:
         config['sqlalchemy.url'] = ckan_db
 
@@ -345,10 +355,13 @@ def genshi_lookup_attr(cls, obj, key):
     if not model.meta.engine:
         model.init_model(engine)
 
-
     for plugin in p.PluginImplementations(p.IConfigurable):
         plugin.configure(config)
 
     # reset the template cache - we do this here so that when we load the
     # environment it is clean
     render.reset_template_info_cache()
+
+    # clear other caches
+    logic.clear_actions_cache()
+    new_authz.clear_auth_functions_cache()

ckan/plugins/core.py

@@ -5,13 +5,15 @@
 import logging
 from inspect import isclass
 from itertools import chain
+
 from pkg_resources import iter_entry_points
 from pyutilib.component.core import PluginGlobals, implements
 from pyutilib.component.core import ExtensionPoint as PluginImplementations
 from pyutilib.component.core import SingletonPlugin as _pca_SingletonPlugin
 from pyutilib.component.core import Plugin as _pca_Plugin
+from paste.deploy.converters import asbool
 
-from ckan.plugins.interfaces import IPluginObserver, IGenshiStreamFilter
+import interfaces
 
 __all__ = [
     'PluginImplementations', 'implements',
@@ -98,59 +100,75 @@ def load_all(config):
     # PCA default behaviour is to activate SingletonPlugins at import time. We
     # only want to activate those listed in the config, so clear
     # everything then activate only those we want.
-    unload_all()
+    unload_all(update=False)
 
     for plugin in plugins:
-        load(plugin)
+        load(plugin, update=False)
+
+    # Load the synchronous search plugin, unless already loaded or
+    # explicitly disabled
+    if not 'synchronous_search' in config.get('ckan.plugins',[]) and \
+            asbool(config.get('ckan.search.automatic_indexing', True)):
+        log.debug('Loading the synchronous search plugin')
+        load('synchronous_search', update=False)
+
+    plugins_update()
 
 
 def reset():
     """
     Clear and reload all configured plugins
     """
+    # FIXME This looks like it should be removed
     from pylons import config
     load_all(config)
 
-def _clear_logic_and_auth_caches():
-    import ckan.logic
-    import ckan.new_authz
-    ckan.logic.clear_actions_cache()
-    ckan.new_authz.clear_auth_functions_cache()
 
-def load(plugin):
+def plugins_update():
+    ''' This is run when plugins have been loaded or unloaded and allows us
+    to run any specific code to ensure that the new plugin setting are
+    correctly setup '''
+    import ckan.config.environment as environment
+    environment.update_config()
+
+
+def load(plugin, update=True):
     """
     Load a single plugin, given a plugin name, class or instance
     """
-    _clear_logic_and_auth_caches()
-    observers = PluginImplementations(IPluginObserver)
+    observers = PluginImplementations(interfaces.IPluginObserver)
     for observer_plugin in observers:
         observer_plugin.before_load(plugin)
     service = _get_service(plugin)
     service.activate()
     for observer_plugin in observers:
         observer_plugin.after_load(service)
 
-    if IGenshiStreamFilter in service.__interfaces__:
+    if interfaces.IGenshiStreamFilter in service.__interfaces__:
        log.warn("Plugin '%s' is using deprecated interface IGenshiStreamFilter" % plugin)
 
+    if update:
+        plugins_update()
+
     return service
 
 
-def unload_all():
+def unload_all(update=True):
     """
     Unload (deactivate) all loaded plugins
     """
     for env in PluginGlobals.env_registry.values():
         for service in env.services.copy():
-            unload(service)
+            unload(service, update=False)
+    if update:
+        plugins_update()
 
 
-def unload(plugin):
+def unload(plugin, update=True):
     """
     Unload a single plugin, given a plugin name, class or instance
     """
-    _clear_logic_and_auth_caches()
-    observers = PluginImplementations(IPluginObserver)
+    observers = PluginImplementations(interfaces.IPluginObserver)
     service = _get_service(plugin)
     for observer_plugin in observers:
         observer_plugin.before_unload(service)
@@ -160,6 +178,9 @@ def unload(plugin):
     for observer_plugin in observers:
         observer_plugin.after_unload(service)
 
+    if update:
+        plugins_update()
+
     return service