From 331087d35b2db7290f1fb83286d9435e08d88ea6 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 5 Mar 2013 10:06:25 +0000 Subject: [PATCH] [#547] Rework the plugins and config stuff --- ckan/config/environment.py | 207 ++++++++++++++++++++----------------- ckan/plugins/core.py | 55 +++++++--- 2 files changed, 148 insertions(+), 114 deletions(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 6f56230f119..0318abdcf32 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -19,6 +19,9 @@ import ckan.lib.helpers as h import ckan.lib.app_globals as app_globals import ckan.lib.render as render +import ckan.lib.search as search +import ckan.logic as logic +import ckan.new_authz as new_authz log = logging.getLogger(__name__) @@ -34,6 +37,11 @@ class _Helpers(object): templates have helper functions provided by extensions that have not been enabled. ''' def __init__(self, helpers): + self.helpers = helpers + self._setup() + + def _setup(self): + helpers = self.helpers functions = {} allowed = helpers.__allowed_functions__[:] # list of functions due to be deprecated @@ -92,7 +100,7 @@ def __getattr__(self, name): def load_environment(global_conf, app_conf): """Configure the Pylons environment via the ``pylons.config`` - object + object. This code should only need to be run once. """ ###### Pylons monkey-patch @@ -127,93 +135,8 @@ def find_controller(self, controller): templates=[]) # Initialize config with the basic options - config.init_app(global_conf, app_conf, package='ckan', paths=paths) - # load all CKAN plugins - p.load_all(config) - - # Load the synchronous search plugin, unless already loaded or - # explicitly disabled - if not 'synchronous_search' in config.get('ckan.plugins',[]) and \ - asbool(config.get('ckan.search.automatic_indexing', True)): - log.debug('Loading the synchronous search plugin') - p.load('synchronous_search') - - for plugin in p.PluginImplementations(p.IConfigurer): - # must do update in place as this does not work: - # config = plugin.update_config(config) - plugin.update_config(config) - - # This is set up before globals are initialized - site_id = os.environ.get('CKAN_SITE_ID') - if site_id: - config['ckan.site_id'] = site_id - - site_url = config.get('ckan.site_url', '') - ckan_host = config['ckan.host'] = urlparse(site_url).netloc - if config.get('ckan.site_id') is None: - if ':' in ckan_host: - ckan_host, port = ckan_host.split(':') - assert ckan_host, 'You need to configure ckan.site_url or ' \ - 'ckan.site_id for SOLR search-index rebuild to work.' - config['ckan.site_id'] = ckan_host - - # ensure that a favicon has been set - favicon = config.get('ckan.favicon', '/images/icons/ckan.ico') - config['ckan.favicon'] = favicon - - # Init SOLR settings and check if the schema is compatible - #from ckan.lib.search import SolrSettings, check_solr_schema_version - - # lib.search is imported here as we need the config enabled and parsed - import ckan.lib.search as search - search.SolrSettings.init(config.get('solr_url'), - config.get('solr_user'), - config.get('solr_password')) - search.check_solr_schema_version() - - config['routes.map'] = routing.make_map() - config['routes.named_routes'] = routing.named_routes - config['pylons.app_globals'] = app_globals.app_globals - # initialise the globals - config['pylons.app_globals']._init() - - # add helper functions - helpers = _Helpers(h) - config['pylons.h'] = helpers - - ## redo template setup to use genshi.search_path - ## (so remove std template setup) - legacy_templates_path = os.path.join(root, 'templates_legacy') - jinja2_templates_path = os.path.join(root, 'templates') - if asbool(config.get('ckan.legacy_templates', 'no')): - # We want the new template path for extra snippets like the - # dataviewer and also for some testing stuff - template_paths = [legacy_templates_path, jinja2_templates_path] - else: - template_paths = [jinja2_templates_path, legacy_templates_path] - - extra_template_paths = config.get('extra_template_paths', '') - if extra_template_paths: - # must be first for them to override defaults - template_paths = extra_template_paths.split(',') + template_paths - config['pylons.app_globals'].template_paths = template_paths - - # Translator (i18n) - translator = Translator(pylons.translator) - - def template_loaded(template): - translator.setup(template) - - # Markdown ignores the logger config, so to get rid of excessive - # markdown debug messages in the log, set it to the level of the - # root logger. - logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) - - # Create the Genshi TemplateLoader - config['pylons.app_globals'].genshi_loader = TemplateLoader( - template_paths, auto_reload=True, callback=template_loaded) ################################################################# # # @@ -296,6 +219,103 @@ def genshi_lookup_attr(cls, obj, key): # # ################################################################# + # Setup the SQLAlchemy database engine + # Suppress a couple of sqlalchemy warnings + msgs = ['^Unicode type received non-unicode bind param value', + "^Did not recognize type 'BIGINT' of column 'size'", + "^Did not recognize type 'tsvector' of column 'search_vector'" + ] + for msg in msgs: + warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) + + # load all CKAN plugins + p.load_all(config) + update_config() + + +def update_config(): + ''' This code needs to be run when the config is changed to take those + changes into account. ''' + + for plugin in p.PluginImplementations(p.IConfigurer): + # must do update in place as this does not work: + # config = plugin.update_config(config) + plugin.update_config(config) + + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + # This is set up before globals are initialized + site_id = os.environ.get('CKAN_SITE_ID') + if site_id: + config['ckan.site_id'] = site_id + + site_url = config.get('ckan.site_url', '') + ckan_host = config['ckan.host'] = urlparse(site_url).netloc + if config.get('ckan.site_id') is None: + if ':' in ckan_host: + ckan_host, port = ckan_host.split(':') + assert ckan_host, 'You need to configure ckan.site_url or ' \ + 'ckan.site_id for SOLR search-index rebuild to work.' + config['ckan.site_id'] = ckan_host + + # ensure that a favicon has been set + favicon = config.get('ckan.favicon', '/images/icons/ckan.ico') + config['ckan.favicon'] = favicon + + # Init SOLR settings and check if the schema is compatible + #from ckan.lib.search import SolrSettings, check_solr_schema_version + + # lib.search is imported here as we need the config enabled and parsed + search.SolrSettings.init(config.get('solr_url'), + config.get('solr_user'), + config.get('solr_password')) + search.check_solr_schema_version() + + config['routes.map'] = routing.make_map() + config['routes.named_routes'] = routing.named_routes + config['pylons.app_globals'] = app_globals.app_globals + # initialise the globals + config['pylons.app_globals']._init() + + # add helper functions + helpers = _Helpers(h) + config['pylons.h'] = helpers + + helpers = config['pylons.h'] + if helpers: + helpers._setup() + + ## redo template setup to use genshi.search_path + ## (so remove std template setup) + legacy_templates_path = os.path.join(root, 'templates_legacy') + jinja2_templates_path = os.path.join(root, 'templates') + if asbool(config.get('ckan.legacy_templates', 'no')): + # We want the new template path for extra snippets like the + # dataviewer and also for some testing stuff + template_paths = [legacy_templates_path, jinja2_templates_path] + else: + template_paths = [jinja2_templates_path, legacy_templates_path] + + extra_template_paths = config.get('extra_template_paths', '') + if extra_template_paths: + # must be first for them to override defaults + template_paths = extra_template_paths.split(',') + template_paths + config['pylons.app_globals'].template_paths = template_paths + + # Translator (i18n) + translator = Translator(pylons.translator) + + def template_loaded(template): + translator.setup(template) + + # Markdown ignores the logger config, so to get rid of excessive + # markdown debug messages in the log, set it to the level of the + # root logger. + logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) + + # Create the Genshi TemplateLoader + config['pylons.app_globals'].genshi_loader = TemplateLoader( + template_paths, auto_reload=True, callback=template_loaded) + # Create Jinja2 environment env = lib.jinja_extensions.Environment( @@ -319,17 +339,7 @@ def genshi_lookup_attr(cls, obj, key): # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) - # Setup the SQLAlchemy database engine - # Suppress a couple of sqlalchemy warnings - msgs = ['^Unicode type received non-unicode bind param value', - "^Did not recognize type 'BIGINT' of column 'size'", - "^Did not recognize type 'tsvector' of column 'search_vector'" - ] - for msg in msgs: - warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) - ckan_db = os.environ.get('CKAN_DB') - if ckan_db: config['sqlalchemy.url'] = ckan_db @@ -345,10 +355,13 @@ def genshi_lookup_attr(cls, obj, key): if not model.meta.engine: model.init_model(engine) - for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # reset the template cache - we do this here so that when we load the # environment it is clean render.reset_template_info_cache() + + # clear other caches + logic.clear_actions_cache() + new_authz.clear_auth_functions_cache() diff --git a/ckan/plugins/core.py b/ckan/plugins/core.py index d6826aa292b..4ab459a5b92 100644 --- a/ckan/plugins/core.py +++ b/ckan/plugins/core.py @@ -5,13 +5,15 @@ import logging from inspect import isclass from itertools import chain + from pkg_resources import iter_entry_points from pyutilib.component.core import PluginGlobals, implements from pyutilib.component.core import ExtensionPoint as PluginImplementations from pyutilib.component.core import SingletonPlugin as _pca_SingletonPlugin from pyutilib.component.core import Plugin as _pca_Plugin +from paste.deploy.converters import asbool -from ckan.plugins.interfaces import IPluginObserver, IGenshiStreamFilter +import interfaces __all__ = [ 'PluginImplementations', 'implements', @@ -98,31 +100,43 @@ def load_all(config): # PCA default behaviour is to activate SingletonPlugins at import time. We # only want to activate those listed in the config, so clear # everything then activate only those we want. - unload_all() + unload_all(update=False) for plugin in plugins: - load(plugin) + load(plugin, update=False) + + # Load the synchronous search plugin, unless already loaded or + # explicitly disabled + if not 'synchronous_search' in config.get('ckan.plugins',[]) and \ + asbool(config.get('ckan.search.automatic_indexing', True)): + log.debug('Loading the synchronous search plugin') + load('synchronous_search', update=False) + + plugins_update() def reset(): """ Clear and reload all configured plugins """ + # FIXME This looks like it should be removed from pylons import config load_all(config) -def _clear_logic_and_auth_caches(): - import ckan.logic - import ckan.new_authz - ckan.logic.clear_actions_cache() - ckan.new_authz.clear_auth_functions_cache() -def load(plugin): +def plugins_update(): + ''' This is run when plugins have been loaded or unloaded and allows us + to run any specific code to ensure that the new plugin setting are + correctly setup ''' + import ckan.config.environment as environment + environment.update_config() + + +def load(plugin, update=True): """ Load a single plugin, given a plugin name, class or instance """ - _clear_logic_and_auth_caches() - observers = PluginImplementations(IPluginObserver) + observers = PluginImplementations(interfaces.IPluginObserver) for observer_plugin in observers: observer_plugin.before_load(plugin) service = _get_service(plugin) @@ -130,27 +144,31 @@ def load(plugin): for observer_plugin in observers: observer_plugin.after_load(service) - if IGenshiStreamFilter in service.__interfaces__: + if interfaces.IGenshiStreamFilter in service.__interfaces__: log.warn("Plugin '%s' is using deprecated interface IGenshiStreamFilter" % plugin) + if update: + plugins_update() + return service -def unload_all(): +def unload_all(update=True): """ Unload (deactivate) all loaded plugins """ for env in PluginGlobals.env_registry.values(): for service in env.services.copy(): - unload(service) + unload(service, update=False) + if update: + plugins_update() -def unload(plugin): +def unload(plugin, update=True): """ Unload a single plugin, given a plugin name, class or instance """ - _clear_logic_and_auth_caches() - observers = PluginImplementations(IPluginObserver) + observers = PluginImplementations(interfaces.IPluginObserver) service = _get_service(plugin) for observer_plugin in observers: observer_plugin.before_unload(service) @@ -160,6 +178,9 @@ def unload(plugin): for observer_plugin in observers: observer_plugin.after_unload(service) + if update: + plugins_update() + return service