diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 834b3ba38b0..872c6dd75db 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -264,8 +264,14 @@ def pager_url(q=None, page=None): c.page = h.Page(collection=[]) c.search_facets_limits = {} for facet in c.search_facets.keys(): - limit = int(request.params.get('_%s_limit' % facet, - g.facets_default_number)) + try: + limit = int(request.params.get('_%s_limit' % facet, + g.facets_default_number)) + except ValueError: + abort(400, _('Parameter "{parameter_name}" is not ' + 'an integer').format( + parameter_name='_%s_limit' % facet + )) c.search_facets_limits[facet] = limit maintain.deprecate_context_item( diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index 3f99118649e..ac12c869213 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -1,7 +1,7 @@ import functools import logging -import types import re +import sys import formencode.validators @@ -194,8 +194,18 @@ def flatten_to_string_key(dict): def check_access(action, context, data_dict=None): - user = context.get('user') + action = new_authz.clean_action_name(action) + # Auth Auditing. We remove this call from the __auth_audit stack to show + # we have called the auth function + try: + audit = context.get('__auth_audit', [])[-1] + except IndexError: + audit = '' + if audit and audit[0] == action: + context['__auth_audit'].pop() + + user = context.get('user') log.debug('check access - user %r, action %s' % (user, action)) if action: @@ -281,13 +291,17 @@ def get_action(action): module = getattr(module, part) for k, v in module.__dict__.items(): if not k.startswith('_'): - # Only load functions from the action module. - if isinstance(v, types.FunctionType): + # Only load functions from the action module or already + # replaced functions. + if (hasattr(v, '__call__') + and (v.__module__ == module_path + or hasattr(v, '__replaced'))): k = new_authz.clean_action_name(k) _actions[k] = v # Whitelist all actions defined in logic/action/get.py as # being side-effect free. + # FIXME This looks wrong should it be an 'or' not 'and' v.side_effect_free = getattr(v, 'side_effect_free', True)\ and action_module_name == 'get' @@ -306,6 +320,9 @@ def get_action(action): ) log.debug('Auth function %r was inserted', plugin.name) resolved_action_plugins[name] = plugin.name + # Extensions are exempted from the auth audit for now + # This needs to be resolved later + auth_function.auth_audit_exempt = True fetched_actions[name] = auth_function # Use the updated ones in preference to the originals. _actions.update(fetched_actions) @@ -326,9 +343,37 @@ def wrapped(context=None, data_dict=None, **kw): except TypeError: # c not registered pass - return _action(context, data_dict, **kw) + + # Auth Auditing + # store this action name in the auth audit so we can see if + # check access was called on the function we store the id of + # the action incase the action is wrapped inside an action + # of the same name. this happens in the datastore + context.setdefault('__auth_audit', []) + context['__auth_audit'].append((action_name, id(_action))) + + # check_access(action_name, context, data_dict=None) + result = _action(context, data_dict, **kw) + try: + audit = context['__auth_audit'][-1] + if audit[0] == action_name and audit[1] == id(_action): + if action_name not in new_authz.auth_functions_list(): + log.debug('No auth function for %s' % action_name) + elif not getattr(_action, 'auth_audit_exempt', False): + raise Exception('Action Auth Audit: %s' % action_name) + # remove from audit stack + context['__auth_audit'].pop() + except IndexError: + pass + return result return wrapped + # If we have been called multiple times for example during tests then + # we need to make sure that we do not rewrap the actions. + if hasattr(_action, '__replaced'): + _actions[action_name] = _action.__replaced + continue + fn = make_wrapped(_action, action_name) # we need to mirror the docstring fn.__doc__ = _action.__doc__ @@ -337,6 +382,22 @@ def wrapped(context=None, data_dict=None, **kw): fn.side_effect_free = True _actions[action_name] = fn + + def replaced_action(action_name): + def warn(context, data_dict): + log.critical('Action `%s` is being called directly ' + 'all action calls should be accessed via ' + 'logic.get_action' % action_name) + return get_action(action_name)(context, data_dict) + return warn + + # Store our wrapped function so it is available. This is to prevent + # rewrapping of actions + module = sys.modules[_action.__module__] + r = replaced_action(action_name) + r.__replaced = fn + module.__dict__[action_name] = r + return _actions.get(action) @@ -400,6 +461,13 @@ def wrapper(context, data_dict): return wrapper +def auth_audit_exempt(action): + ''' Dirty hack to stop auth audit being done ''' + @functools.wraps(action) + def wrapper(context, data_dict): + return action(context, data_dict) + wrapper.auth_audit_exempt = True + return wrapper class UnknownValidator(Exception): pass diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 73bdaedb8cd..48d3e7249f5 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -703,6 +703,7 @@ def organization_create(context, data_dict): return _group_or_org_create(context, data_dict, is_org=True) +@logic.auth_audit_exempt def rating_create(context, data_dict): '''Rate a dataset (package). diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index eab8b255faf..05350a0b061 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -826,6 +826,8 @@ def resource_status_show(context, data_dict): return result_list + +@logic.auth_audit_exempt def revision_show(context, data_dict): '''Return the details of a revision. @@ -1008,7 +1010,7 @@ def user_show(context, data_dict): revisions_list = [] for revision in revisions_q.limit(20).all(): - revision_dict = revision_show(context,{'id':revision.id}) + revision_dict = logic.get_action('revision_show')(context,{'id':revision.id}) revision_dict['state'] = revision.state revisions_list.append(revision_dict) user_dict['activity'] = revisions_list @@ -1020,7 +1022,7 @@ def user_show(context, data_dict): for dataset in dataset_q: try: - dataset_dict = package_show(context, {'id': dataset.id}) + dataset_dict = logic.get_action('package_show')(context, {'id': dataset.id}) except logic.NotAuthorized: continue user_dict['datasets'].append(dataset_dict) @@ -2554,9 +2556,10 @@ def display_name(followee): # Get the followed objects. # TODO: Catch exceptions raised by these *_followee_list() functions? + # FIXME should we be changing the context like this it seems dangerous followee_dicts = [] context['skip_validation'] = True - context['skip_authorization'] = True + context['ignore_auth'] = True for followee_list_function, followee_type in ( (user_followee_list, 'user'), (dataset_followee_list, 'dataset'), @@ -2591,8 +2594,7 @@ def user_followee_list(context, data_dict): :rtype: list of dictionaries ''' - if not context.get('skip_authorization'): - _check_access('user_followee_list', context, data_dict) + _check_access('user_followee_list', context, data_dict) if not context.get('skip_validation'): schema = context.get('schema') or ( @@ -2622,8 +2624,7 @@ def dataset_followee_list(context, data_dict): :rtype: list of dictionaries ''' - if not context.get('skip_authorization'): - _check_access('dataset_followee_list', context, data_dict) + _check_access('dataset_followee_list', context, data_dict) if not context.get('skip_validation'): schema = context.get('schema') or ( @@ -2654,8 +2655,7 @@ def group_followee_list(context, data_dict): :rtype: list of dictionaries ''' - if not context.get('skip_authorization'): - _check_access('group_followee_list', context, data_dict) + _check_access('group_followee_list', context, data_dict) if not context.get('skip_validation'): schema = context.get('schema', diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index be9515265f9..dc3bba6e839 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -768,8 +768,9 @@ def term_translation_update_many(context, data_dict): context['defer_commit'] = True + action = _get_action('term_translation_update') for num, row in enumerate(data_dict['data']): - term_translation_update(context, row) + action(context, row) model.Session.commit() diff --git a/ckan/logic/auth/get.py b/ckan/logic/auth/get.py index f0f27d74764..c00624bb5d6 100644 --- a/ckan/logic/auth/get.py +++ b/ckan/logic/auth/get.py @@ -254,14 +254,17 @@ def followee_list(context, data_dict): return _followee_list(context, data_dict) +@logic.auth_audit_exempt def user_followee_list(context, data_dict): return _followee_list(context, data_dict) +@logic.auth_audit_exempt def dataset_followee_list(context, data_dict): return _followee_list(context, data_dict) +@logic.auth_audit_exempt def group_followee_list(context, data_dict): return _followee_list(context, data_dict) diff --git a/ckan/new_authz.py b/ckan/new_authz.py index 1d391c50900..01fb07927ac 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -11,13 +11,86 @@ log = getLogger(__name__) -# This is a private cache used by get_auth_function() and should never -# be accessed directly + class AuthFunctions: + ''' This is a private cache used by get_auth_function() and should never be + accessed directly we will create an instance of it and then remove it.''' _functions = {} + def clear(self): + ''' clear any stored auth functions. ''' + self._functions.clear() + + def keys(self): + ''' Return a list of known auth functions.''' + if not self._functions: + self._build() + return self._functions.keys() + + def get(self, function): + ''' Return the requested auth function. ''' + if not self._functions: + self._build() + return self._functions.get(function) + + def _build(self): + ''' Gather the auth functions. + + First get the default ones in the ckan/logic/auth directory Rather than + writing them out in full will use __import__ to load anything from + ckan.auth that looks like it might be an authorisation function''' + + module_root = 'ckan.logic.auth' + + for auth_module_name in ['get', 'create', 'update', 'delete']: + module_path = '%s.%s' % (module_root, auth_module_name,) + try: + module = __import__(module_path) + except ImportError: + log.debug('No auth module for action "%s"' % auth_module_name) + continue + + for part in module_path.split('.')[1:]: + module = getattr(module, part) + + for key, v in module.__dict__.items(): + if not key.startswith('_'): + key = clean_action_name(key) + self._functions[key] = v + + # Then overwrite them with any specific ones in the plugins: + resolved_auth_function_plugins = {} + fetched_auth_functions = {} + for plugin in p.PluginImplementations(p.IAuthFunctions): + for name, auth_function in plugin.get_auth_functions().items(): + name = clean_action_name(name) + if name in resolved_auth_function_plugins: + raise Exception( + 'The auth function %r is already implemented in %r' % ( + name, + resolved_auth_function_plugins[name] + ) + ) + log.debug('Auth function %r was inserted', plugin.name) + resolved_auth_function_plugins[name] = plugin.name + fetched_auth_functions[name] = auth_function + # Use the updated ones in preference to the originals. + self._functions.update(fetched_auth_functions) + +_AuthFunctions = AuthFunctions() +#remove the class +del AuthFunctions + + def clear_auth_functions_cache(): - AuthFunctions._functions.clear() + _AuthFunctions.clear() + + +def auth_functions_list(): + '''Returns a list of the names of the auth functions available. Currently + this is to allow the Auth Audit to know if an auth function is available + for a given action.''' + return _AuthFunctions.keys() def clean_action_name(action_name): @@ -46,6 +119,7 @@ def is_sysadmin(username): return True return False + def get_group_or_org_admin_ids(group_id): if not group_id: return [] @@ -57,18 +131,20 @@ def get_group_or_org_admin_ids(group_id): .filter(model.Member.capacity == 'admin') return [a.table_id for a in q.all()] + def is_authorized_boolean(action, context, data_dict=None): ''' runs the auth function but just returns True if allowed else False ''' outcome = is_authorized(action, context, data_dict=data_dict) return outcome.get('success', False) + def is_authorized(action, context, data_dict=None): if context.get('ignore_auth'): return {'success': True} action = clean_action_name(action) - auth_function = _get_auth_function(action) + auth_function = _AuthFunctions.get(action) if auth_function: # sysadmins can do anything unless the auth_sysadmins_check # decorator was used in which case they are treated like all other @@ -81,6 +157,7 @@ def is_authorized(action, context, data_dict=None): else: raise ValueError(_('Authorization function not found: %s' % action)) + # these are the permissions that roles have ROLE_PERMISSIONS = OrderedDict([ ('admin', ['admin']), @@ -88,15 +165,19 @@ def is_authorized(action, context, data_dict=None): ('member', ['read']), ]) + def _trans_role_admin(): return _('Admin') + def _trans_role_editor(): return _('Editor') + def _trans_role_member(): return _('Member') + def trans_role(role): module = sys.modules[__name__] return getattr(module, '_trans_role_%s' % role)() @@ -109,6 +190,7 @@ def roles_list(): roles.append(dict(text=trans_role(role), value=role)) return roles + def roles_trans(): ''' return dict of roles with translation ''' roles = {} @@ -175,6 +257,7 @@ def users_role_for_group_or_org(group_id, user_name): return row.capacity return None + def has_user_permission_for_some_org(user_name, permission): ''' Check if the user has the given permission for the group ''' user_id = get_user_id_for_username(user_name, allow_none=True) @@ -205,6 +288,7 @@ def has_user_permission_for_some_org(user_name, permission): return bool(q.count()) + def get_user_id_for_username(user_name, allow_none=False): ''' Helper function to get user id ''' # first check if we have the user object already and get from there @@ -221,54 +305,6 @@ def get_user_id_for_username(user_name, allow_none=False): return None raise Exception('Not logged in user') -def _get_auth_function(action): - - if action in AuthFunctions._functions: - return AuthFunctions._functions.get(action) - - # Otherwise look in all the plugins to resolve all possible - # First get the default ones in the ckan/logic/auth directory - # Rather than writing them out in full will use __import__ - # to load anything from ckan.auth that looks like it might - # be an authorisation function - - module_root = 'ckan.logic.auth' - - for auth_module_name in ['get', 'create', 'update','delete']: - module_path = '%s.%s' % (module_root, auth_module_name,) - try: - module = __import__(module_path) - except ImportError,e: - log.debug('No auth module for action "%s"' % auth_module_name) - continue - - for part in module_path.split('.')[1:]: - module = getattr(module, part) - - for key, v in module.__dict__.items(): - if not key.startswith('_'): - key = clean_action_name(key) - AuthFunctions._functions[key] = v - - # Then overwrite them with any specific ones in the plugins: - resolved_auth_function_plugins = {} - fetched_auth_functions = {} - for plugin in p.PluginImplementations(p.IAuthFunctions): - for name, auth_function in plugin.get_auth_functions().items(): - name = clean_action_name(name) - if name in resolved_auth_function_plugins: - raise Exception( - 'The auth function %r is already implemented in %r' % ( - name, - resolved_auth_function_plugins[name] - ) - ) - log.debug('Auth function %r was inserted', plugin.name) - resolved_auth_function_plugins[name] = plugin.name - fetched_auth_functions[name] = auth_function - # Use the updated ones in preference to the originals. - AuthFunctions._functions.update(fetched_auth_functions) - return AuthFunctions._functions.get(action) CONFIG_PERMISSIONS_DEFAULTS = { # permission and default @@ -285,6 +321,7 @@ def _get_auth_function(action): CONFIG_PERMISSIONS = {} + def check_config_permission(permission): ''' Returns the permission True/False based on config ''' # set up perms if not already done @@ -298,7 +335,6 @@ def check_config_permission(permission): return False - def auth_is_registered_user(): ''' Do we have a logged in user ''' try: diff --git a/doc/conf.py b/doc/conf.py index 1c9ac651251..06cc6161f3b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -48,6 +48,7 @@ .. |storage_parent_dir| replace:: /var/lib/ckan .. |storage_dir| replace:: |storage_parent_dir|/default .. |reload_apache| replace:: sudo service apache2 reload +.. |restart_apache| replace:: sudo service apache2 restart .. |solr| replace:: Solr .. |restructuredtext| replace:: reStructuredText .. |nginx| replace:: Nginx diff --git a/doc/upgrade-package-to-minor-release.rst b/doc/upgrade-package-to-minor-release.rst index f3bc6bd35f9..5a334e60426 100644 --- a/doc/upgrade-package-to-minor-release.rst +++ b/doc/upgrade-package-to-minor-release.rst @@ -8,5 +8,105 @@ Upgrading a CKAN 2 package install to a new minor release themes or extensions you're using, check the changelog, and backup your database. See :ref:`upgrading`. -.. todo:: Instructions for upgrading a package install to a new minor release +Each :ref:`minor release ` is distributed in its own package, +so for example CKAN ``2.0.X`` and ``2.1.X`` will be installed using the +``python-ckan_2.0_amd64.deb`` and ``python-ckan_2.1_amd64.deb`` packages +respectively. +#. Download the CKAN package for the new minor release you want to upgrade + to (replace the version number with the relevant one):: + + wget http://packaging.ckan.org/python-ckan_2.1_amd64.deb + +#. Install the package with the following command:: + + sudo dpkg -i python-ckan_2.1_amd64.deb + + .. note:: + + If you have changed the |apache| or |nginx| configuration files, you will + get a prompt like the following, asking whether to keep your local changes + or replace the files. You generally would like to keep your local changes + (option ``N``, which is the default), but you can look at the differences + between versions by selecting option ``D``:: + + Configuration file `/etc/apache2/sites-available/ckan_default' + ==> File on system created by you or by a script. + ==> File also in package provided by package maintainer. + What would you like to do about it ? Your options are: + Y or I : install the package maintainer's version + N or O : keep your currently-installed version + D : show the differences between the versions + Z : start a shell to examine the situation + The default action is to keep your current version. + *** ckan_default (Y/I/N/O/D/Z) [default=N] ? + + Your local CKAN configuration file in |config_dir| will not be replaced. + + .. note:: + + The install process will uninstall any existing CKAN extensions or other + libraries located in the ``src`` directory of the CKAN virtualenv. To + enable them again, the installation process will iterate over all folders in + the ``src`` directory, reinstall the requirements listed in + ``pip-requirements.txt`` and ``requirements.txt`` files and run + ``python setup.py develop`` for each. If you are using a custom extension + which does not use this requirements file name or is located elsewhere, + you will need to manually reinstall it. + +#. If there have been changes in the database schema (check the + :doc:`changelog` to find out) you need to update your CKAN database's + schema using the ``db upgrade`` command. + + .. warning :: + + To avoid problems during the database upgrade, comment out any plugins + that you have enabled in your ini file. You can uncomment them again when + the upgrade finishes. + + For example: + + .. parsed-literal:: + + paster db upgrade --config=\ |development.ini| + + See :ref:`paster db` for details of the ``db upgrade`` + command. + +#. If there have been changes in the Solr schema (check the :doc:`changelog` + to find out) you need to update your Solr schema symlink. + + When :ref:`setting up solr` you created a symlink + ``/etc/solr/conf/schema.xml`` linking to a CKAN Solr schema file such as + |virtualenv|/src/ckan/ckan/config/solr/schema-2.0.xml. This symlink + should be updated to point to the latest schema file in + |virtualenv|/src/ckan/ckan/config/solr/, if it doesn't already. + + For example, to update the symlink: + + .. parsed-literal:: + + sudo rm /etc/solr/conf/schema.xml + sudo ln -s |virtualenv|/src/ckan/ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml + + You will need to restart Jetty for the changes to take effect: + + .. parsed-literal:: + + sudo service jetty restart + +#. Rebuild your search index by running the ``ckan search-index rebuild`` + command: + + .. parsed-literal:: + + sudo ckan search-index rebuild -r + + See :ref:`rebuild search index` for details of the + ``ckan search-index rebuild`` command. + +#. Finally, restart Apache: + + .. parsed-literal:: + + |restart_apache| diff --git a/doc/upgrade-source.rst b/doc/upgrade-source.rst index 9f751133597..c8f66645c92 100644 --- a/doc/upgrade-source.rst +++ b/doc/upgrade-source.rst @@ -47,6 +47,12 @@ CKAN release you're upgrading to: sudo rm /etc/solr/conf/schema.xml sudo ln -s |virtualenv|/src/ckan/ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml + You will need to restart Jetty for the changes to take effect: + + .. parsed-literal:: + + sudo service jetty restart + #. If you are upgrading to a new :ref:`major release ` update your CKAN database's schema using the ``db upgrade`` command.