diff --git a/ckan/authz.py b/ckan/authz.py index 65f0a335787..343a351adc0 100644 --- a/ckan/authz.py +++ b/ckan/authz.py @@ -47,6 +47,7 @@ def is_authorized(cls, username, action, domain_object): if isinstance(username, str): username = username.decode('utf8') assert isinstance(username, unicode), type(username) + for extension in cls.extensions: authorized = extension.is_authorized(username, action, diff --git a/ckan/ckan_nose_plugin.py b/ckan/ckan_nose_plugin.py index 785d3543def..0666b7761c3 100644 --- a/ckan/ckan_nose_plugin.py +++ b/ckan/ckan_nose_plugin.py @@ -47,10 +47,21 @@ def options(self, parser, env): '--ckan-migration', action='store_true', dest='ckan_migration', - help='set this when wanting to test migrations') + help='set this when wanting to test migrations') + parser.add_option( + '--docstrings', + action='store_true', + dest='docstrings', + help='set this to display test docstrings instead of module names') def configure(self, settings, config): CkanNose.settings = settings if settings.is_ckan: self.enabled = True self.is_first_test = True + + def describeTest(self, test): + if not CkanNose.settings.docstrings: + # display module name instead of docstring + return False + diff --git a/ckan/config/routing.py b/ckan/config/routing.py index f946f2d29b9..b9f64c08efc 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -8,6 +8,9 @@ from pylons import config from routes import Mapper from ckan.plugins import PluginImplementations, IRoutes +from ckan.controllers.package import register_pluggable_behaviour as register_package_behaviour +from ckan.controllers.group import register_pluggable_behaviour as register_group_behaviour + routing_plugins = PluginImplementations(IRoutes) @@ -23,12 +26,12 @@ def make_map(): map.connect('/error/{action}', controller='error') map.connect('/error/{action}/{id}', controller='error') + map.connect('*url', controller='home', action='cors_options', conditions=dict(method=['OPTIONS'])) + # CUSTOM ROUTES HERE for plugin in routing_plugins: map = plugin.before_map(map) - - map.connect('*url', controller='home', action='cors_options', - conditions=dict(method=['OPTIONS'])) + map.connect('home', '/', controller='home', action='index') map.connect('/locale', controller='home', action='locale') map.connect('about', '/about', controller='home', action='about') @@ -176,7 +179,7 @@ def make_map(): ########### ## /END API ########### - + map.redirect("/packages", "/dataset") map.redirect("/packages/{url:.*}", "/dataset/{url}") map.redirect("/package", "/dataset") @@ -227,17 +230,24 @@ def make_map(): ##map.connect('/group/new', controller='group_formalchemy', action='new') ##map.connect('/group/edit/{id}', controller='group_formalchemy', action='edit') - map.connect('/group', controller='group', action='index') - map.connect('/group/list', controller='group', action='list') - map.connect('/group/new', controller='group', action='new') - map.connect('/group/{action}/{id}', controller='group', + # These named routes are used for custom group forms which will use the + # names below based on the group.type (dataset_group is the default type) + map.connect('group_index', '/group', controller='group', action='index') + map.connect('group_list', '/group/list', controller='group', action='list') + map.connect('group_new', '/group/new', controller='group', action='new') + map.connect('group_action', '/group/{action}/{id}', controller='group', requirements=dict(action='|'.join([ 'edit', 'authz', 'history' ])) ) - map.connect('/group/{id}', controller='group', action='read') + map.connect('group_read', '/group/{id}', controller='group', action='read') + + + register_package_behaviour(map) + register_group_behaviour(map) + # authz group map.redirect("/authorizationgroups", "/authorizationgroup") map.redirect("/authorizationgroups/{url:.*}", "/authorizationgroup/{url}") @@ -278,9 +288,47 @@ def make_map(): map.connect('ckanadmin_index', '/ckan-admin', controller='admin', action='index') map.connect('ckanadmin', '/ckan-admin/{action}', controller='admin') + # Storage routes + map.connect('storage_api', "/api/storage", + controller='ckan.controllers.storage:StorageAPIController', + action='index') + map.connect('storage_api_set_metadata', '/api/storage/metadata/{label:.*}', + controller='ckan.controllers.storage:StorageAPIController', + action='set_metadata', + conditions={'method': ['PUT','POST']}) + map.connect('storage_api_get_metadata', '/api/storage/metadata/{label:.*}', + controller='ckan.controllers.storage:StorageAPIController', + action='get_metadata', + conditions={'method': ['GET']}) + map.connect('storage_api_auth_request', + '/api/storage/auth/request/{label:.*}', + controller='ckan.controllers.storage:StorageAPIController', + action='auth_request') + map.connect('storage_api_auth_form', + '/api/storage/auth/form/{label:.*}', + controller='ckan.controllers.storage:StorageAPIController', + action='auth_form') + map.connect('storage_upload', '/storage/upload', + controller='ckan.controllers.storage:StorageController', + action='upload') + map.connect('storage_upload_handle', '/storage/upload_handle', + controller='ckan.controllers.storage:StorageController', + action='upload_handle') + map.connect('storage_upload_success', '/storage/upload/success', + controller='ckan.controllers.storage:StorageController', + action='success') + map.connect('storage_upload_success_empty', '/storage/upload/success_empty', + controller='ckan.controllers.storage:StorageController', + action='success_empty') + map.connect('storage_file', '/storage/f/{label:.*}', + controller='ckan.controllers.storage:StorageController', + action='file') + + for plugin in routing_plugins: map = plugin.after_map(map) + map.redirect('/*(url)/', '/{url}', _redirect_code='301 Moved Permanently') map.connect('/*url', controller='template', action='view') diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index c4e7fe3f2d9..c5f734aad6a 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -8,7 +8,7 @@ import ckan.authz as authz from ckan.authz import Authorizer from ckan.lib.helpers import Page -from ckan.plugins import PluginImplementations, IGroupController +from ckan.plugins import PluginImplementations, IGroupController, IGroupForm from ckan.lib.navl.dictization_functions import DataError, unflatten, validate from ckan.logic import NotFound, NotAuthorized, ValidationError from ckan.logic import check_access, get_action @@ -17,19 +17,129 @@ from ckan.lib.dictization.model_dictize import package_dictize import ckan.forms -class GroupController(BaseController): - ## hooks for subclasses - group_form = 'group/new_group_form.html' +# Mapping from group-type strings to IDatasetForm instances +_controller_behaviour_for = dict() + +# The fallback behaviour +_default_controller_behaviour = None - def _form_to_db_schema(self): +def register_pluggable_behaviour(map): + """ + Register the various IGroupForm instances. + + This method will setup the mappings between package types and the registered + IGroupForm instances. If it's called more than once an + exception will be raised. + """ + global _default_controller_behaviour + + # Create the mappings and register the fallback behaviour if one is found. + for plugin in PluginImplementations(IGroupForm): + if plugin.is_fallback(): + if _default_controller_behaviour is not None: + raise ValueError, "More than one fallback "\ + "IGroupForm has been registered" + _default_controller_behaviour = plugin + + for group_type in plugin.group_types(): + # Create the routes based on group_type here, this will allow us to have top level + # objects that are actually Groups, but first we need to make sure we are not + # clobbering an existing domain + + # Our version of routes doesn't allow the environ to be passed into the match call + # and so we have to set it on the map instead. This looks like a threading problem + # waiting to happen but it is executed sequentially from instead the routing setup + e = map.environ + map.environ = {'REQUEST_METHOD': 'GET'} + match = map.match('/%s/new' % (group_type,)) + map.environ = e + if match: + raise Exception, "Plugin %r would overwrite existing urls" % plugin + + map.connect('%s_new' % (group_type,), + '/%s/new' % (group_type,), controller='group', action='new') + map.connect('%s_read' % (group_type,), + '/%s/{id}' % (group_type,), controller='group', action='read') + map.connect('%s_action' % (group_type,), + '/%s/{action}/{id}' % (group_type,), controller='group', + requirements=dict(action='|'.join(['edit', 'authz', 'history' ])) + ) + + if group_type in _controller_behaviour_for: + raise ValueError, "An existing IGroupForm is "\ + "already associated with the package type "\ + "'%s'" % group_type + _controller_behaviour_for[group_type] = plugin + + # Setup the fallback behaviour if one hasn't been defined. + if _default_controller_behaviour is None: + _default_controller_behaviour = DefaultGroupForm() + + +def _lookup_plugin(group_type): + """ + Returns the plugin controller associoated with the given group type. + + If the group type is None or cannot be found in the mapping, then the + fallback behaviour is used. + """ + if group_type is None: + return _default_controller_behaviour + return _controller_behaviour_for.get(group_type, + _default_controller_behaviour) + + +class DefaultGroupForm(object): + """ + Provides a default implementation of the pluggable Group controller behaviour. + + This class has 2 purposes: + + - it provides a base class for IGroupForm implementations + to use if only a subset of the method hooks need to be customised. + + - it provides the fallback behaviour if no plugin is setup to provide + the fallback behaviour. + + Note - this isn't a plugin implementation. This is deliberate, as + we don't want this being registered. + """ + + def group_form(self): + return 'group/new_group_form.html' + + def form_to_db_schema(self): return group_form_schema() - def _db_to_form_schema(self): + def db_to_form_schema(self): '''This is an interface to manipulate data from the database into a format suitable for the form (optional)''' - def _setup_template_variables(self, context): + + def check_data_dict(self, data_dict): + '''Check if the return data is correct, mostly for checking out if + spammers are submitting only part of the form + + # Resources might not exist yet (eg. Add Dataset) + surplus_keys_schema = ['__extras', '__junk', 'state', 'groups', + 'extras_validation', 'save', 'return_to', + 'resources'] + + schema_keys = package_form_schema().keys() + keys_in_schema = set(schema_keys) - set(surplus_keys_schema) + + missing_keys = keys_in_schema - set(data_dict.keys()) + + if missing_keys: + #print data_dict + #print missing_keys + log.info('incorrect form fields posted') + raise DataError(data_dict) + ''' + pass + + def setup_template_variables(self, context, data_dict): c.is_sysadmin = Authorizer().is_sysadmin(c.user) ## This is messy as auths take domain object not data_dict @@ -44,6 +154,27 @@ def _setup_template_variables(self, context): except NotAuthorized: c.auth_for_change_state = False +############## End of pluggable group behaviour ############## + + +class GroupController(BaseController): + + ## hooks for subclasses + + def _group_form(self, group_type=None): + return _lookup_plugin(group_type).group_form() + + def _form_to_db_schema(self, group_type=None): + return _lookup_plugin(group_type).form_to_db_schema() + + def _db_to_form_schema(self, group_type=None): + '''This is an interface to manipulate data from the database + into a format suitable for the form (optional)''' + return _lookup_plugin(group_type).form_to_db_schema() + + def _setup_template_variables(self, context, data_dict, group_type=None): + return _lookup_plugin(group_type).setup_template_variables(context,data_dict) + ## end hooks def index(self): @@ -70,9 +201,10 @@ def index(self): def read(self, id): + group_type = self._get_group_type(id.split('@')[0]) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, - 'schema': self._form_to_db_schema()} + 'schema': self._form_to_db_schema(group_type=type)} data_dict = {'id': id} try: c.group_dict = get_action('group_show')(context, data_dict) @@ -114,32 +246,40 @@ def read(self, id): return render('group/read.html') def new(self, data=None, errors=None, error_summary=None): + + group_type = request.path.strip('/').split('/')[0] + if group_type == 'group': + group_type = None + if data: + data['type'] = group_type + context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, 'schema': self._form_to_db_schema(), - 'save': 'save' in request.params} + 'save': 'save' in request.params } try: check_access('group_create',context) except NotAuthorized: abort(401, _('Unauthorized to create a group')) if context['save'] and not data: - return self._save_new(context) + return self._save_new(context, group_type) data = data or {} errors = errors or {} error_summary = error_summary or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary} - self._setup_template_variables(context) - c.form = render(self.group_form, extra_vars=vars) + self._setup_template_variables(context,data) + c.form = render(self._group_form(group_type=group_type), extra_vars=vars) return render('group/new.html') def edit(self, id, data=None, errors=None, error_summary=None): + group_type = self._get_group_type(id.split('@')[0]) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, 'save': 'save' in request.params, - 'schema': self._form_to_db_schema(), + 'schema': self._form_to_db_schema(group_type=group_type), } data_dict = {'id': id} @@ -172,17 +312,45 @@ def edit(self, id, data=None, errors=None, error_summary=None): errors = errors or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary} - self._setup_template_variables(context) - c.form = render(self.group_form, extra_vars=vars) + self._setup_template_variables(context, data, group_type=group_type) + c.form = render(self._group_form(group_type), extra_vars=vars) return render('group/edit.html') - def _save_new(self, context): + def _get_group_type(self, id): + """ + Given the id of a group it determines the plugin to load + based on the group's type name (type). The plugin found + will be returned, or None if there is no plugin associated with + the type. + + Uses a minimal context to do so. The main use of this method + is for figuring out which plugin to delegate to. + + aborts if an exception is raised. + """ + global _controller_behaviour_for + + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author} + try: + data = get_action('group_show')(context, {'id': id}) + except NotFound: + abort(404, _('Group not found')) + except NotAuthorized: + abort(401, _('Unauthorized to read group %s') % id) + return data['type'] + + + def _save_new(self, context, group_type=None): try: data_dict = clean_dict(unflatten( tuplize_dict(parse_params(request.params)))) + data_dict['type'] = group_type or 'group' context['message'] = data_dict.get('log_message', '') group = get_action('group_create')(context, data_dict) - h.redirect_to(controller='group', action='read', id=group['name']) + + # Redirect to the appropriate _read route for the type of group + h.redirect_to( group['type'] + '_read', id=group['name']) except NotAuthorized: abort(401, _('Unauthorized to read group %s') % '') except NotFound, e: diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 9e17c05309c..b1a7a23d6d3 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -25,6 +25,7 @@ from ckan.logic import tuplize_dict, clean_dict, parse_params, flatten_to_string_key from ckan.lib.dictization import table_dictize from ckan.lib.i18n import get_lang +from ckan.plugins import PluginImplementations, IDatasetForm import ckan.forms import ckan.authz import ckan.rating @@ -47,26 +48,107 @@ def search_url(params): ("text", "x-graphviz", ["dot"]), ] -class PackageController(BaseController): - - ## hooks for subclasses - package_form = 'package/new_package_form.html' - - def _form_to_db_schema(self): +############## Methods and variables related to the pluggable ############## +############## behaviour of the package controller ############## + +# Mapping from package-type strings to IDatasetForm instances +_controller_behaviour_for = dict() + +# The fallback behaviour +_default_controller_behaviour = None + +def register_pluggable_behaviour(map): + """ + Register the various IDatasetForm instances. + + This method will setup the mappings between package types and the registered + IDatasetForm instances. If it's called more than once an + exception will be raised. + """ + global _default_controller_behaviour + + # Check this method hasn't been invoked already. + # TODO: This method seems to be being invoked more than once during running of + # the tests. So I've disbabled this check until I figure out why. + #if _default_controller_behaviour is not None: + #raise ValueError, "Pluggable package controller behaviour is already defined "\ + #"'%s'" % _default_controller_behaviour + + # Create the mappings and register the fallback behaviour if one is found. + for plugin in PluginImplementations(IDatasetForm): + if plugin.is_fallback(): + if _default_controller_behaviour is not None: + raise ValueError, "More than one fallback "\ + "IDatasetForm has been registered" + _default_controller_behaviour = plugin + + for package_type in plugin.package_types(): + # Create a connection between the newly named type and the package controller + # but first we need to make sure we are not clobbering an existing domain + map.connect('/%s/new' % (package_type,), controller='package', action='new') + map.connect('%s_read' % (package_type,), '/%s/{id}' % (package_type,), controller='package', action='read') + map.connect('%s_action' % (package_type,), + '/%s/{action}/{id}' % (package_type,), controller='package', + requirements=dict(action='|'.join(['edit', 'authz', 'history' ])) + ) + + if package_type in _controller_behaviour_for: + raise ValueError, "An existing IDatasetForm is "\ + "already associated with the package type "\ + "'%s'" % package_type + _controller_behaviour_for[package_type] = plugin + + # Setup the fallback behaviour if one hasn't been defined. + if _default_controller_behaviour is None: + _default_controller_behaviour = DefaultDatasetForm() + +def _lookup_plugin(package_type): + """ + Returns the plugin controller associoated with the given package type. + + If the package type is None or cannot be found in the mapping, then the + fallback behaviour is used. + """ + #from pdb import set_trace; set_trace() + if package_type is None: + return _default_controller_behaviour + return _controller_behaviour_for.get(package_type, + _default_controller_behaviour) + +class DefaultDatasetForm(object): + """ + Provides a default implementation of the pluggable package controller behaviour. + + This class has 2 purposes: + + - it provides a base class for IDatasetForm implementations + to use if only a subset of the 5 method hooks need to be customised. + + - it provides the fallback behaviour if no plugin is setup to provide + the fallback behaviour. + + Note - this isn't a plugin implementation. This is deliberate, as + we don't want this being registered. + """ + + def package_form(self): + return 'package/new_package_form.html' + + def form_to_db_schema(self): return package_form_schema() - def _db_to_form_schema(self): + def db_to_form_schema(self): '''This is an interface to manipulate data from the database into a format suitable for the form (optional)''' - def _check_data_dict(self, data_dict): + def check_data_dict(self, data_dict): '''Check if the return data is correct, mostly for checking out if spammers are submitting only part of the form''' # Resources might not exist yet (eg. Add Dataset) surplus_keys_schema = ['__extras', '__junk', 'state', 'groups', 'extras_validation', 'save', 'return_to', - 'resources'] + 'resources', 'type'] schema_keys = package_form_schema().keys() keys_in_schema = set(schema_keys) - set(surplus_keys_schema) @@ -76,10 +158,10 @@ def _check_data_dict(self, data_dict): if missing_keys: #print data_dict #print missing_keys - log.info('incorrect form fields posted') + log.info('incorrect form fields posted, missing %s' % missing_keys ) raise DataError(data_dict) - def _setup_template_variables(self, context, data_dict): + def setup_template_variables(self, context, data_dict): c.groups_authz = get_action('group_list_authz')(context, data_dict) data_dict.update({'available_only':True}) c.groups_available = get_action('group_list_authz')(context, data_dict) @@ -98,7 +180,28 @@ def _setup_template_variables(self, context, data_dict): except NotAuthorized: c.auth_for_change_state = False - ## end hooks +############## End of pluggable package behaviour stuff ############## + +class PackageController(BaseController): + + def _package_form(self, package_type=None): + return _lookup_plugin(package_type).package_form() + + def _form_to_db_schema(self, package_type=None): + return _lookup_plugin(package_type).form_to_db_schema() + + def _db_to_form_schema(self, package_type=None): + '''This is an interface to manipulate data from the database + into a format suitable for the form (optional)''' + return _lookup_plugin(package_type).db_to_form_schema() + + def _check_data_dict(self, data_dict, package_type=None): + '''Check if the return data is correct, mostly for checking out if + spammers are submitting only part of the form''' + return _lookup_plugin(package_type).check_data_dict(data_dict) + + def _setup_template_variables(self, context, data_dict, package_type=None): + return _lookup_plugin(package_type).setup_template_variables(context, data_dict) authorizer = ckan.authz.Authorizer() @@ -182,9 +285,10 @@ def pager_url(q=None, page=None): def read(self, id): + package_type = self._get_package_type(id.split('@')[0]) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, - 'schema': self._form_to_db_schema()} + 'schema': self._form_to_db_schema(package_type=package_type)} data_dict = {'id': id} # interpret @ or @ suffix @@ -233,9 +337,10 @@ def read(self, id): return render('package/read.html') def comments(self, id): + package_type = self._get_package_type(id) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, - 'schema': self._form_to_db_schema()} + 'schema': self._form_to_db_schema(package_type=package_type)} #check if package exists try: @@ -325,10 +430,15 @@ def history(self, id): return render('package/history.html') def new(self, data=None, errors=None, error_summary=None): + + package_type = request.path.strip('/').split('/')[0] + if package_type == 'group': + package_type = None + context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, 'save': 'save' in request.params, - 'schema': self._form_to_db_schema()} + 'schema': self._form_to_db_schema(package_type=package_type)} try: check_access('package_create',context) @@ -344,24 +454,25 @@ def new(self, data=None, errors=None, error_summary=None): vars = {'data': data, 'errors': errors, 'error_summary': error_summary} self._setup_template_variables(context, {'id': id}) - c.form = render(self.package_form, extra_vars=vars) + c.form = render(self._package_form(package_type=package_type), extra_vars=vars) return render('package/new.html') def edit(self, id, data=None, errors=None, error_summary=None): + package_type = self._get_package_type(id) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, 'save': 'save' in request.params, 'moderated': config.get('moderated'), 'pending': True, - 'schema': self._form_to_db_schema()} + 'schema': self._form_to_db_schema(package_type=package_type)} if context['save'] and not data: return self._save_edit(id, context) try: old_data = get_action('package_show')(context, {'id':id}) - schema = self._db_to_form_schema() + schema = self._db_to_form_schema(package_type=package_type) if schema and not data: old_data, errors = validate(old_data, schema, context=context) data = data or old_data @@ -383,21 +494,22 @@ def edit(self, id, data=None, errors=None, error_summary=None): errors = errors or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary} - self._setup_template_variables(context, {'id': id}) + self._setup_template_variables(context, {'id': id}, package_type=package_type) - c.form = render(self.package_form, extra_vars=vars) + c.form = render(self._package_form(package_type=package_type), extra_vars=vars) return render('package/edit.html') def read_ajax(self, id, revision=None): + package_type=self._get_package_type(id) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, - 'schema': self._form_to_db_schema(), + 'schema': self._form_to_db_schema(package_type=package_type), 'revision_id': revision} try: data = get_action('package_show')(context, {'id': id}) - schema = self._db_to_form_schema() + schema = self._db_to_form_schema(package_type=package_type) if schema: data, errors = validate(data, schema) except NotAuthorized: @@ -444,10 +556,36 @@ def history_ajax(self, id): response.headers['Content-Type'] = 'application/json;charset=utf-8' return json.dumps(data) - def _save_new(self, context): + def _get_package_type(self, id): + """ + Given the id of a package it determines the plugin to load + based on the package's type name (type). The plugin found + will be returned, or None if there is no plugin associated with + the type. + + Uses a minimal context to do so. The main use of this method + is for figuring out which plugin to delegate to. + + aborts if an exception is raised. + """ + global _controller_behaviour_for + + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author} + try: + data = get_action('package_show')(context, {'id': id}) + except NotFound: + abort(404, _('Package not found')) + except NotAuthorized: + abort(401, _('Unauthorized to read package %s') % id) + + return data['type'] + + def _save_new(self, context, package_type=None): try: data_dict = clean_dict(unflatten( tuplize_dict(parse_params(request.POST)))) + data_dict['type'] = package_type self._check_data_dict(data_dict) context['message'] = data_dict.get('log_message', '') pkg = get_action('package_create')(context, data_dict) @@ -468,9 +606,10 @@ def _save_new(self, context): def _save_edit(self, id, context): try: + package_type = self._get_package_type(id) data_dict = clean_dict(unflatten( tuplize_dict(parse_params(request.POST)))) - self._check_data_dict(data_dict) + self._check_data_dict(data_dict, package_type=package_type) context['message'] = data_dict.get('log_message', '') if not context['moderated']: context['pending'] = False diff --git a/ckan/controllers/storage.py b/ckan/controllers/storage.py new file mode 100644 index 00000000000..35aba2822da --- /dev/null +++ b/ckan/controllers/storage.py @@ -0,0 +1,399 @@ +import os +import re +import urllib +import uuid +from datetime import datetime +from cgi import FieldStorage + +from ofs import get_impl +from pylons import request, response +from pylons.controllers.util import abort, redirect_to +from pylons import config +from paste.fileapp import FileApp +from paste.deploy.converters import asbool + +from ckan.lib.base import BaseController, c, request, render, config, h, abort +from ckan.lib.jsonp import jsonpify +import ckan.model as model +import ckan.authz as authz + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO +try: + import json +except: + import simplejson as json + +from logging import getLogger +log = getLogger(__name__) + + +# pairtree_version0_1 file for identifying folders +BUCKET = config.get('ckan.storage.bucket', 'default') +key_prefix = config.get('ckan.storage.key_prefix', 'file/') +storage_dir = config.get('ckan.storage.directory', '') + +_eq_re = re.compile(r"^(.*)(=[0-9]*)$") +def fix_stupid_pylons_encoding(data): + """ + Fix an apparent encoding problem when calling request.body + TODO: Investigate whether this is fixed in later versions? + """ + if data.startswith("%") or data.startswith("+"): + data = urllib.unquote_plus(data) + m = _eq_re.match(data) + if m: + data = m.groups()[0] + return data + + +def get_ofs(): + """ + Return a configured instance of the appropriate OFS driver, in all + cases here this will be the local file storage so we fix the implementation + to use pairtree. + """ + return get_impl("pairtree")(storage_dir=storage_dir) + + +pairtree_marker_done = False +def create_pairtree_marker(): + """ + Make sure that the file pairtree_version0_1 is present in storage_dir + and if not then create it. + """ + global pairtree_marker_done + if pairtree_marker_done or not storage_dir: + return + + path = os.path.join(storage_dir, 'pairtree_version0_1') + if not os.path.exists( path ): + open(path, 'w').close() + + pairtree_marker_done = True + + + +def authorize(method, bucket, key, user, ofs): + """ + Check authz for the user with a given bucket/key combo within a + particular ofs implementation. + """ + if not method in ['POST', 'GET', 'PUT', 'DELETE']: + abort(400) + if method != 'GET': + # do not allow overwriting + if ofs.exists(bucket, key): + abort(409) + # now check user stuff + username = user.name if user else '' + is_authorized = authz.Authorizer.is_authorized(username, 'file-upload', model.System()) + if not is_authorized: + h.flash_error('Not authorized to upload files.') + abort(401) + + + +class StorageController(BaseController): + '''Upload to storage backend. + ''' + def __before__(self, action, **params): + super(StorageController, self).__before__(action, **params) + if not storage_dir: + abort(404) + else: + create_pairtree_marker() + + @property + def ofs(self): + return get_ofs() + + + def upload(self): + label = key_prefix + request.params.get('filepath', str(uuid.uuid4())) + c.data = { + 'action': h.url_for('storage_upload_handle'), + 'fields': [ + { + 'name': 'key', + 'value': label + } + ] + } + return render('storage/index.html') + + def upload_handle(self): + bucket_id = BUCKET + params = dict(request.params.items()) + stream = params.get('file') + label = params.get('key') + authorize('POST', BUCKET, label, c.userobj, self.ofs) + if not label: + abort(400, "No label") + if not isinstance(stream, FieldStorage): + abort(400, "No file stream.") + del params['file'] + params['filename-original'] = stream.filename + #params['_owner'] = c.userobj.name if c.userobj else "" + params['uploaded-by'] = c.userobj.name if c.userobj else "" + + self.ofs.put_stream(bucket_id, label, stream.file, params) + success_action_redirect = h.url_for('storage_upload_success', qualified=True, + bucket=BUCKET, label=label) + # Do not redirect here as it breaks js file uploads (get infinite loop + # in FF and crash in Chrome) + return self.success(label) + + def success(self, label=None): + label=request.params.get('label', label) + h.flash_success('Upload successful') + c.file_url = h.url_for('storage_file', + label=label, + qualified=True + ) + c.upload_url = h.url_for('storage_upload') + return render('storage/success.html') + + def success_empty(self, label=None): + # very simple method that just returns 200 OK + return '' + + def file(self, label): + exists = self.ofs.exists(BUCKET, label) + if not exists: + # handle erroneous trailing slash by redirecting to url w/o slash + if label.endswith('/'): + label = label[:-1] + # This may be best being cached_url until we have moved it into + # permanent storage + file_url = h.url_for( 'storage_file', label=label ) + h.redirect_to(file_url) + else: + abort(404) + + file_url = self.ofs.get_url(BUCKET, label) + if file_url.startswith("file://"): + metadata = self.ofs.get_metadata(BUCKET, label) + filepath = file_url[len("file://"):] + headers = { + # 'Content-Disposition':'attachment; filename="%s"' % label, + 'Content-Type':metadata.get('_format', 'text/plain')} + fapp = FileApp(filepath, headers=None, **headers) + return fapp(request.environ, self.start_response) + else: + h.redirect_to(file_url) + + + +class StorageAPIController(BaseController): + + def __before__(self, action, **params): + super(StorageAPIController, self).__before__(action, **params) + if not storage_dir: + abort(404) + else: + create_pairtree_marker() + + @property + def ofs(self): + return get_ofs() + + @jsonpify + def index(self): + info = { + 'metadata/{label}': { + 'description': 'Get or set metadata for this item in storage', + }, + 'auth/request/{label}': { + 'description': self.auth_request.__doc__, + }, + 'auth/form/{label}': { + 'description': self.auth_form.__doc__, + } + } + return info + + def set_metadata(self, label): + bucket = BUCKET + if not label.startswith("/"): label = "/" + label + + try: + data = fix_stupid_pylons_encoding(request.body) + if data: + metadata = json.loads(data) + else: + metadata = {} + except: + abort(400) + + try: + b = self.ofs._require_bucket(bucket) + except: + abort(409) + + k = self.ofs._get_key(b, label) + if k is None: + k = b.new_key(label) + metadata = metadata.copy() + metadata["_creation_time"] = str(datetime.utcnow()) + self.ofs._update_key_metadata(k, metadata) + k.set_contents_from_file(StringIO('')) + elif request.method == "PUT": + old = self.ofs.get_metadata(bucket, label) + to_delete = [] + for ok in old.keys(): + if ok not in metadata: + to_delete.append(ok) + if to_delete: + self.ofs.del_metadata_keys(bucket, label, to_delete) + self.ofs.update_metadata(bucket, label, metadata) + else: + self.ofs.update_metadata(bucket, label, metadata) + + k.make_public() + k.close() + + return self.get_metadata(bucket, label) + + @jsonpify + def get_metadata(self, label): + bucket = BUCKET + url = h.url_for('storage_file', + label=label, + qualified=True + ) + if not self.ofs.exists(bucket, label): + abort(404) + metadata = self.ofs.get_metadata(bucket, label) + metadata["_location"] = url + return metadata + + @jsonpify + def auth_request(self, label): + '''Provide authentication information for a request so a client can + interact with backend storage directly. + + :param label: label. + :param kwargs: sent either via query string for GET or json-encoded + dict for POST). Interpreted as http headers for request plus an + (optional) method parameter (being the HTTP method). + + Examples of headers are: + + Content-Type + Content-Encoding (optional) + Content-Length + Content-MD5 + Expect (should be '100-Continue') + + :return: is a json hash containing various attributes including a + headers dictionary containing an Authorization field which is good for + 15m. + + ''' + bucket = BUCKET + if request.POST: + try: + data = fix_stupid_pylons_encoding(request.body) + headers = json.loads(data) + except Exception, e: + from traceback import print_exc + msg = StringIO() + print_exc(msg) + log.error(msg.seek(0).read()) + abort(400) + else: + headers = dict(request.params) + if 'method' in headers: + method = headers['method'] + del headers['method'] + else: + method = 'POST' + + authorize(method, bucket, label, c.userobj, self.ofs) + + http_request = self.ofs.authenticate_request(method, bucket, label, + headers) + return { + 'host': http_request.host, + 'method': http_request.method, + 'path': http_request.path, + 'headers': http_request.headers + } + + def _get_remote_form_data(self, label): + method = 'POST' + content_length_range = int( + config.get('ckan.storage.max_content_length', + 50000000)) + acl = 'public-read' + fields = [ { + 'name': self.ofs.conn.provider.metadata_prefix + 'uploaded-by', + 'value': c.userobj.name + }] + conditions = [ '{"%s": "%s"}' % (x['name'], x['value']) for x in + fields ] + # In FF redirect to this breaks js upload as FF attempts to open file + # (presumably because mimetype = javascript) and this stops js + # success_action_redirect = h.url_for('storage_api_get_metadata', qualified=True, + # label=label) + success_action_redirect = h.url_for('storage_upload_success_empty', qualified=True, + label=label) + data = self.ofs.conn.build_post_form_args( + BUCKET, + label, + expires_in=72000, + max_content_length=content_length_range, + success_action_redirect=success_action_redirect, + acl=acl, + fields=fields, + conditions=conditions + ) + # HACK: fix up some broken stuff from boto + # e.g. should not have content-length-range in list of fields! + for idx,field in enumerate(data['fields']): + if field['name'] == 'content-length-range': + del data['fields'][idx] + return data + + def _get_form_data(self, label): + data = { + 'action': h.url_for('storage_upload_handle', qualified=True), + 'fields': [ + { + 'name': 'key', + 'value': label + } + ] + } + return data + + @jsonpify + def auth_form(self, label): + '''Provide fields for a form upload to storage including + authentication. + + :param label: label. + :return: json-encoded dictionary with action parameter and fields list. + ''' + bucket = BUCKET + if request.POST: + try: + data = fix_stupid_pylons_encoding(request.body) + headers = json.loads(data) + except Exception, e: + from traceback import print_exc + msg = StringIO() + print_exc(msg) + log.error(msg.seek(0).read()) + abort(400) + else: + headers = dict(request.params) + + method = 'POST' + authorize(method, bucket, label, c.userobj, self.ofs) + data = self._get_form_data(label) + return data + diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 879a9165d9e..7a9a71c79c0 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -182,7 +182,7 @@ def package_dictize(pkg, context): q = select([rel_rev]).where(rel_rev.c.object_package_id == pkg.id) result = _execute_with_revision(q, rel_rev, context) result_dict["relationships_as_object"] = obj_list_dictize(result, context) - + # Extra properties from the domain object # We need an actual Package object for this, not a PackageRevision if isinstance(pkg,PackageRevision): @@ -191,11 +191,15 @@ def package_dictize(pkg, context): # isopen result_dict['isopen'] = pkg.isopen if isinstance(pkg.isopen,bool) else pkg.isopen() + # type + result_dict['type']= pkg.type + # creation and modification date result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() \ if pkg.metadata_modified else None result_dict['metadata_created'] = pkg.metadata_created.isoformat() \ if pkg.metadata_created else None + return result_dict def _get_members(context, group, member_type): diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index e61c5ed5f1d..49fcecb955d 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -143,6 +143,14 @@ def subnav_link(c, text, action, **kwargs): url_for(action=action, **kwargs), class_=('active' if c.action == action else '') ) + +def subnav_named_route(c, text, routename,**kwargs): + """ Generate a subnav element based on a named route """ + return link_to( + text, + url_for(routename, **kwargs), + class_=('active' if c.action == kwargs['action'] else '') + ) def facet_items(c, name, limit=10): from pylons import request diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 73a31107544..0cd266304ad 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -247,7 +247,14 @@ def run(self, query): query['q'] = "*:*" # number of results - query['rows'] = min(1000, int(query.get('rows', 10))) + rows_to_return = min(1000, int(query.get('rows', 10))) + if rows_to_return > 0: + # #1683 Work around problem of last result being out of order + # in SOLR 1.4 + rows_to_query = rows_to_return + 1 + else: + rows_to_query = rows_to_return + query['rows'] = rows_to_query # order by score if no 'sort' term given order_by = query.get('sort') @@ -297,6 +304,9 @@ def run(self, query): self.count = response.get('numFound', 0) self.results = response.get('docs', []) + # #1683 Filter out the last row that is sometimes out of order + self.results = self.results[:rows_to_return] + # get any extras and add to 'extras' dict for result in self.results: extra_keys = filter(lambda x: x.startswith('extras_'), result.keys()) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index b3be534b8d6..4d913365384 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -166,6 +166,7 @@ def resource_update(context, data_dict): context["resource"] = resource if not resource: + logging.error('Could not find resource ' + id) raise NotFound(_('Resource was not found.')) check_access('resource_update', context, data_dict) @@ -181,7 +182,7 @@ def resource_update(context, data_dict): if 'message' in context: rev.message = context['message'] else: - rev.message = _(u'REST API: Update object %s') % data.get("name") + rev.message = _(u'REST API: Update object %s') % data.get("name", "") resource = resource_dict_save(data, context) if not context.get('defer_commit'): @@ -379,7 +380,7 @@ def task_status_update(context, data_dict): if task_status is None: raise NotFound(_('TaskStatus was not found.')) - + check_access('task_status_update', context, data_dict) data, errors = validate(data_dict, schema, context) diff --git a/ckan/logic/auth/update.py b/ckan/logic/auth/update.py index 1206be19c10..b2b15fc6831 100644 --- a/ckan/logic/auth/update.py +++ b/ckan/logic/auth/update.py @@ -1,5 +1,5 @@ from ckan.logic import check_access_old, NotFound -from ckan.logic.auth import get_package_object, get_group_object, get_authorization_group_object, \ +from ckan.logic.auth import get_package_object, get_resource_object, get_group_object, get_authorization_group_object, \ get_user_object, get_resource_object from ckan.logic.auth.create import check_group_auth, package_relationship_create from ckan.authz import Authorizer @@ -150,6 +150,9 @@ def task_status_update(context, data_dict): model = context['model'] user = context['user'] + if 'ignore_auth' in context and context['ignore_auth']: + return {'success': True} + authorized = Authorizer().is_sysadmin(unicode(user)) if not authorized: return {'success': False, 'msg': _('User %s not authorized to update task_status table') % str(user)} diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 9ec6244c3be..3142df75a95 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -142,6 +142,7 @@ def package_form_schema(): schema['extras_validation'] = [duplicate_extras_key, ignore] schema['save'] = [ignore] schema['return_to'] = [ignore] + schema['type'] = [ignore_missing, unicode] ##changes schema.pop("id") @@ -159,6 +160,7 @@ def default_group_schema(): 'name': [not_empty, unicode, name_validator, group_name_validator], 'title': [ignore_missing, unicode], 'description': [ignore_missing, unicode], + 'type': [ignore_missing, unicode], 'state': [ignore_not_group_admin, ignore_missing], 'created': [ignore], 'extras': default_extras_schema(), diff --git a/ckan/migration/versions/047_rename_package_group_member.py b/ckan/migration/versions/047_rename_package_group_member.py index dd3bca18c27..f277762a0dd 100644 --- a/ckan/migration/versions/047_rename_package_group_member.py +++ b/ckan/migration/versions/047_rename_package_group_member.py @@ -101,6 +101,7 @@ def upgrade(migrate_engine): ADD COLUMN "type" text; ALTER TABLE "package_revision" ADD COLUMN "type" text; + COMMIT; ''' diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py index b94e940e5d2..cd0ba3f03c1 100644 --- a/ckan/model/__init__.py +++ b/ckan/model/__init__.py @@ -284,26 +284,6 @@ def _get_revision_user(self): Revision.groups = property(_get_groups) Revision.user = property(_get_revision_user) -def strptimestamp(s): - '''Convert a string of an ISO date into a datetime.datetime object. - - raises TypeError if the number of numbers in the string is not between 3 - and 7 (see datetime constructor). - raises ValueError if any of the numbers are out of range. - ''' - # TODO: METHOD DEPRECATED - use ckan.lib.helpers.date_str_to_datetime - log.warn('model.strptimestamp is deprecated - use ckan.lib.helpers.date_str_to_datetime instead') - import datetime, re - return datetime.datetime(*map(int, re.split('[^\d]', s))) - -def strftimestamp(t): - '''Takes a datetime.datetime and returns it as an ISO string. For - a pretty printed string, use ckan.lib.helpers.render_datetime. - ''' - # TODO: METHOD DEPRECATED - use ckan.lib.helpers.datetime_to_date_str - log.warn('model.strftimestamp is deprecated - use ckan.lib.helpers.datetime_to_date_str instead') - return t.isoformat() - def revision_as_dict(revision, include_packages=True, include_groups=True,ref_package_by='name'): revision_dict = OrderedDict(( ('id', revision.id), diff --git a/ckan/model/authz.py b/ckan/model/authz.py index 85b474483ab..cbe67224322 100644 --- a/ckan/model/authz.py +++ b/ckan/model/authz.py @@ -47,7 +47,8 @@ class Action(Enum): SITE_READ = u'read-site' USER_READ = u'read-user' USER_CREATE = u'create-user' - + UPLOAD_ACTION = u'file-upload' + class Role(Enum): ADMIN = u'admin' EDITOR = u'editor' @@ -67,12 +68,14 @@ class Role(Enum): (Role.EDITOR, Action.USER_READ), (Role.EDITOR, Action.SITE_READ), (Role.EDITOR, Action.READ), + (Role.EDITOR, Action.UPLOAD_ACTION), (Role.ANON_EDITOR, Action.EDIT), (Role.ANON_EDITOR, Action.PACKAGE_CREATE), (Role.ANON_EDITOR, Action.USER_CREATE), (Role.ANON_EDITOR, Action.USER_READ), (Role.ANON_EDITOR, Action.SITE_READ), (Role.ANON_EDITOR, Action.READ), + (Role.ANON_EDITOR, Action.UPLOAD_ACTION), (Role.READER, Action.USER_CREATE), (Role.READER, Action.USER_READ), (Role.READER, Action.SITE_READ), diff --git a/ckan/model/package.py b/ckan/model/package.py index 70df1c6d448..a0272b12406 100644 --- a/ckan/model/package.py +++ b/ckan/model/package.py @@ -37,6 +37,7 @@ Column('maintainer_email', types.UnicodeText), Column('notes', types.UnicodeText), Column('license_id', types.UnicodeText), + Column('type', types.UnicodeText), ) diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index 14d2440c642..94c7d693e45 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -12,13 +12,13 @@ 'IDomainObjectModification', 'IGroupController', 'IPackageController', 'IPluginObserver', 'IConfigurable', 'IConfigurer', 'IAuthorizer', - 'IActions', 'IResourceUrlChange' + 'IActions', 'IResourceUrlChange', 'IDatasetForm', + 'IGroupForm', ] from inspect import isclass from pyutilib.component.core import Interface as _pca_Interface - class Interface(_pca_Interface): @classmethod @@ -361,3 +361,185 @@ def get_auth_functions(self): implementation overrides """ +class IDatasetForm(Interface): + """ + Allows customisation of the package controller as a plugin. + + The behaviour of the plugin is determined by 5 method hooks: + + - package_form(self) + - form_to_db_schema(self) + - db_to_form_schema(self) + - check_data_dict(self, data_dict) + - setup_template_variables(self, context, data_dict) + + Furthermore, there can be many implementations of this plugin registered + at once. With each instance associating itself with 0 or more package + type strings. When a package controller action is invoked, the package + type determines which of the registered plugins to delegate to. Each + implementation must implement two methods which are used to determine the + package-type -> plugin mapping: + + - is_fallback(self) + - package_types(self) + + Implementations might want to consider mixing in + ckan.controllers.package.DefaultPluggablePackageController which provides + default behaviours for the 5 method hooks. + + """ + + ##### These methods control when the plugin is delegated to ##### + + def is_fallback(self): + """ + Returns true iff this provides the fallback behaviour, when no other + plugin instance matches a package's type. + + There must be exactly one fallback controller defined, any attempt to + register more than one will throw an exception at startup. If there's + no fallback registered at startup the + ckan.controllers.package.DefaultPluggablePackageController is used + as the fallback. + """ + + def package_types(self): + """ + Returns an iterable of package type strings. + + If a request involving a package of one of those types is made, then + this plugin instance will be delegated to. + + There must only be one plugin registered to each package type. Any + attempts to register more than one plugin instance to a given package + type will raise an exception at startup. + """ + + ##### End of control methods + + ##### Hooks for customising the PackageController's behaviour ##### + ##### TODO: flesh out the docstrings a little more. ##### + def package_form(self): + """ + Returns a string representing the location of the template to be + rendered. e.g. "package/new_package_form.html". + """ + + def form_to_db_schema(self): + """ + Returns the schema for mapping package data from a form to a format + suitable for the database. + """ + + def db_to_form_schema(self): + """ + Returns the schema for mapping package data from the database into a + format suitable for the form (optional) + """ + + def check_data_dict(self, data_dict): + """ + Check if the return data is correct. + + raise a DataError if not. + """ + + def setup_template_variables(self, context, data_dict): + """ + Add variables to c just prior to the template being rendered. + """ + + ##### End of hooks ##### + + +class IGroupForm(Interface): + """ + Allows customisation of the group controller as a plugin. + + The behaviour of the plugin is determined by 5 method hooks: + + - package_form(self) + - form_to_db_schema(self) + - db_to_form_schema(self) + - check_data_dict(self, data_dict) + - setup_template_variables(self, context, data_dict) + + Furthermore, there can be many implementations of this plugin registered + at once. With each instance associating itself with 0 or more package + type strings. When a package controller action is invoked, the package + type determines which of the registered plugins to delegate to. Each + implementation must implement two methods which are used to determine the + package-type -> plugin mapping: + + - is_fallback(self) + - package_types(self) + + Implementations might want to consider mixing in + ckan.controllers.package.DefaultPluggablePackageController which provides + default behaviours for the 5 method hooks. + + """ + + ##### These methods control when the plugin is delegated to ##### + + def is_fallback(self): + """ + Returns true iff this provides the fallback behaviour, when no other + plugin instance matches a package's type. + + There must be exactly one fallback controller defined, any attempt to + register more than one will throw an exception at startup. If there's + no fallback registered at startup the + ckan.controllers.group.DefaultPluggableGroupController is used + as the fallback. + """ + + def group_types(self): + """ + Returns an iterable of group type strings. + + If a request involving a package of one of those types is made, then + this plugin instance will be delegated to. + + There must only be one plugin registered to each group type. Any + attempts to register more than one plugin instance to a given group + type will raise an exception at startup. + """ + + ##### End of control methods + + ##### Hooks for customising the PackageController's behaviour ##### + ##### TODO: flesh out the docstrings a little more. ##### + + def package_form(self): + """ + Returns a string representing the location of the template to be + rendered. e.g. "group/new_group_form.html". + """ + + def form_to_db_schema(self): + """ + Returns the schema for mapping group data from a form to a format + suitable for the database. + """ + + def db_to_form_schema(self): + """ + Returns the schema for mapping group data from the database into a + format suitable for the form (optional) + """ + + def check_data_dict(self, data_dict): + """ + Check if the return data is correct. + + raise a DataError if not. + """ + + def setup_template_variables(self, context, data_dict): + """ + Add variables to c just prior to the template being rendered. + """ + + ##### End of hooks ##### + diff --git a/ckan/templates/group/layout.html b/ckan/templates/group/layout.html index a5204e7af2c..e4633f8ea3f 100644 --- a/ckan/templates/group/layout.html +++ b/ckan/templates/group/layout.html @@ -8,13 +8,14 @@
    -
  • ${h.subnav_link(c, h.icon('group') + _('View'), controller='group', action='read', id=c.group.name)}
    • -
  • - ${h.subnav_link(c, h.icon('group_edit') + _('Edit'), controller='group', action='edit', id=c.group.name)} +
  • ${h.subnav_named_route(c, h.icon('group') + _('View'), c.group.type + '_read',controller='group', action='read', id=c.group.name)}
    • +
  • + + ${h.subnav_named_route( c,h.icon('group_edit') + _('Edit'), c.group.type + '_action', action='edit', id=c.group.name )}
    • -
  • ${h.subnav_link(c, h.icon('page_white_stack') + _('History'), controller='group', action='history', id=c.group.name)}
    • +
  • ${h.subnav_named_route(c, h.icon('page_white_stack') + _('History'), c.group.type + '_action', controller='group', action='history', id=c.group.name)}
  • - ${h.subnav_link(c, h.icon('lock') + _('Authorization'), controller='group', action='authz', id=c.group.name)} + ${h.subnav_named_route(c, h.icon('lock') + _('Authorization'), c.group.type + '_action', controller='group', action='authz', id=c.group.name)}
    • +
    +
    +
    + +
    +
    + ${h.file('file', size=50)} +
    +
    + +
    + +
    +
    + + + + + + diff --git a/ckan/templates/storage/success.html b/ckan/templates/storage/success.html new file mode 100644 index 00000000000..c3c387ddce0 --- /dev/null +++ b/ckan/templates/storage/success.html @@ -0,0 +1,22 @@ + + + Upload + + + + +
    +

    Upload - Successful

    + +

    Filed uploaded to:

    +

    ${c.file_url}

    + +

    Upload another »

    +
    + + + + diff --git a/ckan/tests/functional/api/test_action.py b/ckan/tests/functional/api/test_action.py index 5b6eb327058..464b69c3123 100644 --- a/ckan/tests/functional/api/test_action.py +++ b/ckan/tests/functional/api/test_action.py @@ -573,7 +573,7 @@ def test_13_group_list_by_size(self): res = self.app.post('/api/action/group_list', params=postparams) res_obj = json.loads(res.body) - assert_equal(res_obj['result'], ['david', + assert_equal(sorted(res_obj['result']), ['david', 'roger']) def test_13_group_list_by_size_all_fields(self): diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index 1e01297c845..7c52f030352 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -102,7 +102,7 @@ def test_read_plugin_hook(self): name = u'david' offset = url_for(controller='group', action='read', id=name) res = self.app.get(offset, status=200, extra_environ={'REMOTE_USER': 'russianfan'}) - assert plugin.calls['read'] == 1, plugin.calls + assert plugin.calls['read'] == 2, plugin.calls plugins.unload(plugin) def test_read_and_authorized_to_edit(self): diff --git a/ckan/tests/functional/test_package.py b/ckan/tests/functional/test_package.py index b46bef39d90..11d57d23931 100644 --- a/ckan/tests/functional/test_package.py +++ b/ckan/tests/functional/test_package.py @@ -355,7 +355,13 @@ def test_read_plugin_hook(self): name = u'annakarenina' offset = url_for(controller='package', action='read', id=name) res = self.app.get(offset) - assert plugin.calls['read'] == 1, plugin.calls + + # There are now two reads of the package. The first to find out + # the package's type. And the second is the actual read that + # existed before. I don't know if this is a problem? I expect it + # can be fixed by allowing the package to be passed in to the plugin, + # either via the function argument, or adding it to the c object. + assert plugin.calls['read'] == 2, plugin.calls plugins.unload(plugin) def test_resource_list(self): diff --git a/ckan/tests/functional/test_pagination.py b/ckan/tests/functional/test_pagination.py index adc95ddd9d9..436622d0469 100644 --- a/ckan/tests/functional/test_pagination.py +++ b/ckan/tests/functional/test_pagination.py @@ -1,8 +1,32 @@ +import re + +from nose.tools import assert_equal + from ckan.lib.create_test_data import CreateTestData import ckan.model as model from ckan.tests import TestController, url_for, setup_test_search_index -class TestPagination(TestController): +def scrape_search_results(response, object_type): + assert object_type in ('dataset', 'group', 'user') + results = re.findall('href="/%s/%s_(\d\d)"' % (object_type, object_type), + str(response)) + return results + +def test_scrape(): + html = ''' +
  • + user_00 +
    • + ... +
  • + user_01 +
    • + + ''' + res = scrape_search_results(html, 'user') + assert_equal(res, ['00', '01']) + +class TestPaginationPackage(TestController): @classmethod def setup_class(cls): setup_test_search_index() @@ -10,63 +34,103 @@ def setup_class(cls): # no. entities per page is hardcoded into the controllers, so # create enough of each here so that we can test pagination - cls.num_groups = 21 cls.num_packages_in_large_group = 51 - cls.num_users = 21 - groups = [u'group_%s' % str(i).zfill(2) for i in range(1, cls.num_groups)] - users = [u'user_%s' % str(i).zfill(2) for i in range(cls.num_users)] packages = [] for i in range(cls.num_packages_in_large_group): packages.append({ - 'name': u'package_%s' % str(i).zfill(2), + 'name': u'dataset_%s' % str(i).zfill(2), 'groups': u'group_00' }) - CreateTestData.create_arbitrary( - packages, extra_group_names=groups, extra_user_names = users, - ) + CreateTestData.create_arbitrary(packages) @classmethod def teardown_class(self): model.repo.rebuild_db() - - def test_search(self): + + def test_package_search_p1(self): res = self.app.get(url_for(controller='package', action='search', q='groups:group_00')) assert 'href="/dataset?q=groups%3Agroup_00&page=2"' in res - assert 'href="/dataset/package_00"' in res, res - assert 'href="/dataset/package_19"' in res, res + pkg_numbers = scrape_search_results(res, 'dataset') + assert_equal(['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19'], pkg_numbers) + def test_package_search_p2(self): res = self.app.get(url_for(controller='package', action='search', q='groups:group_00', page=2)) assert 'href="/dataset?q=groups%3Agroup_00&page=1"' in res - assert 'href="/dataset/package_20"' in res - assert 'href="/dataset/package_39"' in res + pkg_numbers = scrape_search_results(res, 'dataset') + assert_equal(['20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', '35', '36', '37', '38', '39'], pkg_numbers) + + def test_group_read_p1(self): + res = self.app.get(url_for(controller='group', action='read', id='group_00')) + assert 'href="/group/group_00?page=2' in res + pkg_numbers = scrape_search_results(res, 'dataset') + assert_equal(['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29'], pkg_numbers) + + def test_group_read_p2(self): + res = self.app.get(url_for(controller='group', action='read', id='group_00', page=2)) + assert 'href="/group/group_00?page=1' in res + pkg_numbers = scrape_search_results(res, 'dataset') + assert_equal(['30', '31', '32', '33', '34', '35', '36', '37', '38', '39', '40', '41', '42', '43', '44', '45', '46', '47', '48', '49', '50'], pkg_numbers) + +class TestPaginationGroup(TestController): + @classmethod + def setup_class(cls): + # no. entities per page is hardcoded into the controllers, so + # create enough of each here so that we can test pagination + cls.num_groups = 21 + + groups = [u'group_%s' % str(i).zfill(2) for i in range(0, cls.num_groups)] + + CreateTestData.create_arbitrary( + [], extra_group_names=groups + ) + + @classmethod + def teardown_class(self): + model.repo.rebuild_db() def test_group_index(self): res = self.app.get(url_for(controller='group', action='index')) - assert 'href="/group?page=2"' in res - assert 'href="/group/group_19"' in res + assert 'href="/group?page=2"' in res, res + grp_numbers = scrape_search_results(res, 'group') + assert_equal(['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19'], grp_numbers) res = self.app.get(url_for(controller='group', action='index', page=2)) assert 'href="/group?page=1"' in res - assert 'href="/group/group_20"' in res + grp_numbers = scrape_search_results(res, 'group') + assert_equal(['20'], grp_numbers) - def test_group_read(self): - res = self.app.get(url_for(controller='group', action='read', id='group_00')) - assert 'href="/group/group_00?page=2' in res - assert 'href="/dataset/package_29"' in res +class TestPaginationUsers(TestController): + @classmethod + def setup_class(cls): + # Delete default user as it appears in the first page of results + model.User.by_name(u'logged_in').purge() + model.repo.commit_and_remove() - res = self.app.get(url_for(controller='group', action='read', id='group_00', page=2)) - assert 'href="/group/group_00?page=1' in res - assert 'href="/dataset/package_30"' in res + # no. entities per page is hardcoded into the controllers, so + # create enough of each here so that we can test pagination + cls.num_users = 21 + + users = [u'user_%s' % str(i).zfill(2) for i in range(cls.num_users)] + + CreateTestData.create_arbitrary( + [], extra_user_names = users, + ) + + @classmethod + def teardown_class(self): + model.repo.rebuild_db() def test_users_index(self): # allow for 2 extra users shown on user listing, 'logged_in' and 'visitor' res = self.app.get(url_for(controller='user', action='index')) - assert 'href="/user/user_18"' in res assert 'href="/user?q=&order_by=name&page=2"' in res + user_numbers = scrape_search_results(res, 'user') + assert_equal(['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19'], user_numbers) res = self.app.get(url_for(controller='user', action='index', page=2)) - assert 'href="/user/user_20"' in res assert 'href="/user?q=&order_by=name&page=1"' in res + user_numbers = scrape_search_results(res, 'user') + assert_equal(['20'], user_numbers) diff --git a/ckan/tests/functional/test_storage.py b/ckan/tests/functional/test_storage.py new file mode 100644 index 00000000000..bed299334e4 --- /dev/null +++ b/ckan/tests/functional/test_storage.py @@ -0,0 +1,67 @@ +import os +from paste.deploy import appconfig +import paste.fixture +from ckan.config.middleware import make_app +import ckan.model as model +from ckan.tests import conf_dir, url_for, CreateTestData +from ckan.controllers.admin import get_sysadmins + +class TestStorageAPIController: + @classmethod + def setup_class(cls): + config = appconfig('config:test.ini', relative_to=conf_dir) + config.local_conf['ckan.storage.directory'] = '/tmp/' + wsgiapp = make_app(config.global_conf, **config.local_conf) + cls.app = paste.fixture.TestApp(wsgiapp) + + def test_index(self): + url = url_for('storage_api') + res = self.app.get(url) + out = res.json + assert len(res.json) == 3 + + def test_authz(self): + url = url_for('storage_api_auth_form', label='abc') + res = self.app.get(url, status=[200]) + +class TestStorageAPIControllerLocal: + @classmethod + def setup_class(cls): + config = appconfig('config:test.ini', relative_to=conf_dir) + config.local_conf['ckan.storage.directory'] = '/tmp/' + wsgiapp = make_app(config.global_conf, **config.local_conf) + cls.app = paste.fixture.TestApp(wsgiapp) + CreateTestData.create() + model.Session.remove() + user = model.User.by_name('tester') + cls.extra_environ = {'Authorization': str(user.apikey)} + + @classmethod + def teardown_class(cls): + CreateTestData.delete() + + def test_auth_form(self): + url = url_for('storage_api_auth_form', label='abc') + res = self.app.get(url, extra_environ=self.extra_environ, status=200) + assert res.json['action'] == u'http://localhost/storage/upload_handle', res.json + assert res.json['fields'][-1]['value'] == 'abc', res + + url = url_for('storage_api_auth_form', label='abc/xxx') + res = self.app.get(url, extra_environ=self.extra_environ, status=200) + assert res.json['fields'][-1]['value'] == 'abc/xxx' + + def test_metadata(self): + url = url_for('storage_api_get_metadata', label='abc') + res = self.app.get(url, status=404) + + # TODO: test get metadata on real setup ... + label = 'abc' + url = url_for('storage_api_set_metadata', + extra_environ=self.extra_environ, + label=label, + data=dict( + label=label + ) + ) + # res = self.app.get(url, status=404) + diff --git a/ckan/tests/functional/test_upload.py b/ckan/tests/functional/test_upload.py new file mode 100644 index 00000000000..0c25e4b5efa --- /dev/null +++ b/ckan/tests/functional/test_upload.py @@ -0,0 +1,56 @@ +import os +from paste.deploy import appconfig +import paste.fixture + +from ckan.config.middleware import make_app +from ckan.tests import conf_dir, url_for, CreateTestData +import ckan.model as model + + +class TestStorageController: + @classmethod + def setup_class(cls): + config = appconfig('config:test.ini', relative_to=conf_dir) + config.local_conf['ckan.storage.directory'] = '/tmp' + wsgiapp = make_app(config.global_conf, **config.local_conf) + cls.app = paste.fixture.TestApp(wsgiapp) + CreateTestData.create() + + @classmethod + def teardown_class(cls): + model.Session.remove() + model.repo.rebuild_db() + + def test_02_authorization(self): + from ckan.model.authz import Action + import ckan.model as model + import ckan.authz as authz + john = model.User(name=u'john') + model.Session.add(john) + is_authorized = authz.Authorizer.is_authorized(john.name, Action.UPLOAD_ACTION, model.System()) + assert is_authorized + + def test_03_authorization_wui(self): + url = url_for('storage_upload') + res = self.app.get(url, status=[200] ) + if res.status == 302: + res = res.follow() + assert 'Login' in res, res + + def test_04_index(self): + extra_environ = {'REMOTE_USER': 'tester'} + url = url_for('storage_upload') + out = self.app.get(url, extra_environ=extra_environ) + assert 'Upload' in out, out + #assert 'action="https://commondatastorage.googleapis.com/ckan' in out, out + #assert 'key" value="' in out, out + #assert 'policy" value="' in out, out + #assert 'failure_action_redirect' in out, out + #assert 'success_action_redirect' in out, out + + url = url_for('storage_upload', filepath='xyz.txt') + out = self.app.get(url, extra_environ=extra_environ) + assert 'file/xyz.txt' in out, out + + # TODO: test file upload itself + diff --git a/ckan/tests/lib/test_cli.py b/ckan/tests/lib/test_cli.py index f6e383f0c78..4adda216572 100644 --- a/ckan/tests/lib/test_cli.py +++ b/ckan/tests/lib/test_cli.py @@ -19,6 +19,10 @@ def setup_class(cls): model.Package.by_name(u'warandpeace').delete() model.repo.commit_and_remove() + @classmethod + def teardown_class(cls): + model.repo.rebuild_db() + def test_simple_dump_csv(self): csv_filepath = '/tmp/dump.tmp' self.db.args = ('simple-dump-csv %s' % csv_filepath).split() diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index 9d594bba9e4..3d13ce07244 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -49,6 +49,7 @@ def setup_class(cls): 'license_id': u'other-open', 'maintainer': None, 'maintainer_email': None, + 'type': None, 'name': u'annakarenina', 'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n\n\n', 'relationships_as_object': [], @@ -103,11 +104,6 @@ def teardown_class(cls): model.repo.rebuild_db() model.Session.remove() - def teardonwn(self): - model.Session.remove() - - - def remove_changable_columns(self, dict): for key, value in dict.items(): if key.endswith('id') and key <> 'license_id': @@ -150,6 +146,7 @@ def test_01_dictize_main_objects_simple(self): 'maintainer': None, 'maintainer_email': None, 'name': u'annakarenina', + 'type': None, 'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n\n\n', 'state': u'active', 'title': u'A Novel By Tolstoy', @@ -883,6 +880,7 @@ def test_16_group_dictized(self): 'license_id': u'other-open', 'maintainer': None, 'maintainer_email': None, + 'type': None, 'name': u'annakarenina3', 'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n\n\n', 'state': u'active', @@ -897,6 +895,7 @@ def test_16_group_dictized(self): 'license_id': u'other-open', 'maintainer': None, 'maintainer_email': None, + 'type': None, 'name': u'annakarenina2', 'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n\n\n', 'state': u'active', diff --git a/ckan/tests/lib/test_dictization_schema.py b/ckan/tests/lib/test_dictization_schema.py index abb7d828160..2cac490b366 100644 --- a/ckan/tests/lib/test_dictization_schema.py +++ b/ckan/tests/lib/test_dictization_schema.py @@ -141,6 +141,7 @@ def test_2_group_schema(self): expected = {'description': u'These are books that David likes.', 'id': group.id, 'name': u'david', + 'type': u'group', 'packages': sorted([{'id': group_pack[0].id}, {'id': group_pack[1].id, }], key=lambda x:x["id"]), diff --git a/ckan/tests/models/test_repo.py b/ckan/tests/models/test_repo.py index 12b7a8a31a8..d08b2b83e28 100644 --- a/ckan/tests/models/test_repo.py +++ b/ckan/tests/models/test_repo.py @@ -14,12 +14,14 @@ '', '', '', + '', '', '', '', '', '', '', + '', '', '', '', diff --git a/ckan/tests/test_plugins.py b/ckan/tests/test_plugins.py index d2d002ff664..ad32731e578 100644 --- a/ckan/tests/test_plugins.py +++ b/ckan/tests/test_plugins.py @@ -168,7 +168,7 @@ def test_mapper_plugin_fired(self): config['ckan.plugins'] = 'mapper_plugin' plugins.load_all(config) CreateTestData.create_arbitrary([{'name':u'testpkg'}]) - mapper_plugin = PluginGlobals.plugin_registry['MapperPlugin'].__instance__ + mapper_plugin = PluginGlobals.env().plugin_registry['MapperPlugin'].__instance__ assert len(mapper_plugin.added) == 2 # resource group table added automatically assert mapper_plugin.added[0].name == 'testpkg' @@ -176,7 +176,7 @@ def test_routes_plugin_fired(self): local_config = appconfig('config:%s' % config['__file__'], relative_to=conf_dir) local_config.local_conf['ckan.plugins'] = 'routes_plugin' app = make_app(local_config.global_conf, **local_config.local_conf) - routes_plugin = PluginGlobals.plugin_registry['RoutesPlugin'].__instance__ + routes_plugin = PluginGlobals.env().plugin_registry['RoutesPlugin'].__instance__ assert routes_plugin.calls_made == ['before_map', 'after_map'], \ routes_plugin.calls_made diff --git a/doc/common-error-messages.rst b/doc/common-error-messages.rst index a072112f056..e453daddca4 100644 --- a/doc/common-error-messages.rst +++ b/doc/common-error-messages.rst @@ -129,3 +129,7 @@ This occurs when installing CKAN source to a virtual environment when using an o This occurs when upgrading to CKAN 1.5.1 with a database with duplicate user names. See :ref:`upgrading` +``ERROR: must be member of role "okfn"`` & ``WARNING: no privileges could be revoked for "public"`` +===================================================================================================== + +These are seen when loading a CKAN database from another machine. It is the result of the database tables being owned by a user that doesn't exist on the new machine. The owner of the table is not important, so this error is harmless and can be ignored. \ No newline at end of file diff --git a/doc/configuration.rst b/doc/configuration.rst index 71756db831c..6b069fa7e9a 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -167,6 +167,8 @@ And there is an option for the default expiry time if not specified:: ckan.cache.default_expires = 600 + + Authentication Settings ----------------------- @@ -242,6 +244,35 @@ Default value: (none) If you want to specify the ordering of all or some of the locales as they are offered to the user, then specify them here in the required order. Any locales that are available but not specified in this option, will still be offered at the end of the list. +Storage Settings +---------------- + +.. index:: + single: ckan.storage.bucket, ckan.storage.directory + +ckan.storage.bucket +^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.storage.bucket = ckan + +Default value: ``None`` + +This setting will change the bucket name for the uploaded files. + +ckan.storage.directory +^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.storage.directory = /data/uploads/ + +Default value: ``None`` + +Use this to specify where uploaded files should be stored, and also to turn on the handling of file storage. The folder should exist, and will automatically be turned into a valid pairtree repository if it is not already. + + Theming Settings ---------------- diff --git a/doc/file-upload.rst b/doc/file-upload.rst new file mode 100644 index 00000000000..f093b1f2bbc --- /dev/null +++ b/doc/file-upload.rst @@ -0,0 +1,19 @@ +============ +File uploads +============ + +CKAN allows users to upload files directly to file storage on the CKAN server. The uploaded files will be stored in the configured location. + +The important settings for the CKAN .ini file are + +:: + + ckan.storage.bucket = ckan + ckan.storage.directory = /data/uploads/ + +The directory where files will be stored should exist or be created before the system is used. + +It is also possible to have uploaded CSV and Excel files stored in the Webstore which provides a structured data store built on a relational database backend. The configuration of this process is described at `the CKAN wiki `_. + +Storing data in the webstore allows for the direct retrieval of the data in a tabular format. It is possible to fetch a single row of the data, all of the data and have it returned in HTML, CSV or JSON format. More information and the API documentation for the webstore is available at `read the docs +`_. \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst index a5539df13c0..57669fece51 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,6 +31,7 @@ Contents: database_dumps upgrade i18n + file-upload configuration api test diff --git a/doc/install-from-package.rst b/doc/install-from-package.rst index a6e927fecda..17b98a483ee 100644 --- a/doc/install-from-package.rst +++ b/doc/install-from-package.rst @@ -721,7 +721,7 @@ Now you need to make some manual changes. In the following commands replace ``st If you get error ``sqlalchemy.exc.IntegrityError: (IntegrityError) could not create unique index "user_name_key`` then you need to rename users with duplicate names before it will work. For example:: sudo -u ckanstd paster --plugin=pylons shell /etc/ckan/std/std.ini - model.meta.engine.execute('SELECT name, count(name) AS NumOccurrences FROM "user" GROUP BY name HAVING(COUNT(name)>0);').fetchall() + model.meta.engine.execute('SELECT name, count(name) AS NumOccurrences FROM "user" GROUP BY name HAVING(COUNT(name)>1);').fetchall() users = model.Session.query(model.User).filter_by(name='https://www.google.com/accounts/o8/id?id=ABCDEF').all() users[1].name = users[1].name[:-1] model.repo.commit_and_remove() diff --git a/requires/lucid_missing.txt b/requires/lucid_missing.txt index 284c18fc29d..89e2eb8cac7 100644 --- a/requires/lucid_missing.txt +++ b/requires/lucid_missing.txt @@ -14,9 +14,10 @@ # Packages already on pypi.python.org solrpy==0.9.4 formalchemy==1.4.1 +pairtree==0.7.1-T +ofs>=0.4.1 apachemiddleware==0.1.1 licenses==0.6.1 # markupsafe is required by webhelpers==1.2 required by formalchemy with SQLAlchemy 0.6 markupsafe==0.9.2 -