diff --git a/Dockerfile b/Dockerfile index f33fcc2f6a6..c8d2fe835ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.9.10 +FROM phusion/baseimage:0.9.15 MAINTAINER Open Knowledge # Disable SSH diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 49317cf0b0f..1ee0b66c034 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -825,10 +825,15 @@ def activity(self, id, offset=0): except (NotFound, NotAuthorized): abort(404, _('Group not found')) - # Add the group's activity stream (already rendered to HTML) to the - # template context for the group/read.html template to retrieve later. - c.group_activity_stream = self._action('group_activity_list_html')( - context, {'id': c.group_dict['id'], 'offset': offset}) + try: + # Add the group's activity stream (already rendered to HTML) to the + # template context for the group/read.html + # template to retrieve later. + c.group_activity_stream = self._action('group_activity_list_html')( + context, {'id': c.group_dict['id'], 'offset': offset}) + + except ValidationError as error: + base.abort(400) return render(self._activity_template(group_type), extra_vars={'group_type': group_type}) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index f939435aae1..0064500eb35 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -1145,7 +1145,7 @@ def resource_download(self, id, resource_id, filename=None): abort(404, _('Resource not found')) if rsc.get('url_type') == 'upload': - upload = uploader.ResourceUpload(rsc) + upload = uploader.get_resource_uploader(rsc) filepath = upload.get_path(rsc['id']) fileapp = paste.fileapp.FileApp(filepath) try: diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index d911c214bfb..4c6ef562d55 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -585,8 +585,11 @@ def activity(self, id, offset=0): self._setup_template_variables(context, data_dict) - c.user_activity_stream = get_action('user_activity_list_html')( - context, {'id': c.user_dict['id'], 'offset': offset}) + try: + c.user_activity_stream = get_action('user_activity_list_html')( + context, {'id': c.user_dict['id'], 'offset': offset}) + except ValidationError: + base.abort(400) return render('user/activity_stream.html') diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 2dd8fec9a7d..b236f879a9c 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -142,13 +142,13 @@ def redirect_to(*args, **kw): return _redirect_to(url_for(*args, **kw)) +@maintain.deprecated('h.url is deprecated please use h.url_for') @core_helper def url(*args, **kw): - '''Create url adding i18n information if selected - wrapper for pylons.url''' - locale = kw.pop('locale', None) - my_url = _pylons_default_url(*args, **kw) - return _local_url(my_url, locale=locale, **kw) + ''' + Deprecated: please use `url_for` instead + ''' + return url_for(*args, **kw) @core_helper @@ -1039,7 +1039,7 @@ def pager_url(page, partial=None, **kwargs): if routes_dict.get('id'): kwargs['id'] = routes_dict['id'] kwargs['page'] = page - return url(**kwargs) + return url_for(**kwargs) class Page(paginate.Page): diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index c3145dd8054..c6c7c251726 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -1,3 +1,7 @@ +from formencode.validators import OneOf + +import ckan.model +import ckan.plugins as plugins from ckan.lib.navl.validators import (ignore_missing, keep_extras, not_empty, @@ -6,60 +10,7 @@ if_empty_same_as, not_missing, ignore_empty - ) -from ckan.logic.validators import (package_id_not_changed, - package_id_exists, - package_id_or_name_exists, - resource_id_exists, - name_validator, - package_name_validator, - package_version_validator, - group_name_validator, - tag_length_validator, - tag_name_validator, - tag_string_convert, - duplicate_extras_key, - ignore_not_package_admin, - ignore_not_group_admin, - ignore_not_sysadmin, - no_http, - tag_not_uppercase, - user_name_validator, - user_password_validator, - user_both_passwords_entered, - user_passwords_match, - user_password_not_empty, - isodate, - int_validator, - natural_number_validator, - is_positive_integer, - boolean_validator, - user_about_validator, - vocabulary_name_validator, - vocabulary_id_not_changed, - vocabulary_id_exists, - user_id_exists, - user_id_or_name_exists, - object_id_validator, - activity_type_exists, - resource_id_exists, - tag_not_in_vocabulary, - group_id_exists, - owner_org_validator, - user_name_exists, - role_exists, - url_validator, - datasets_with_no_organization_cannot_be_private, - list_of_strings, - if_empty_guess_format, - clean_format, - no_loops_in_hierarchy, - filter_fields_and_values_should_have_same_length, - filter_fields_and_values_exist_and_are_valid, - extra_key_not_in_root_schema, - empty_if_not_sysadmin, - package_id_does_not_exist, - ) + ) from ckan.logic.converters import (convert_user_name_or_id_to_id, convert_package_name_or_id_to_id, convert_group_name_or_id_to_id, @@ -68,20 +19,65 @@ remove_whitespace, extras_unicode_convert, ) -from formencode.validators import OneOf -import ckan.model -import ckan.plugins as plugins +from ckan.logic.validators import ( + package_id_not_changed, + package_id_or_name_exists, + name_validator, + package_name_validator, + package_version_validator, + group_name_validator, + tag_length_validator, + tag_name_validator, + tag_string_convert, + duplicate_extras_key, + ignore_not_package_admin, + ignore_not_group_admin, + ignore_not_sysadmin, + no_http, + user_name_validator, + user_password_validator, + user_both_passwords_entered, + user_passwords_match, + user_password_not_empty, + isodate, + int_validator, + natural_number_validator, + is_positive_integer, + boolean_validator, + user_about_validator, + vocabulary_name_validator, + vocabulary_id_not_changed, + vocabulary_id_exists, + object_id_validator, + activity_type_exists, + resource_id_exists, + tag_not_in_vocabulary, + group_id_exists, + owner_org_validator, + user_name_exists, + role_exists, + datasets_with_no_organization_cannot_be_private, + list_of_strings, + if_empty_guess_format, + clean_format, + no_loops_in_hierarchy, + filter_fields_and_values_should_have_same_length, + filter_fields_and_values_exist_and_are_valid, + extra_key_not_in_root_schema, + empty_if_not_sysadmin, + package_id_does_not_exist, + ) def default_resource_schema(): - schema = { 'id': [ignore_empty, unicode], 'revision_id': [ignore_missing, unicode], 'package_id': [ignore], 'url': [not_empty, unicode, remove_whitespace], 'description': [ignore_missing, unicode], - 'format': [if_empty_guess_format, ignore_missing, clean_format, unicode], + 'format': [if_empty_guess_format, ignore_missing, clean_format, + unicode], 'hash': [ignore_missing, unicode], 'state': [ignore], 'position': [ignore], @@ -102,11 +98,13 @@ def default_resource_schema(): return schema + def default_update_resource_schema(): schema = default_resource_schema() schema['revision_id'] = [ignore] return schema + def default_tags_schema(): schema = { 'name': [not_missing, @@ -114,7 +112,7 @@ def default_tags_schema(): unicode, tag_length_validator, tag_name_validator, - ], + ], 'vocabulary_id': [ignore_missing, unicode, vocabulary_id_exists], 'revision_timestamp': [ignore], 'state': [ignore], @@ -122,13 +120,14 @@ def default_tags_schema(): } return schema + def default_create_tag_schema(): schema = default_tags_schema() # When creating a tag via the tag_create() logic action function, a # vocabulary_id _must_ be given (you cannot create free tags via this # function). schema['vocabulary_id'] = [not_missing, not_empty, unicode, - vocabulary_id_exists, tag_not_in_vocabulary] + vocabulary_id_exists, tag_not_in_vocabulary] # You're not allowed to specify your own ID when creating a tag. schema['id'] = [empty] return schema @@ -137,7 +136,8 @@ def default_create_tag_schema(): def default_create_package_schema(): schema = { '__before': [duplicate_extras_key, ignore], - 'id': [empty_if_not_sysadmin, ignore_missing, unicode, package_id_does_not_exist], + 'id': [empty_if_not_sysadmin, ignore_missing, unicode, + package_id_does_not_exist], 'revision_id': [ignore], 'name': [not_empty, unicode, name_validator, package_name_validator], 'title': [if_empty_same_as("name"), unicode], @@ -147,14 +147,14 @@ def default_create_package_schema(): 'maintainer_email': [ignore_missing, unicode], 'license_id': [ignore_missing, unicode], 'notes': [ignore_missing, unicode], - 'url': [ignore_missing, unicode],#, URL(add_http=False)], + 'url': [ignore_missing, unicode], # , URL(add_http=False)], 'version': [ignore_missing, unicode, package_version_validator], 'state': [ignore_not_package_admin, ignore_missing], 'type': [ignore_missing, unicode], 'owner_org': [owner_org_validator, unicode], 'log_message': [ignore_missing, unicode, no_http], 'private': [ignore_missing, boolean_validator, - datasets_with_no_organization_cannot_be_private], + datasets_with_no_organization_cannot_be_private], '__extras': [ignore], '__junk': [empty], 'resources': default_resource_schema(), @@ -174,6 +174,7 @@ def default_create_package_schema(): } return schema + def default_update_package_schema(): schema = default_create_package_schema() @@ -186,7 +187,7 @@ def default_update_package_schema(): # Supplying the package name when updating a package is optional (you can # supply the id to identify the package instead). schema['name'] = [ignore_missing, name_validator, package_name_validator, - unicode] + unicode] # Supplying the package title when updating a package is optional, if it's # not supplied the title will not be changed. @@ -196,6 +197,7 @@ def default_update_package_schema(): return schema + def default_show_package_schema(): schema = default_create_package_schema() @@ -203,22 +205,20 @@ def default_show_package_schema(): schema['id'] = [] schema.update({ - 'tags': {'__extras': [ckan.lib.navl.validators.keep_extras]}}) + 'tags': {'__extras': [keep_extras]}}) # Add several keys to the 'resources' subschema so they don't get stripped # from the resource dicts by validation. schema['resources'].update({ 'format': [ignore_missing, clean_format, unicode], - 'created': [ckan.lib.navl.validators.ignore_missing], + 'created': [ignore_missing], 'position': [not_empty], - 'last_modified': [ckan.lib.navl.validators.ignore_missing], - 'cache_last_updated': [ckan.lib.navl.validators.ignore_missing], + 'last_modified': [], + 'cache_last_updated': [], 'revision_id': [], 'package_id': [], - 'cache_last_updated': [], 'size': [], 'state': [], - 'last_modified': [], 'mimetype': [], 'cache_url': [], 'name': [], @@ -228,17 +228,17 @@ def default_show_package_schema(): }) schema.update({ - 'state': [ckan.lib.navl.validators.ignore_missing], + 'state': [ignore_missing], 'isopen': [ignore_missing], 'license_url': [ignore_missing], 'revision_id': [], - }) + }) schema['groups'].update({ 'description': [ignore_missing], 'display_name': [ignore_missing], 'image_display_url': [ignore_missing], - }) + }) # Remove validators for several keys from the schema so validation doesn't # strip the keys from the package dicts if the values are 'missing' (i.e. @@ -252,8 +252,8 @@ def default_show_package_schema(): schema['url'] = [] schema['version'] = [] - # Add several keys that are missing from default_create_package_schema(), so - # validation doesn't strip the keys from the package dicts. + # Add several keys that are missing from default_create_package_schema(), + # so validation doesn't strip the keys from the package dicts. schema['metadata_created'] = [] schema['metadata_modified'] = [] schema['creator_user_id'] = [] @@ -268,8 +268,8 @@ def default_show_package_schema(): return schema -def default_group_schema(): +def default_group_schema(): schema = { 'id': [ignore_missing, unicode], 'revision_id': [ignore], @@ -288,8 +288,8 @@ def default_group_schema(): '__junk': [ignore], 'packages': { "id": [not_empty, unicode, package_id_or_name_exists], - "title":[ignore_missing, unicode], - "name":[ignore_missing, unicode], + "title": [ignore_missing, unicode], + "name": [ignore_missing, unicode], "__extras": [ignore] }, 'users': { @@ -305,9 +305,10 @@ def default_group_schema(): } return schema + def group_form_schema(): schema = default_group_schema() - #schema['extras_validation'] = [duplicate_extras_key, ignore] + # schema['extras_validation'] = [duplicate_extras_key, ignore] schema['packages'] = { "name": [not_empty, unicode], "title": [ignore_missing], @@ -327,6 +328,7 @@ def default_update_group_schema(): schema["name"] = [ignore_missing, group_name_validator, unicode] return schema + def default_show_group_schema(): schema = default_group_schema() @@ -334,17 +336,17 @@ def default_show_group_schema(): schema['num_followers'] = [] schema['created'] = [] schema['display_name'] = [] - schema['extras'] = {'__extras': [ckan.lib.navl.validators.keep_extras]} + schema['extras'] = {'__extras': [keep_extras]} schema['package_count'] = [] - schema['packages'] = {'__extras': [ckan.lib.navl.validators.keep_extras]} + schema['packages'] = {'__extras': [keep_extras]} schema['revision_id'] = [] schema['state'] = [] - schema['users'] = {'__extras': [ckan.lib.navl.validators.keep_extras]} + schema['users'] = {'__extras': [keep_extras]} return schema -def default_extras_schema(): +def default_extras_schema(): schema = { 'id': [ignore], 'key': [not_empty, extra_key_not_in_root_schema, unicode], @@ -356,20 +358,21 @@ def default_extras_schema(): } return schema -def default_relationship_schema(): +def default_relationship_schema(): schema = { - 'id': [ignore_missing, unicode], - 'subject': [ignore_missing, unicode], - 'object': [ignore_missing, unicode], - 'type': [not_empty, OneOf(ckan.model.PackageRelationship.get_all_types())], - 'comment': [ignore_missing, unicode], - 'state': [ignore], + 'id': [ignore_missing, unicode], + 'subject': [ignore_missing, unicode], + 'object': [ignore_missing, unicode], + 'type': [not_empty, + OneOf(ckan.model.PackageRelationship.get_all_types())], + 'comment': [ignore_missing, unicode], + 'state': [ignore], } return schema -def default_create_relationship_schema(): +def default_create_relationship_schema(): schema = default_relationship_schema() schema['id'] = [empty] schema['subject'] = [not_empty, unicode, package_id_or_name_exists] @@ -377,8 +380,8 @@ def default_create_relationship_schema(): return schema -def default_update_relationship_schema(): +def default_update_relationship_schema(): schema = default_relationship_schema() schema['id'] = [ignore_missing, package_id_not_changed] @@ -391,15 +394,13 @@ def default_update_relationship_schema(): return schema - - def default_user_schema(): - schema = { 'id': [ignore_missing, unicode], 'name': [not_empty, name_validator, user_name_validator, unicode], 'fullname': [ignore_missing, unicode], - 'password': [user_password_validator, user_password_not_empty, ignore_missing, unicode], + 'password': [user_password_validator, user_password_not_empty, + ignore_missing, unicode], 'password_hash': [ignore_missing, ignore_not_sysadmin, unicode], 'email': [not_empty, unicode], 'about': [ignore_missing, user_about_validator, unicode], @@ -413,37 +414,45 @@ def default_user_schema(): } return schema + def user_new_form_schema(): schema = default_user_schema() - schema['password1'] = [unicode,user_both_passwords_entered,user_password_validator,user_passwords_match] + schema['password1'] = [unicode, user_both_passwords_entered, + user_password_validator, user_passwords_match] schema['password2'] = [unicode] return schema + def user_edit_form_schema(): schema = default_user_schema() schema['password'] = [ignore_missing] - schema['password1'] = [ignore_missing,unicode,user_password_validator,user_passwords_match] - schema['password2'] = [ignore_missing,unicode] + schema['password1'] = [ignore_missing, unicode, user_password_validator, + user_passwords_match] + schema['password2'] = [ignore_missing, unicode] return schema + def default_update_user_schema(): schema = default_user_schema() - schema['name'] = [ignore_missing, name_validator, user_name_validator, unicode] - schema['password'] = [user_password_validator,ignore_missing, unicode] + schema['name'] = [ignore_missing, name_validator, user_name_validator, + unicode] + schema['password'] = [user_password_validator, ignore_missing, unicode] return schema + def default_generate_apikey_user_schema(): schema = default_update_user_schema() schema['apikey'] = [not_empty, unicode] return schema + def default_user_invite_schema(): schema = { 'email': [not_empty, unicode], @@ -452,6 +461,7 @@ def default_user_invite_schema(): } return schema + def default_task_status_schema(): schema = { 'id': [ignore], @@ -466,6 +476,7 @@ def default_task_status_schema(): } return schema + def default_vocabulary_schema(): schema = { 'id': [ignore_missing, unicode, vocabulary_id_exists], @@ -474,41 +485,46 @@ def default_vocabulary_schema(): } return schema + def default_create_vocabulary_schema(): schema = default_vocabulary_schema() schema['id'] = [empty] return schema + def default_update_vocabulary_schema(): schema = default_vocabulary_schema() schema['id'] = [ignore_missing, vocabulary_id_not_changed] schema['name'] = [ignore_missing, vocabulary_name_validator] return schema + def default_create_activity_schema(): schema = { 'id': [ignore], 'timestamp': [ignore], 'user_id': [not_missing, not_empty, unicode, - convert_user_name_or_id_to_id], + convert_user_name_or_id_to_id], 'object_id': [not_missing, not_empty, unicode, object_id_validator], # We don't bother to validate revision ID, since it's always created # internally by the activity_create() logic action function. 'revision_id': [], 'activity_type': [not_missing, not_empty, unicode, - activity_type_exists], + activity_type_exists], 'data': [ignore_empty, ignore_missing], } return schema + def default_follow_user_schema(): schema = {'id': [not_missing, not_empty, unicode, - convert_user_name_or_id_to_id]} + convert_user_name_or_id_to_id]} return schema + def default_follow_dataset_schema(): schema = {'id': [not_missing, not_empty, unicode, - convert_package_name_or_id_to_id]} + convert_package_name_or_id_to_id]} return schema @@ -523,7 +539,7 @@ def member_schema(): def default_follow_group_schema(): schema = {'id': [not_missing, not_empty, unicode, - convert_group_name_or_id_to_id]} + convert_group_name_or_id_to_id]} return schema @@ -576,8 +592,9 @@ def default_package_search_schema(): 'facet.mincount': [ignore_missing, natural_number_validator], 'facet.limit': [ignore_missing, int_validator], 'facet.field': [ignore_missing, convert_to_json_if_string, - list_of_strings], - 'extras': [ignore_missing] # Not used by Solr, but useful for extensions + list_of_strings], + 'extras': [ignore_missing] # Not used by Solr, + # but useful for extensions } return schema @@ -627,14 +644,13 @@ def default_update_resource_view_schema(resource_view): 'id': [not_missing, not_empty, unicode], 'resource_id': [ignore_missing, resource_id_exists], 'title': [ignore_missing, unicode], - 'view_type': [ignore],# can not change after create + 'view_type': [ignore], # cannot change after create 'package_id': [ignore] }) return schema def default_update_configuration_schema(): - schema = { 'ckan.site_title': [unicode], 'ckan.site_logo': [unicode], diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index 005b5d6a68b..e29d390f132 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -1492,7 +1492,7 @@ class IUploader(Interface): upload resources and group images. ''' - def get_uploader(self): + def get_uploader(self, upload_to, old_filename): '''Return an uploader object to upload general files that must implement the following methods: diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 711475b1f7f..97f9301d71c 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -86,7 +86,7 @@ {# Allows custom attributes to be added to the tag #} - + {# The page block allows you to add content to the page. Most of the time it is diff --git a/ckan/templates/group/snippets/feeds.html b/ckan/templates/group/snippets/feeds.html index fb7c48c26c1..dabed46ae1d 100644 --- a/ckan/templates/group/snippets/feeds.html +++ b/ckan/templates/group/snippets/feeds.html @@ -1,4 +1,4 @@ -{%- set dataset_feed = h.url(controller='feed', action='group', id=c.group_dict.name) -%} -{%- set history_feed = h.url(controller='revision', action='list', format='atom', days=1) -%} +{%- set dataset_feed = h.url_for(controller='feed', action='group', id=c.group_dict.name) -%} +{%- set history_feed = h.url_for(controller='revision', action='list', format='atom', days=1) -%} diff --git a/ckan/templates/header.html b/ckan/templates/header.html index 46d91d591d3..117a5dc5ca9 100644 --- a/ckan/templates/header.html +++ b/ckan/templates/header.html @@ -82,10 +82,10 @@ {% block header_logo %} {% if g.site_logo %} - + {% else %}

- {{ g.site_title }} + {{ g.site_title }}

{% if g.site_description %}

{{ g.site_description }}

{% endif %} {% endif %} diff --git a/ckan/templates/organization/snippets/feeds.html b/ckan/templates/organization/snippets/feeds.html index 8fdffd00bb7..dd26a4d5043 100644 --- a/ckan/templates/organization/snippets/feeds.html +++ b/ckan/templates/organization/snippets/feeds.html @@ -1,4 +1,4 @@ -{%- set dataset_feed = h.url(controller='feed', action='organization', id=c.group_dict.name) -%} -{%- set history_feed = h.url(controller='revision', action='list', format='atom', days=1) -%} +{%- set dataset_feed = h.url_for(controller='feed', action='organization', id=c.group_dict.name) -%} +{%- set history_feed = h.url_for(controller='revision', action='list', format='atom', days=1) -%} diff --git a/ckan/templates/package/read_base.html b/ckan/templates/package/read_base.html index e968accd0b2..448f31a8706 100644 --- a/ckan/templates/package/read_base.html +++ b/ckan/templates/package/read_base.html @@ -27,7 +27,7 @@

{% set timestamp = h.render_datetime(c.revision_date, with_hours=True) %} - {% set url = h.url(controller='package', action='read', id=pkg.name) %} + {% set url = h.url_for(controller='package', action='read', id=pkg.name) %} {% trans timestamp=timestamp, url=url %}This is an old revision of this dataset, as edited at {{ timestamp }}. It may differ significantly from the current revision.{% endtrans %}

diff --git a/ckan/templates/package/snippets/resource_view.html b/ckan/templates/package/snippets/resource_view.html index 36ec061ce9d..77cf21a4e82 100644 --- a/ckan/templates/package/snippets/resource_view.html +++ b/ckan/templates/package/snippets/resource_view.html @@ -6,7 +6,7 @@ href="#embed-{{ resource_view['id'] }}" data-module="resource-view-embed" data-module-id="{{ resource_view['id'] }}" - data-module-url="{{ h.url('resource_view', id=package['name'], resource_id=resource['id'], view_id=resource_view['id'], qualified=True) }}"> + data-module-url="{{ h.url_for('resource_view', id=package['name'], resource_id=resource['id'], view_id=resource_view['id'], qualified=True) }}"> {{ _("Embed") }} @@ -37,20 +37,20 @@ {% if not to_preview %} {% set current_filters = request.str_GET.get('filters') %} {% if current_filters %} - {% set src = h.url(qualified=true, controller='package', + {% set src = h.url_for(qualified=true, controller='package', action='resource_view', id=package['name'], resource_id=resource['id'], view_id=resource_view['id'], filters=current_filters) %} {% else %} - {% set src = h.url(qualified=true, controller='package', + {% set src = h.url_for(qualified=true, controller='package', action='resource_view', id=package['name'], resource_id=resource['id'], view_id=resource_view['id']) %} {% endif %} {% else %} {# When previewing we need to stick the whole resource_view as a param as there is no other way to pass to information on to the iframe #} - {% set src = h.url(qualified=true, controller='package', action='resource_view', id=package['name'], resource_id=resource['id']) + '?' + h.urlencode({'resource_view': h.dump_json(resource_view)}) %} + {% set src = h.url_for(qualified=true, controller='package', action='resource_view', id=package['name'], resource_id=resource['id']) + '?' + h.urlencode({'resource_view': h.dump_json(resource_view)}) %} {% endif %}