diff --git a/MANIFEST.in b/MANIFEST.in index 4043246a3df..692d90ba45f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include ckan/config/deployment.ini_tmpl recursive-include ckan/public * recursive-include ckan/config *.ini +recursive-include ckan/config *.json recursive-include ckan/config *.xml recursive-include ckan/i18n * recursive-include ckan/templates * diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index b15d1c5657b..7b9f632516f 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -136,7 +136,8 @@ ckan.feeds.author_link = #ckan.activity_streams_enabled = true #ckan.activity_list_limit = 31 #ckan.activity_streams_email_notifications = true -# ckan.email_notifications_since = 2 days +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s ## Email settings diff --git a/ckan/config/resource_formats.json b/ckan/config/resource_formats.json new file mode 100644 index 00000000000..ae9a821d45c --- /dev/null +++ b/ckan/config/resource_formats.json @@ -0,0 +1,72 @@ +[ + ["_comment", + "JSON field order as follows:", + ["Format", "Description", "Mimetype", ["List of alternative representations"]], + "where:", + " * Format - the short name for it, usually the file extension, because it will be displayed in many places, such as in the search results.", + " * Description - the name, human-friendly, to be displayed on the resource page. ", + " * Mimetype - canonical mimetype for the format. It must be unique to this resource format. It should be listed here: https://www.iana.org/assignments/media-types/media-types.xhtml or here: http://hg.python.org/cpython/file/2.7/Lib/mimetypes.py#l403", + " * List of alternative representations - these are other names that the user might type when they mean this format, or alternative mime-types or any other identifier. (They must be unique to this resource format.)" + ], + ["PPTX", "Powerpoint OOXML Presentation", "application/vnd.openxmlformats-officedocument.presentationml.presentation", []], + ["EXE", "Windows Executable Program", "application/x-msdownload", []], + ["DOC", "Word Document", "application/ms-word", []], + ["KML", "KML File", "application/vnd.google-earth.kml+xml", []], + ["XLS", "Excel Document", "application/vnd.ms-excel", []], + ["WCS", "Web Coverage Service", "wcs", []], + ["JS", "JavaScript", "application/x-javascript", []], + ["MDB", "Access Database", "application/x-msaccess", []], + ["NetCDF", "NetCDF File", "application/netcdf", []], + ["ArcGIS Map Service", "ArcGIS Map Service", "ArcGIS Map Service", ["arcgis map service"]], + ["TSV", "Tab Separated Values File", "text/tsv", []], + ["WFS", "Web Feature Service", null, []], + ["ArcGIS Online Map", "ArcGIS Online Map", "ArcGIS Online Map", ["web map application"]], + ["Perl", "Perl Script", "text/x-perl", []], + ["KMZ", "KMZ File", "application/vnd.google-earth.kmz+xml", ["application/vnd.google-earth.kmz"]], + ["OWL", "Web Ontology Language", "application/owl+xml", []], + ["N3", "N3 Triples", "application/x-n3", []], + ["ZIP", "Zip File", "application/zip", ["zip", "http://purl.org/NET/mediatypes/application/zip"]], + ["GZ", "Gzip File", "application/gzip", ["application/x-gzip"]], + ["QGIS", "QGIS File", "application/x-qgis", []], + ["ODS", "OpenDocument Spreadsheet", "application/vnd.oasis.opendocument.spreadsheet", []], + ["ODT", "OpenDocument Text", "application/vnd.oasis.opendocument.text", []], + ["JSON", "JavaScript Object Notation", "application/json", []], + ["BMP", "Bitmap Image File", "image/x-ms-bmp", []], + ["HTML", "Web Page", "text/html", ["htm", "http://purl.org/net/mediatypes/text/html"]], + ["RAR", "RAR Compressed File", "application/rar", []], + ["TIFF", "TIFF Image File", "image/tiff", []], + ["ODB", "OpenDocument Database", "application/vnd.oasis.opendocument.database", []], + ["TXT", "Text File", "text/plain", []], + ["DCR", "Adobe Shockwave format", "application/x-director", []], + ["ODF", "OpenDocument Math Formula", "application/vnd.oasis.opendocument.formula", []], + ["ODG", "OpenDocument Image", "application/vnd.oasis.opendocument.graphics", []], + ["XML", "XML File", "application/xml", ["text/xml", "http://purl.org/net/mediatypes/application/xml"]], + ["XLSX", "Excel OOXML Spreadsheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", []], + ["DOCX", "Word OOXML Document", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", []], + ["BIN", "Binary Data", "application/octet-stream", ["bin"]], + ["XSLT", "Extensible Stylesheet Language Transformations", "application/xslt+xml", []], + ["WMS", "Web Mapping Service", "WMS", ["wms"]], + ["SVG", "SVG vector image", "image/svg+xml", ["svg"]], + ["PPT", "Powerpoint Presentation", "application/vnd.ms-powerpoint", []], + ["JPEG", "JPG Image File", "image/jpeg", ["jpeg", "jpg"]], + ["SPARQL", "SPARQL end-point", "application/sparql-results+xml", []], + ["GIF", "GIF Image File", "image/gif", []], + ["RDF", "RDF", "application/rdf+xml", ["rdf/xml"]], + ["E00", " ARC/INFO interchange file format", "application/x-e00", []], + ["PDF", "PDF File", "application/pdf", []], + ["CSV", "Comma Separated Values File", "text/csv", ["text/comma-separated-values"]], + ["ODC", "OpenDocument Chart", "application/vnd.oasis.opendocument.chart", []], + ["Atom Feed", "Atom Feed", "application/atom+xml", []], + ["MrSID", "MrSID", "image/x-mrsid", []], + ["ArcGIS Map Preview", "ArcGIS Map Preview", "ArcGIS Map Preview", ["arcgis map preview"]], + ["XYZ", "XYZ Chemical File", "chemical/x-xyz", []], + ["MOP", "MOPAC Input format", "chemical/x-mopac-input", []], + ["Esri REST", "Esri Rest API Endpoint", "Esri REST", ["arcgis_rest"]], + ["dBase", "dBase Database", "application/x-dbf", ["dbf"]], + ["MXD", "ESRI ArcGIS project file", "application/x-mxd", []], + ["TAR", "TAR Compressed File", "application/x-tar", []], + ["PNG", "PNG Image File", "image/png", []], + ["RSS", "RSS feed", "application/rss+xml", []], + ["GeoJSON", "Geographic JavaScript Object Notation", null, []], + ["SHP", "Shapefile", null, ["esri shapefile"]] +] diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index fbaeefdc9da..a8c44751f06 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -1,6 +1,5 @@ import logging from urllib import quote -from urlparse import urlparse from pylons import config @@ -356,7 +355,7 @@ def login(self, error=None): def logged_in(self): # redirect if needed came_from = request.params.get('came_from', '') - if self._sane_came_from(came_from): + if h.url_is_local(came_from): return h.redirect_to(str(came_from)) if c.user: @@ -390,7 +389,7 @@ def logout(self): def logged_out(self): # redirect if needed came_from = request.params.get('came_from', '') - if self._sane_came_from(came_from): + if h.url_is_local(came_from): return h.redirect_to(str(came_from)) h.redirect_to(controller='user', action='logged_out_page') @@ -686,14 +685,3 @@ def unfollow(self, id): or e.error_dict) h.flash_error(error_message) h.redirect_to(controller='user', action='read', id=id) - - def _sane_came_from(self, url): - '''Returns True if came_from is local''' - if not url or (len(url) >= 2 and url.startswith('//')): - return False - parsed = urlparse(url) - if parsed.scheme: - domain = urlparse(h.url_for('/', qualified=True)).netloc - if domain != parsed.netloc: - return False - return True diff --git a/ckan/controllers/util.py b/ckan/controllers/util.py index bb6a40f2dd1..840c6833a04 100644 --- a/ckan/controllers/util.py +++ b/ckan/controllers/util.py @@ -2,6 +2,8 @@ import ckan.lib.base as base import ckan.lib.i18n as i18n +import ckan.lib.helpers as h +from ckan.common import _ class UtilController(base.BaseController): @@ -10,7 +12,13 @@ class UtilController(base.BaseController): def redirect(self): ''' redirect to the url parameter. ''' url = base.request.params.get('url') - return base.redirect(url) + if not url: + base.abort(400, _('Missing Value') + ': url') + + if h.url_is_local(url): + return base.redirect(url) + else: + base.abort(403, _('Redirecting to external site is not allowed.')) def primer(self): ''' Render all html components out onto a single page. diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 324531ee70d..dc7444ce0a2 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -432,7 +432,7 @@ def create(cls, auth_profile="", package_type=None): ) pr2 = model.Resource( url=u'http://www.annakarenina.com/index.json', - format=u'json', + format=u'JSON', description=u'Index of the novel', hash=u'def456', extras={'size_extra': u'345'}, diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index db7b28ba3ad..8913f07e4c9 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -118,30 +118,6 @@ def extras_list_dictize(extras_list, context): return sorted(result_list, key=lambda x: x["key"]) -def _unified_resource_format(format_): - ''' Convert resource formats into a more uniform set. - eg .json, json, JSON, text/json all converted to JSON.''' - - format_clean = format_.lower().split('/')[-1].replace('.', '') - formats = { - 'csv' : 'CSV', - 'zip' : 'ZIP', - 'pdf' : 'PDF', - 'xls' : 'XLS', - 'json' : 'JSON', - 'kml' : 'KML', - 'xml' : 'XML', - 'shape' : 'SHAPE', - 'rdf' : 'RDF', - 'txt' : 'TXT', - 'text' : 'TEXT', - 'html' : 'HTML', - } - if format_clean in formats: - format_new = formats[format_clean] - else: - format_new = format_.lower() - return format_new def resource_dictize(res, context): model = context['model'] @@ -150,7 +126,6 @@ def resource_dictize(res, context): extras = resource.pop("extras", None) if extras: resource.update(extras) - resource['format'] = _unified_resource_format(res.format) # some urls do not have the protocol this adds http:// to these url = resource['url'] ## for_edit is only called at the times when the dataset is to be edited diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py index 0ab9c44706b..086bcf7e8e2 100644 --- a/ckan/lib/dictization/model_save.py +++ b/ckan/lib/dictization/model_save.py @@ -38,12 +38,6 @@ def resource_dict_save(res_dict, context): continue if key == 'url' and not new and obj.url <> value: obj.url_changed = True - # this is an internal field so ignore - # FIXME This helps get the tests to pass but is a hack and should - # be fixed properly. basically don't update the format if not needed - if (key == 'format' and (value == obj.format - or value == d.model_dictize._unified_resource_format(obj.format))): - continue setattr(obj, key, value) else: # resources save extras directly onto the object, instead diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index eb35715aeb8..1c733a2baa1 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -9,9 +9,11 @@ import datetime import logging import re +import os import urllib import pprint import copy +import urlparse from urllib import urlencode from paste.deploy.converters import asbool @@ -227,6 +229,18 @@ def _add_i18n_to_url(url_to_amend, **kw): return url +def url_is_local(url): + '''Returns True if url is local''' + if not url or url.startswith('//'): + return False + parsed = urlparse.urlparse(url) + if parsed.scheme: + domain = urlparse.urlparse(url_for('/', qualified=True)).netloc + if domain != parsed.netloc: + return False + return True + + def full_current_url(): ''' Returns the fully qualified current url (eg http://...) useful for sharing etc ''' @@ -1817,6 +1831,57 @@ def get_site_statistics(): return stats +_RESOURCE_FORMATS = None + +def resource_formats(): + ''' Returns the resource formats as a dict, sourced from the resource format JSON file. + key: potential user input value + value: [canonical mimetype lowercased, canonical format (lowercase), human readable form] + Fuller description of the fields are described in + `ckan/config/resource_formats.json`. + ''' + global _RESOURCE_FORMATS + if not _RESOURCE_FORMATS: + _RESOURCE_FORMATS = {} + format_file_path = config.get('ckan.resource_formats') + if not format_file_path: + format_file_path = os.path.join( + os.path.dirname(os.path.realpath(ckan.config.__file__)), + 'resource_formats.json' + ) + with open(format_file_path) as format_file: + try: + file_resource_formats = json.loads(format_file.read()) + except ValueError, e: # includes simplejson.decoder.JSONDecodeError + raise ValueError('Invalid JSON syntax in %s: %s' % (format_file_path, e)) + + for format_line in file_resource_formats: + if format_line[0] == '_comment': + continue + line = [format_line[2], format_line[0], format_line[1]] + alternatives = format_line[3] if len(format_line) == 4 else [] + for item in line + alternatives: + if item: + item = item.lower() + if item in _RESOURCE_FORMATS \ + and _RESOURCE_FORMATS[item] != line: + raise ValueError('Duplicate resource format ' + 'identifier in %s: %s' % + (format_file_path, item)) + _RESOURCE_FORMATS[item] = line + + return _RESOURCE_FORMATS + + +def unified_resource_format(format): + formats = resource_formats() + format_clean = format.lower() + if format_clean in formats: + format_new = formats[format_clean][1] + else: + format_new = format + return format_new + def check_config_permission(permission): return new_authz.check_config_permission(permission) diff --git a/ckan/lib/munge.py b/ckan/lib/munge.py index a47c8c02258..8cc3c47d2a9 100644 --- a/ckan/lib/munge.py +++ b/ckan/lib/munge.py @@ -107,7 +107,7 @@ def munge_tag(tag): def munge_filename(filename): filename = substitute_ascii_equivalents(filename) - filename = filename.lower().strip() + filename = filename.strip() filename = re.sub(r'[^a-zA-Z0-9. ]', '', filename).replace(' ', '-') filename = _munge_to_length(filename, 3, 100) return filename diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index eda536bcb43..59be5a43b05 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -447,22 +447,6 @@ def wrapped(context=None, data_dict=None, **kw): fn.side_effect_free = True _actions[action_name] = fn - - def replaced_action(action_name): - def warn(context, data_dict): - log.critical('Action `%s` is being called directly ' - 'all action calls should be accessed via ' - 'logic.get_action' % action_name) - return get_action(action_name)(context, data_dict) - return warn - - # Store our wrapped function so it is available. This is to prevent - # rewrapping of actions - module = sys.modules[_action.__module__] - r = replaced_action(action_name) - r.__replaced = fn - module.__dict__[action_name] = r - return _actions.get(action) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 840c51c6e29..ef9459ac922 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -47,6 +47,47 @@ _text = sqlalchemy.text +def _filter_activity_by_user(activity_list, users=[]): + ''' + Return the given ``activity_list`` with activities from the specified + users removed. The users parameters should be a list of ids. + + A *new* filtered list is returned, the given ``activity_list`` itself is + not modified. + ''' + if not len(users): + return activity_list + new_list = [] + for activity in activity_list: + if activity.user_id not in users: + new_list.append(activity) + return new_list + + +def _activity_stream_get_filtered_users(): + ''' + Get the list of users from the :ref:`ckan.hide_activity_from_users` config + option and return a list of their ids. If the config is not specified, + returns the id of the site user. + ''' + users = config.get('ckan.hide_activity_from_users') + if users: + user_list = users.split() + else: + context = {'model': model, 'ignore_auth': True} + site_user = logic.get_action('get_site_user')(context) + users = [site_user.get('name')] + return model.User.user_ids_for_name_or_id(users) + + +def _package_list_with_resources(context, package_revision_list): + package_list = [] + for package in package_revision_list: + result_dict = model_dictize.package_dictize(package,context) + package_list.append(result_dict) + return package_list + + def site_read(context, data_dict=None): '''Return ``True``. @@ -2200,8 +2241,11 @@ def user_activity_list(context, data_dict): limit = int( data_dict.get('limit', config.get('ckan.activity_list_limit', 31))) - activity_objects = model.activity.user_activity_list(user.id, limit=limit, - offset=offset) + _activity_objects = model.activity.user_activity_list(user.id, limit=limit, + offset=offset) + activity_objects = _filter_activity_by_user(_activity_objects, + _activity_stream_get_filtered_users()) + return model_dictize.activity_list_dictize(activity_objects, context) @@ -2239,8 +2283,11 @@ def package_activity_list(context, data_dict): limit = int( data_dict.get('limit', config.get('ckan.activity_list_limit', 31))) - activity_objects = model.activity.package_activity_list( - package.id, limit=limit, offset=offset) + _activity_objects = model.activity.package_activity_list(package.id, + limit=limit, offset=offset) + activity_objects = _filter_activity_by_user(_activity_objects, + _activity_stream_get_filtered_users()) + return model_dictize.activity_list_dictize(activity_objects, context) @@ -2277,8 +2324,11 @@ def group_activity_list(context, data_dict): group_show = logic.get_action('group_show') group_id = group_show(context, {'id': group_id})['id'] - activity_objects = model.activity.group_activity_list( - group_id, limit=limit, offset=offset) + _activity_objects = model.activity.group_activity_list(group_id, + limit=limit, offset=offset) + activity_objects = _filter_activity_by_user(_activity_objects, + _activity_stream_get_filtered_users()) + return model_dictize.activity_list_dictize(activity_objects, context) @@ -2306,8 +2356,11 @@ def organization_activity_list(context, data_dict): org_show = logic.get_action('organization_show') org_id = org_show(context, {'id': org_id})['id'] - activity_objects = model.activity.group_activity_list( - org_id, limit=limit, offset=offset) + _activity_objects = model.activity.group_activity_list(org_id, + limit=limit, offset=offset) + activity_objects = _filter_activity_by_user(_activity_objects, + _activity_stream_get_filtered_users()) + return model_dictize.activity_list_dictize(activity_objects, context) @@ -2333,8 +2386,10 @@ def recently_changed_packages_activity_list(context, data_dict): limit = int( data_dict.get('limit', config.get('ckan.activity_list_limit', 31))) - activity_objects = model.activity.recently_changed_packages_activity_list( - limit=limit, offset=offset) + _activity_objects = model.activity.recently_changed_packages_activity_list( + limit=limit, offset=offset) + activity_objects = _filter_activity_by_user(_activity_objects, + _activity_stream_get_filtered_users()) return model_dictize.activity_list_dictize(activity_objects, context) @@ -2959,9 +3014,11 @@ def dashboard_activity_list(context, data_dict): # FIXME: Filter out activities whose subject or object the user is not # authorized to read. - activity_objects = model.activity.dashboard_activity_list( - user_id, limit=limit, offset=offset) + _activity_objects = model.activity.dashboard_activity_list(user_id, + limit=limit, offset=offset) + activity_objects = _filter_activity_by_user(_activity_objects, + _activity_stream_get_filtered_users()) activity_dicts = model_dictize.activity_list_dictize( activity_objects, context) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 3cdd9608cb7..7780c7b2fc7 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -52,6 +52,8 @@ url_validator, datasets_with_no_organization_cannot_be_private, list_of_strings, + if_empty_guess_format, + clean_format, no_loops_in_hierarchy, ) from ckan.logic.converters import (convert_user_name_or_id_to_id, @@ -72,7 +74,7 @@ def default_resource_schema(): 'package_id': [ignore], 'url': [not_empty, unicode],#, URL(add_http=False)], 'description': [ignore_missing, unicode], - 'format': [ignore_missing, unicode], + 'format': [if_empty_guess_format, ignore_missing, clean_format, unicode], 'hash': [ignore_missing, unicode], 'state': [ignore], 'position': [ignore], @@ -170,6 +172,8 @@ def default_create_package_schema(): def default_update_package_schema(): schema = default_create_package_schema() + schema['resources'] = default_update_resource_schema() + # Users can (optionally) supply the package id when updating a package, but # only to identify the package to be updated, they cannot change the id. schema['id'] = [ignore_missing, package_id_not_changed] @@ -199,6 +203,7 @@ def default_show_package_schema(): # Add several keys to the 'resources' subschema so they don't get stripped # from the resource dicts by validation. schema['resources'].update({ + 'format': [ignore_missing, clean_format, unicode], 'created': [ckan.lib.navl.validators.ignore_missing], 'position': [not_empty], 'last_modified': [ckan.lib.navl.validators.ignore_missing], diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 85cc0dfd8bc..208fb3dd44e 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -1,6 +1,7 @@ import datetime from itertools import count import re +import mimetypes import ckan.lib.navl.dictization_functions as df import ckan.logic as logic @@ -684,7 +685,6 @@ def url_validator(key, data, errors, context): errors[key].append(_('Please provide a valid URL')) - def user_name_exists(user_name, context): model = context['model'] session = context['session'] @@ -736,6 +736,20 @@ def list_of_strings(key, data, errors, context): if not isinstance(x, basestring): raise Invalid('%s: %s' % (_('Not a string'), x)) +def if_empty_guess_format(key, data, errors, context): + value = data[key] + resource_id = data.get(key[:-1] + ('id',)) + + # if resource_id then an update + if (not value or value is Missing) and not resource_id: + url = data.get(key[:-1] + ('url',), '') + mimetype, encoding = mimetypes.guess_type(url) + if mimetype: + data[key] = mimetype + +def clean_format(format): + return h.unified_resource_format(format) + def no_loops_in_hierarchy(key, data, errors, context): '''Checks that the parent groups specified in the data would not cause a loop in the group hierarchy, and therefore cause the recursion up/down diff --git a/ckan/model/resource.py b/ckan/model/resource.py index b983e7cf4e7..e3442ad0dda 100644 --- a/ckan/model/resource.py +++ b/ckan/model/resource.py @@ -119,9 +119,6 @@ def as_dict(self, core_columns_only=False): _dict[k] = v if self.resource_group and not core_columns_only: _dict["package_id"] = self.resource_group.package_id - # FIXME format unification needs doing better - import ckan.lib.dictization.model_dictize as model_dictize - _dict[u'format'] = model_dictize._unified_resource_format(self.format) return _dict def get_package_id(self): diff --git a/ckan/model/user.py b/ckan/model/user.py index da92a57c11a..0d7848b2ef3 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -243,6 +243,18 @@ def search(cls, querystr, sqlalchemy_query=None, user_name=None): query = query.filter(or_(*filters)) return query + @classmethod + def user_ids_for_name_or_id(self, user_list=[]): + ''' + This function returns a list of ids from an input that can be a list of + names or ids + ''' + query = meta.Session.query(self.id) + query = query.filter(or_(self.name.in_(user_list), + self.id.in_(user_list))) + return [user.id for user in query.all()] + + meta.mapper(User, user_table, properties={'password': synonym('_password', map_column=True)}, order_by=user_table.c.name) diff --git a/ckan/new_tests/controllers/test_util.py b/ckan/new_tests/controllers/test_util.py new file mode 100644 index 00000000000..0f64311569c --- /dev/null +++ b/ckan/new_tests/controllers/test_util.py @@ -0,0 +1,48 @@ +from nose.tools import assert_equal +from pylons.test import pylonsapp +import paste.fixture + +from routes import url_for as url_for + + +# This is stolen from the old tests and should probably go in __init__.py +# if it is what we want. +class WsgiAppCase(object): + wsgiapp = pylonsapp + assert wsgiapp, 'You need to run nose with --with-pylons' + # Either that, or this file got imported somehow before the tests started + # running, meaning the pylonsapp wasn't setup yet (which is done in + # pylons.test.py:begin()) + app = paste.fixture.TestApp(wsgiapp) + + +class TestUtil(WsgiAppCase): + def test_redirect_ok(self): + response = self.app.get( + url=url_for(controller='util', action='redirect'), + params={'url': '/dataset'}, + status=302, + ) + assert_equal(response.header_dict.get('Location'), + 'http://localhost/dataset') + + def test_redirect_external(self): + response = self.app.get( + url=url_for(controller='util', action='redirect'), + params={'url': 'http://nastysite.com'}, + status=403, + ) + + def test_redirect_no_params(self): + response = self.app.get( + url=url_for(controller='util', action='redirect'), + params={}, + status=400, + ) + + def test_redirect_no_params_2(self): + response = self.app.get( + url=url_for(controller='util', action='redirect'), + params={'url': ''}, + status=400, + ) diff --git a/ckan/new_tests/logic/test_validators.py b/ckan/new_tests/logic/test_validators.py index dcdc46cb046..42d8013112e 100644 --- a/ckan/new_tests/logic/test_validators.py +++ b/ckan/new_tests/logic/test_validators.py @@ -318,6 +318,60 @@ def call_validator(*args, **kwargs): # TODO: Test user_name_validator()'s behavior when there's a 'user_obj' in # the context dict. + def test_if_empty_guess_format(self): + + import ckan.logic.validators as validators + import ckan.lib.navl.dictization_functions as dictization_functions + + data = {'name': 'package_name', 'resources': [ + {'url': 'http://fakedomain/my.csv', 'format': ''}, + {'url': 'http://fakedomain/my.pdf', + 'format': dictization_functions.Missing}, + {'url': 'http://fakedomain/my.pdf', 'format': 'pdf'}, + {'url': 'http://fakedomain/my.pdf', + 'id': 'fake_resource_id', 'format': ''} + ]} + data = dictization_functions.flatten_dict(data) + + @t.does_not_modify_errors_dict + def call_validator(*args, **kwargs): + return validators.if_empty_guess_format(*args, **kwargs) + + new_data = copy.deepcopy(data) + call_validator(key=('resources', 0, 'format'), data=new_data, + errors={}, context={}) + assert new_data[('resources', 0, 'format')] == 'text/csv' + + new_data = copy.deepcopy(data) + call_validator(key=('resources', 1, 'format'), data=new_data, + errors={}, context={}) + assert new_data[('resources', 1, 'format')] == 'application/pdf' + + new_data = copy.deepcopy(data) + call_validator(key=('resources', 2, 'format'), data=new_data, + errors={}, context={}) + assert new_data[('resources', 2, 'format')] == 'pdf' + + new_data = copy.deepcopy(data) + call_validator(key=('resources', 3, 'format'), data=new_data, + errors={}, context={}) + assert new_data[('resources', 3, 'format')] == '' + + def test_clean_format(self): + import ckan.logic.validators as validators + + format = validators.clean_format('csv') + assert format == 'CSV' + + format = validators.clean_format('text/csv') + assert format == 'CSV' + + format = validators.clean_format('not a format') + assert format == 'not a format' + + format = validators.clean_format('') + assert format == '' + def test_datasets_with_org_can_be_private_when_creating(self): import ckan.logic.validators as validators diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html index 522524350a7..01024d6ed35 100644 --- a/ckan/templates/macros/form.html +++ b/ckan/templates/macros/form.html @@ -118,11 +118,12 @@ {% macro markdown(name, id='', label='', value='', placeholder='', error="", classes=[], attrs={}, is_required=false) %} {% set classes = (classes|list) %} {% do classes.append('control-full') %} + {% set markdown_tooltip = "
__Bold text__ or _italic text_
# title
## secondary title
### etc
* list
* of
* items
http://auto.link.ed/

Full markdown syntax

Please note: HTML tags are stripped out for security reasons

" %} {%- set extra_html = caller() if caller -%} {% call input_block(id or name, label or name, error, classes, control_classes=["editor"], extra_html=extra_html, is_required=is_required) %} - {% trans %}You can use Markdown formatting here{% endtrans %} + {% trans %}You can use Markdown formatting here{% endtrans %} {% endcall %} {% endmacro %} diff --git a/ckan/templates/package/search.html b/ckan/templates/package/search.html index b1aeec9ab7b..27105f09e01 100644 --- a/ckan/templates/package/search.html +++ b/ckan/templates/package/search.html @@ -50,7 +50,7 @@ {% block package_search_results_api_inner %} {% set api_link = h.link_to(_('API'), h.url_for(controller='api', action='get_api', ver=3)) %} - {% set api_doc_link = h.link_to(_('API Docs'), 'http://docs.ckan.org/en/{0}/api.html'.format(g.ckan_doc_version)) %} + {% set api_doc_link = h.link_to(_('API Docs'), 'http://docs.ckan.org/en/{0}/api/'.format(g.ckan_doc_version)) %} {% if g.dumps_url -%} {% set dump_link = h.link_to(_('full {format} dump').format(format=g.dumps_format), g.dumps_url) %} {% trans %} diff --git a/ckan/templates/package/snippets/resource_form.html b/ckan/templates/package/snippets/resource_form.html index bf8434a9f27..e1e7e52f04d 100644 --- a/ckan/templates/package/snippets/resource_form.html +++ b/ckan/templates/package/snippets/resource_form.html @@ -35,6 +35,10 @@ {% block basic_fields_format %} {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/resource/format_autocomplete?incomplete=?'} %} {% call form.input('format', id='field-format', label=_('Format'), placeholder=_('eg. CSV, XML or JSON'), value=data.format, error=errors.format, classes=['control-medium'], attrs=format_attrs) %} + + + {{ _('This will be guessed automatically. Leave blank if you wish') }} + {% endcall %} {% endblock %} diff --git a/ckan/tests/functional/api/base.py b/ckan/tests/functional/api/base.py index d90768e2948..0c391d808f0 100644 --- a/ckan/tests/functional/api/base.py +++ b/ckan/tests/functional/api/base.py @@ -326,14 +326,14 @@ class BaseModelApiTestCase(ApiTestCase, ControllerTestCase): 'url': u'http://blahblahblah.mydomain', 'resources': [{ u'url':u'http://blah.com/file.xml', - u'format':u'xml', + u'format':u'XML', u'description':u'Main file', u'hash':u'abc123', u'alt_url':u'alt_url', u'size_extra':u'200', }, { u'url':u'http://blah.com/file2.xml', - u'format':u'xml', + u'format':u'XML', u'description':u'Second file', u'hash':u'def123', u'alt_url':u'alt_url', diff --git a/ckan/tests/functional/api/model/test_package.py b/ckan/tests/functional/api/model/test_package.py index c19cdad4cef..3354cc71089 100644 --- a/ckan/tests/functional/api/model/test_package.py +++ b/ckan/tests/functional/api/model/test_package.py @@ -399,14 +399,14 @@ def assert_package_update_ok(self, package_ref_attribute, 'title':u'newtesttitle', 'resources': [{ u'url':u'http://blah.com/file2.xml', - u'format':u'xml', + u'format':u'XML', u'description':u'Appendix 1', u'hash':u'def123', u'alt_url':u'alt123', u'size_extra':u'400', },{ u'url':u'http://blah.com/file3.xml', - u'format':u'xml', + u'format':u'XML', u'description':u'Appenddic 2', u'hash':u'ghi123', u'alt_url':u'alt123', @@ -459,14 +459,14 @@ def assert_package_update_ok(self, package_ref_attribute, self.assert_equal(len(package.resources), 2) resource = package.resources[0] self.assert_equal(resource.url, u'http://blah.com/file2.xml') - self.assert_equal(resource.format, u'xml') + self.assert_equal(resource.format, u'XML') self.assert_equal(resource.description, u'Appendix 1') self.assert_equal(resource.hash, u'def123') self.assert_equal(resource.alt_url, u'alt123') self.assert_equal(resource.extras['size_extra'], u'400') resource = package.resources[1] self.assert_equal(resource.url, 'http://blah.com/file3.xml') - self.assert_equal(resource.format, u'xml') + self.assert_equal(resource.format, u'XML') self.assert_equal(resource.description, u'Appenddic 2') self.assert_equal(resource.hash, u'ghi123') self.assert_equal(resource.alt_url, u'alt123') @@ -672,13 +672,13 @@ def test_package_update_delete_resource(self): 'name': self.package_fixture_data['name'], 'resources': [{ u'url':u'http://blah.com/file2.xml', - u'format':u'xml', + u'format':u'XML', u'description':u'Appendix 1', u'hash':u'def123', u'alt_url':u'alt123', },{ u'url':u'http://blah.com/file3.xml', - u'format':u'xml', + u'format':u'XML', u'description':u'Appenddic 2', u'hash':u'ghi123', u'alt_url':u'alt123', diff --git a/ckan/tests/lib/test_resource_search.py b/ckan/tests/lib/test_resource_search.py index 68e6fd5724d..d9d3e422751 100644 --- a/ckan/tests/lib/test_resource_search.py +++ b/ckan/tests/lib/test_resource_search.py @@ -128,9 +128,7 @@ def test_12_search_all_fields(self): assert res_dict['package_id'] == pkg1.id assert res_dict['url'] == ab.url assert res_dict['description'] == ab.description - # FIXME: This needs to be fixed before this branch is merged to master - from ckan.lib.dictization.model_dictize import _unified_resource_format - assert res_dict['format'] == _unified_resource_format(ab.format) + assert res_dict['format'] == ab.format assert res_dict['hash'] == ab.hash assert res_dict['position'] == 0 diff --git a/doc/images/manage_users.jpg b/doc/images/manage_users.jpg index 0be4a650dc2..047037c2546 100644 Binary files a/doc/images/manage_users.jpg and b/doc/images/manage_users.jpg differ diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index c2dc8593a63..d096af28aa2 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -1149,6 +1149,20 @@ Default value: ``infinite`` Email notifications for events older than this time delta will not be sent. Accepted formats: '2 days', '14 days', '4:35:00' (hours, minutes, seconds), '7 days, 3:23:34', etc. +.. _ckan.hide_activity_from_users: + +ckan.hide_activity_from_users +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.hide_activity_from_users = sysadmin + +Hides activity from the specified users from activity stream. If unspecified, +it'll use :ref:`ckan.site_id` to hide activity by the site user. The site user +is a sysadmin user on every ckan user with a username that's equal to +:ref:`ckan.site_id`. This user is used by ckan for performing actions from the +command-line. .. _config-feeds: @@ -1296,10 +1310,26 @@ locale, or ``/de/some/url`` when using the "de" locale, for example. This lets you change this. You can use any path that you want, adding ``{{LANG}}`` where you want the locale code to go. +.. _ckan.resource_formats: + +ckan.resource_formats +^^^^^^^^^^^^^^^^^^^^^ + +Example:: + ckan.resource_formats = /path/to/resource_formats + +Default value: ckan/config/resource_formats.json +The purpose of this file is to supply a thorough list of resource formats +and to make sure the formats are normalized when saved to the database +and presented. +The format of the file is a JSON object with following format:: + ["Format", "Description", "Mimetype", ["List of alternative representations"]] +Please look in ckan/config/resource_formats.json for full details and and as an +example. Form Settings diff --git a/doc/maintaining/paster.rst b/doc/maintaining/paster.rst index 8074e1a20ff..833ea812fd9 100644 --- a/doc/maintaining/paster.rst +++ b/doc/maintaining/paster.rst @@ -74,6 +74,15 @@ with the ``--help`` option:: Troubleshooting Paster Commands ------------------------------- +Permission Error +================ + +If you receive 'Permission Denied' error, try running paster with sudo. + +.. parsed-literal:: + + sudo |virtualenv|/bin/paster db clean -c |production.ini| + Virtualenv not activated, or not in ckan dir ============================================ @@ -471,7 +480,7 @@ won't clear the index before starting rebuilding it:: paster --plugin=ckan search-index rebuild -r --config=/etc/ckan/std/std.ini -There is also an option available which works like the refresh option but tries to use all processes on the +There is also an option available which works like the refresh option but tries to use all processes on the computer to reindex faster:: paster --plugin=ckan search-index rebuild_fast --config=/etc/ckan/std/std.ini diff --git a/doc/sysadmin-guide.rst b/doc/sysadmin-guide.rst index ca43aeba654..b974df0b0a8 100644 --- a/doc/sysadmin-guide.rst +++ b/doc/sysadmin-guide.rst @@ -140,12 +140,12 @@ This is useful if, for example, a user has forgotten their user ID. For non-sysadmin users, the search on this page will only match public parts of the profile, so they cannot search by e-mail address. -On their user profile, you will see an "Edit" button. CKAN displays the user -settings page. You can change any settings for the user, including their -username, name and password. +On their user profile, you will see a "Manage" button. CKAN displays the user +settings page. You can delete the user or change any of its settings, including +their username, name and password. .. image:: /images/manage_users.jpg -.. note:: - - At present, it is not possible to delete users. +.. versionadded:: 2.2 + Previous versions of CKAN didn't allow you to delete users through the + web interface.