Merge branch 'master' into 1251-resource-view

MANIFEST.in

@@ -1,6 +1,7 @@
 include ckan/config/deployment.ini_tmpl
 recursive-include ckan/public *
 recursive-include ckan/config *.ini
+recursive-include ckan/config *.json
 recursive-include ckan/config *.xml
 recursive-include ckan/i18n *
 recursive-include ckan/templates *

ckan/config/deployment.ini_tmpl

@@ -136,7 +136,8 @@ ckan.feeds.author_link =
 #ckan.activity_streams_enabled = true
 #ckan.activity_list_limit = 31
 #ckan.activity_streams_email_notifications = true
-# ckan.email_notifications_since = 2 days
+#ckan.email_notifications_since = 2 days
+ckan.hide_activity_from_users = %(ckan.site_id)s
 
 
 ## Email settings

ckan/config/resource_formats.json

@@ -0,0 +1,72 @@
+[
+  ["_comment",
+   "JSON field order as follows:",
+   ["Format", "Description", "Mimetype", ["List of alternative representations"]],
+   "where:",
+   " * Format - the short name for it, usually the file extension, because it will be displayed in many places, such as in the search results.",
+   " * Description - the name, human-friendly, to be displayed on the resource page. ",
+   " * Mimetype - canonical mimetype for the format. It must be unique to this resource format. It should be listed here: https://www.iana.org/assignments/media-types/media-types.xhtml or here: http://hg.python.org/cpython/file/2.7/Lib/mimetypes.py#l403",
+   " * List of alternative representations - these are other names that the user might type when they mean this format, or alternative mime-types or any other identifier. (They must be unique to this resource format.)"
+  ],
+  ["PPTX", "Powerpoint OOXML Presentation", "application/vnd.openxmlformats-officedocument.presentationml.presentation", []],
+  ["EXE", "Windows Executable Program", "application/x-msdownload", []],
+  ["DOC", "Word Document", "application/ms-word", []],
+  ["KML", "KML File", "application/vnd.google-earth.kml+xml", []],
+  ["XLS", "Excel Document", "application/vnd.ms-excel", []],
+  ["WCS", "Web Coverage Service", "wcs", []],
+  ["JS", "JavaScript", "application/x-javascript", []],
+  ["MDB", "Access Database", "application/x-msaccess", []],
+  ["NetCDF", "NetCDF File", "application/netcdf", []],
+  ["ArcGIS Map Service", "ArcGIS Map Service", "ArcGIS Map Service", ["arcgis map service"]],
+  ["TSV", "Tab Separated Values File", "text/tsv", []],
+  ["WFS", "Web Feature Service", null, []],
+  ["ArcGIS Online Map", "ArcGIS Online Map", "ArcGIS Online Map", ["web map application"]],
+  ["Perl", "Perl Script", "text/x-perl", []],
+  ["KMZ", "KMZ File", "application/vnd.google-earth.kmz+xml", ["application/vnd.google-earth.kmz"]],
+  ["OWL", "Web Ontology Language", "application/owl+xml", []],
+  ["N3", "N3 Triples", "application/x-n3", []],
+  ["ZIP", "Zip File", "application/zip", ["zip", "http://purl.org/NET/mediatypes/application/zip"]],
+  ["GZ", "Gzip File", "application/gzip", ["application/x-gzip"]],
+  ["QGIS", "QGIS File", "application/x-qgis", []],
+  ["ODS", "OpenDocument Spreadsheet", "application/vnd.oasis.opendocument.spreadsheet", []],
+  ["ODT", "OpenDocument Text", "application/vnd.oasis.opendocument.text", []],
+  ["JSON", "JavaScript Object Notation", "application/json", []],
+  ["BMP", "Bitmap Image File", "image/x-ms-bmp", []],
+  ["HTML", "Web Page", "text/html", ["htm", "http://purl.org/net/mediatypes/text/html"]],
+  ["RAR", "RAR Compressed File", "application/rar", []],
+  ["TIFF", "TIFF Image File", "image/tiff", []],
+  ["ODB", "OpenDocument Database", "application/vnd.oasis.opendocument.database", []],
+  ["TXT", "Text File", "text/plain", []],
+  ["DCR", "Adobe Shockwave format", "application/x-director", []],
+  ["ODF", "OpenDocument Math Formula", "application/vnd.oasis.opendocument.formula", []],
+  ["ODG", "OpenDocument Image", "application/vnd.oasis.opendocument.graphics", []],
+  ["XML", "XML File", "application/xml", ["text/xml", "http://purl.org/net/mediatypes/application/xml"]],
+  ["XLSX", "Excel OOXML Spreadsheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", []],
+  ["DOCX", "Word OOXML Document", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", []],
+  ["BIN", "Binary Data", "application/octet-stream", ["bin"]],
+  ["XSLT", "Extensible Stylesheet Language Transformations", "application/xslt+xml", []],
+  ["WMS", "Web Mapping Service", "WMS", ["wms"]],
+  ["SVG", "SVG vector image", "image/svg+xml", ["svg"]],
+  ["PPT", "Powerpoint Presentation", "application/vnd.ms-powerpoint", []],
+  ["JPEG", "JPG Image File", "image/jpeg", ["jpeg", "jpg"]],
+  ["SPARQL", "SPARQL end-point", "application/sparql-results+xml", []],
+  ["GIF", "GIF Image File", "image/gif", []],
+  ["RDF", "RDF", "application/rdf+xml", ["rdf/xml"]],
+  ["E00", " ARC/INFO interchange file format", "application/x-e00", []],
+  ["PDF", "PDF File", "application/pdf", []],
+  ["CSV", "Comma Separated Values File", "text/csv", ["text/comma-separated-values"]],
+  ["ODC", "OpenDocument Chart", "application/vnd.oasis.opendocument.chart", []],
+  ["Atom Feed", "Atom Feed", "application/atom+xml", []],
+  ["MrSID", "MrSID", "image/x-mrsid", []],
+  ["ArcGIS Map Preview", "ArcGIS Map Preview", "ArcGIS Map Preview", ["arcgis map preview"]],
+  ["XYZ", "XYZ Chemical File", "chemical/x-xyz", []],
+  ["MOP", "MOPAC Input format", "chemical/x-mopac-input", []],
+  ["Esri REST", "Esri Rest API Endpoint", "Esri REST", ["arcgis_rest"]],
+  ["dBase", "dBase Database", "application/x-dbf", ["dbf"]],
+  ["MXD", "ESRI ArcGIS project file", "application/x-mxd", []],
+  ["TAR", "TAR Compressed File", "application/x-tar", []],
+  ["PNG", "PNG Image File", "image/png", []],
+  ["RSS", "RSS feed", "application/rss+xml", []],
+  ["GeoJSON", "Geographic JavaScript Object Notation", null, []],
+  ["SHP", "Shapefile", null, ["esri shapefile"]]
+]

ckan/controllers/user.py

@@ -1,6 +1,5 @@
 import logging
 from urllib import quote
-from urlparse import urlparse
 
 from pylons import config
 
@@ -356,7 +355,7 @@ def login(self, error=None):
     def logged_in(self):
         # redirect if needed
         came_from = request.params.get('came_from', '')
-        if self._sane_came_from(came_from):
+        if h.url_is_local(came_from):
             return h.redirect_to(str(came_from))
 
         if c.user:
@@ -390,7 +389,7 @@ def logout(self):
     def logged_out(self):
         # redirect if needed
         came_from = request.params.get('came_from', '')
-        if self._sane_came_from(came_from):
+        if h.url_is_local(came_from):
             return h.redirect_to(str(came_from))
         h.redirect_to(controller='user', action='logged_out_page')
 
@@ -686,14 +685,3 @@ def unfollow(self, id):
                              or e.error_dict)
             h.flash_error(error_message)
         h.redirect_to(controller='user', action='read', id=id)
-
-    def _sane_came_from(self, url):
-        '''Returns True if came_from is local'''
-        if not url or (len(url) >= 2 and url.startswith('//')):
-            return False
-        parsed = urlparse(url)
-        if parsed.scheme:
-            domain = urlparse(h.url_for('/', qualified=True)).netloc
-            if domain != parsed.netloc:
-                return False
-        return True

ckan/controllers/util.py

@@ -2,6 +2,8 @@
 
 import ckan.lib.base as base
 import ckan.lib.i18n as i18n
+import ckan.lib.helpers as h
+from ckan.common import _
 
 
 class UtilController(base.BaseController):
@@ -10,7 +12,13 @@ class UtilController(base.BaseController):
     def redirect(self):
         ''' redirect to the url parameter. '''
         url = base.request.params.get('url')
-        return base.redirect(url)
+        if not url:
+            base.abort(400, _('Missing Value') + ': url')
+
+        if h.url_is_local(url):
+            return base.redirect(url)
+        else:
+            base.abort(403, _('Redirecting to external site is not allowed.'))
 
     def primer(self):
         ''' Render all html components out onto a single page.

ckan/lib/create_test_data.py

@@ -432,7 +432,7 @@ def create(cls, auth_profile="", package_type=None):
             )
         pr2 = model.Resource(
             url=u'http://www.annakarenina.com/index.json',
-            format=u'json',
+            format=u'JSON',
             description=u'Index of the novel',
             hash=u'def456',
             extras={'size_extra': u'345'},

ckan/lib/dictization/model_dictize.py

@@ -118,30 +118,6 @@ def extras_list_dictize(extras_list, context):
 
     return sorted(result_list, key=lambda x: x["key"])
 
-def _unified_resource_format(format_):
-    ''' Convert resource formats into a more uniform set.
-    eg .json, json, JSON, text/json all converted to JSON.'''
-
-    format_clean = format_.lower().split('/')[-1].replace('.', '')
-    formats = {
-        'csv' : 'CSV',
-        'zip' : 'ZIP',
-        'pdf' : 'PDF',
-        'xls' : 'XLS',
-        'json' : 'JSON',
-        'kml' : 'KML',
-        'xml' : 'XML',
-        'shape' : 'SHAPE',
-        'rdf' : 'RDF',
-        'txt' : 'TXT',
-        'text' : 'TEXT',
-        'html' : 'HTML',
-    }
-    if format_clean in formats:
-        format_new = formats[format_clean]
-    else:
-        format_new = format_.lower()
-    return format_new
 
 def resource_dictize(res, context):
     model = context['model']
@@ -150,7 +126,6 @@ def resource_dictize(res, context):
     extras = resource.pop("extras", None)
     if extras:
         resource.update(extras)
-    resource['format'] = _unified_resource_format(res.format)
     # some urls do not have the protocol this adds http:// to these
     url = resource['url']
     ## for_edit is only called at the times when the dataset is to be edited

ckan/lib/dictization/model_save.py

@@ -38,12 +38,6 @@ def resource_dict_save(res_dict, context):
                     continue
             if key == 'url' and not new and obj.url <> value:
                 obj.url_changed = True
-            # this is an internal field so ignore
-            # FIXME This helps get the tests to pass but is a hack and should
-            # be fixed properly. basically don't update the format if not needed
-            if (key == 'format' and (value == obj.format
-                    or value == d.model_dictize._unified_resource_format(obj.format))):
-                continue
             setattr(obj, key, value)
         else:
             # resources save extras directly onto the object, instead

ckan/lib/helpers.py

@@ -9,9 +9,11 @@
 import datetime
 import logging
 import re
+import os
 import urllib
 import pprint
 import copy
+import urlparse
 from urllib import urlencode
 
 from paste.deploy.converters import asbool
@@ -227,6 +229,18 @@ def _add_i18n_to_url(url_to_amend, **kw):
     return url
 
 
+def url_is_local(url):
+    '''Returns True if url is local'''
+    if not url or url.startswith('//'):
+        return False
+    parsed = urlparse.urlparse(url)
+    if parsed.scheme:
+        domain = urlparse.urlparse(url_for('/', qualified=True)).netloc
+        if domain != parsed.netloc:
+            return False
+    return True
+
+
 def full_current_url():
     ''' Returns the fully qualified current url (eg http://...) useful
     for sharing etc '''
@@ -1817,6 +1831,57 @@ def get_site_statistics():
 
     return stats
 
+_RESOURCE_FORMATS = None
+
+def resource_formats():
+    ''' Returns the resource formats as a dict, sourced from the resource format JSON file.
+    key:  potential user input value
+    value:  [canonical mimetype lowercased, canonical format (lowercase), human readable form]
+    Fuller description of the fields are described in
+    `ckan/config/resource_formats.json`.
+    '''
+    global _RESOURCE_FORMATS
+    if not _RESOURCE_FORMATS:
+        _RESOURCE_FORMATS = {}
+        format_file_path = config.get('ckan.resource_formats')
+        if not format_file_path:
+            format_file_path = os.path.join(
+                os.path.dirname(os.path.realpath(ckan.config.__file__)),
+                'resource_formats.json'
+            )
+        with open(format_file_path) as format_file:
+            try:
+                file_resource_formats = json.loads(format_file.read())
+            except ValueError, e:  # includes simplejson.decoder.JSONDecodeError
+                raise ValueError('Invalid JSON syntax in %s: %s' % (format_file_path, e))
+
+            for format_line in file_resource_formats:
+                if format_line[0] == '_comment':
+                    continue
+                line = [format_line[2], format_line[0], format_line[1]]
+                alternatives = format_line[3] if len(format_line) == 4 else []
+                for item in line + alternatives:
+                    if item:
+                        item = item.lower()
+                        if item in _RESOURCE_FORMATS \
+                                and _RESOURCE_FORMATS[item] != line:
+                            raise ValueError('Duplicate resource format '
+                                             'identifier in %s: %s' %
+                                             (format_file_path, item))
+                        _RESOURCE_FORMATS[item] = line
+
+    return _RESOURCE_FORMATS
+
+
+def unified_resource_format(format):
+    formats = resource_formats()
+    format_clean = format.lower()
+    if format_clean in formats:
+        format_new = formats[format_clean][1]
+    else:
+        format_new = format
+    return format_new
+
 def check_config_permission(permission):
     return new_authz.check_config_permission(permission)
 

ckan/lib/munge.py

@@ -107,7 +107,7 @@ def munge_tag(tag):
 
 def munge_filename(filename):
     filename = substitute_ascii_equivalents(filename)
-    filename = filename.lower().strip()
+    filename = filename.strip()
     filename = re.sub(r'[^a-zA-Z0-9. ]', '', filename).replace(' ', '-')
     filename = _munge_to_length(filename, 3, 100)
     return filename

ckan/logic/__init__.py

@@ -447,22 +447,6 @@ def wrapped(context=None, data_dict=None, **kw):
             fn.side_effect_free = True
         _actions[action_name] = fn
 
-
-        def replaced_action(action_name):
-            def warn(context, data_dict):
-                log.critical('Action `%s` is being called directly '
-                             'all action calls should be accessed via '
-                             'logic.get_action' % action_name)
-                return get_action(action_name)(context, data_dict)
-            return warn
-
-        # Store our wrapped function so it is available.  This is to prevent
-        # rewrapping of actions
-        module = sys.modules[_action.__module__]
-        r = replaced_action(action_name)
-        r.__replaced = fn
-        module.__dict__[action_name] = r
-
     return _actions.get(action)