Merge tag 'ckan-2.6.4' into canada-v2.6

[release]: Release tag

CHANGELOG.rst

@@ -7,6 +7,13 @@
 Changelog
 ---------
 
+v2.6.4 2017-09-27
+=================
+
+* Mail recepient header override (#3781)
+* Skip url parsing in redirect (#3499)
+* Support non root for fanstatic (#3618)
+
 v2.6.3 2017-08-02
 =================
 

ckan/__init__.py

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-__version__ = '2.6.3'
+__version__ = '2.6.4'
 
 __description__ = 'CKAN Software'
 __long_description__ = \

ckan/config/middleware/pylons_app.py

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 import os
+import re
 
 from pylons.wsgiapp import PylonsApp
 
@@ -87,6 +88,10 @@ def make_pylons_stack(conf, full_stack=True, static_files=True,
             'bottom': True,
             'bundle': True,
         }
+    root_path = config.get('ckan.root_path', None)
+    if root_path:
+        root_path = re.sub('/{{LANG}}', '', root_path)
+        fanstatic_config['base_url'] = root_path
     app = Fanstatic(app, **fanstatic_config)
 
     for plugin in PluginImplementations(IMiddleware):

ckan/controllers/package.py

@@ -1071,7 +1071,7 @@ def resource_read(self, id, resource_id):
         except KeyError:
             c.package['isopen'] = False
 
-        # TODO: find a nicer way of doing this
+        # Deprecated: c.datastore_api - use h.action_url instead
         c.datastore_api = '%s/api/action' % \
             config.get('ckan.site_url', '').rstrip('/')
 

ckan/controllers/user.py

@@ -251,7 +251,7 @@ def _save_new(self, context):
         if not c.user:
             # log the user in programatically
             set_repoze_user(data_dict['name'])
-            h.redirect_to(controller='user', action='me', __ckan_no_root=True)
+            h.redirect_to(controller='user', action='me')
         else:
             # #1799 User has managed to register whilst logged in - warn user
             # they are not re-logged in as new user.
@@ -402,8 +402,7 @@ def login(self, error=None):
         if not c.user:
             came_from = request.params.get('came_from')
             if not came_from:
-                came_from = h.url_for(controller='user', action='logged_in',
-                                      __ckan_no_root=True)
+                came_from = h.url_for(controller='user', action='logged_in')
             c.login_handler = h.url_for(
                 self._get_repoze_handler('login_handler_path'),
                 came_from=came_from)
@@ -441,10 +440,9 @@ def logout(self):
         # Do any plugin logout stuff
         for item in p.PluginImplementations(p.IAuthenticator):
             item.logout()
-        url = h.url_for(controller='user', action='logged_out_page',
-                        __ckan_no_root=True)
+        url = h.url_for(controller='user', action='logged_out_page')
         h.redirect_to(self._get_repoze_handler('logout_handler_path') +
-                      '?came_from=' + url)
+                      '?came_from=' + url, parse_url=True)
 
     def logged_out(self):
         # redirect if needed

ckan/lib/helpers.py

@@ -17,6 +17,7 @@
 import copy
 import urlparse
 from urllib import urlencode
+import uuid
 
 from paste.deploy import converters
 from webhelpers.html import HTML, literal, tags
@@ -27,7 +28,7 @@
 from bleach import clean as clean_html, ALLOWED_TAGS, ALLOWED_ATTRIBUTES
 from pylons import url as _pylons_default_url
 from ckan.common import config
-from routes import redirect_to as _redirect_to
+from routes import redirect_to as _routes_redirect_to
 from routes import url_for as _routes_default_url_for
 import i18n
 
@@ -148,10 +149,36 @@ def redirect_to(*args, **kw):
 
         toolkit.redirect_to('dataset_read', id='changed')
 
+    If given a single string as argument, this redirects without url parsing
+
+        toolkit.redirect_to('http://example.com')
+        toolkit.redirect_to('/dataset')
+        toolkit.redirect_to('/some/other/path')
+
     '''
     if are_there_flash_messages():
         kw['__no_cache__'] = True
-    return _redirect_to(url_for(*args, **kw))
+
+    # Routes router doesn't like unicode args
+    uargs = map(lambda arg: str(arg) if isinstance(arg, unicode) else arg,
+                args)
+
+    _url = ''
+    skip_url_parsing = False
+    parse_url = kw.pop('parse_url', False)
+    if uargs and len(uargs) is 1 and isinstance(uargs[0], basestring) \
+            and (uargs[0].startswith('/') or is_url(uargs[0])) \
+            and parse_url is False:
+        skip_url_parsing = True
+        _url = uargs[0]
+
+    if skip_url_parsing is False:
+        _url = url_for(*uargs, **kw)
+
+    if _url.startswith('/'):
+        _url = str(config['ckan.site_url'].rstrip('/') + _url)
+
+    return _routes_redirect_to(_url)
 
 
 @maintain.deprecated('h.url is deprecated please use h.url_for')
@@ -503,6 +530,7 @@ def pop_messages(self):
     def are_there_messages(self):
         return bool(session.get(self.session_key))
 
+
 flash = _Flash()
 # this is here for backwards compatability
 _flash = flash
@@ -1062,6 +1090,7 @@ def linked_gravatar(email_hash, size=100, default=None):
         '%s</a>' % gravatar(email_hash, size, default)
     )
 
+
 _VALID_GRAVATAR_DEFAULTS = ['404', 'mm', 'identicon', 'monsterid',
                             'wavatar', 'retro']
 
@@ -1492,6 +1521,7 @@ def process_names(items):
         items.append(item)
     return items
 
+
 # these are the types of objects that can be followed
 _follow_objects = ['dataset', 'user', 'group']
 
@@ -1830,6 +1860,7 @@ def get_request_param(parameter_name, default=None):
     searches. '''
     return request.params.get(parameter_name, default)
 
+
 # find all inner text of html eg `<b>moo</b>` gets `moo` but not of <a> tags
 # as this would lead to linkifying links if they are urls.
 RE_MD_GET_INNER_HTML = re.compile(
@@ -2139,6 +2170,7 @@ def SI_number_span(number):
                          + '">')
     return output + formatters.localised_SI_number(number) + literal('</span>')
 
+
 # add some formatter functions
 localised_number = formatters.localised_number
 localised_SI_number = formatters.localised_SI_number
@@ -2239,6 +2271,7 @@ def get_site_statistics():
         logic.get_action('organization_list')({}, {}))
     return stats
 
+
 _RESOURCE_FORMATS = None
 
 
@@ -2358,6 +2391,14 @@ def radio(selected, id, checked):
         value="%s" type="radio">') % (selected, id, selected, id))
 
 
+@core_helper
+def sanitize_id(id_):
+    '''Given an id (uuid4), if it has any invalid characters it raises
+    ValueError.
+    '''
+    return str(uuid.UUID(id_))
+
+
 core_helper(flash, name='flash')
 core_helper(localised_number)
 core_helper(localised_SI_number)

ckan/lib/mailer.py

@@ -8,7 +8,6 @@
 from email.mime.text import MIMEText
 from email.header import Header
 from email import Utils
-from urlparse import urljoin
 
 from ckan.common import config
 import paste.deploy.converters
@@ -33,7 +32,10 @@ def _mail_recipient(recipient_name, recipient_email,
     mail_from = config.get('smtp.mail_from')
     msg = MIMEText(body.encode('utf-8'), 'plain', 'utf-8')
     for k, v in headers.items():
-        msg[k] = v
+        if k in msg.keys():
+            msg.replace_header(k, v)
+        else:
+            msg.add_header(k, v)
     subject = Header(subject.encode('utf-8'), 'utf-8')
     msg['Subject'] = subject
     msg['From'] = _("%s <%s>") % (sender_name, mail_from)

ckan/templates/ajax_snippets/api_info.html

@@ -1,18 +1,18 @@
 {#
 Displays information about accessing a resource via the API.
 
-datastore_root_url - The root API url.
 resource_id - The resource id
 embedded - If true will not include the "modal" classes on the snippet.
 
 Example
 
-    {% snippet 'ajax_snippets/api_info.html', datastore_root_url=datastore_root_url, resource_id=resource_id, embedded=true %}
+    {% snippet 'ajax_snippets/api_info.html', resource_id=resource_id, embedded=true %}
 
 #}
 
-{% set sql_example_url = datastore_root_url + '/datastore_search_sql?sql=SELECT * from "' + resource_id + '" WHERE title LIKE \'jones\'' %}
-
+{% set resource_id = h.sanitize_id(resource_id) %}
+{% set sql_example_url = h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search_sql', qualified=True) + '?sql=SELECT * from "' + resource_id + '" WHERE title LIKE \'jones\'' %}
+{# not urlencoding the sql because its clearer #}
 <div{% if not embedded %} class="modal"{% endif %}>
   <div class="modal-header">
     <h3>
@@ -40,19 +40,19 @@ <h3>
             <tbody>
               <tr>
                 <th scope="row">{{ _('Create') }}</th>
-                <td><code>{{ datastore_root_url }}/datastore_create</code></td>
+                <td><code>{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_create', qualified=True) }}</code></td>
               </tr>
               <tr>
                 <th scope="row">{{ _('Update / Insert') }}</th>
-                <td><code>{{ datastore_root_url }}/datastore_upsert</code></td>
+                <td><code>{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_upsert', qualified=True) }}</code></td>
               </tr>
               <tr>
                 <th scope="row">{{ _('Query') }}</th>
-                <td><code>{{ datastore_root_url }}/datastore_search</code></td>
+                <td><code>{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', qualified=True) }}</code></td>
               </tr>
               <tr>
                 <th scope="row">{{ _('Query (via SQL)') }}</th>
-                <td><code>{{ datastore_root_url }}/datastore_search_sql</code></td>
+                <td><code>{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search_sql', qualified=True) }}</code></td>
               </tr>
 
             </tbody>
@@ -69,13 +69,12 @@ <h3>
         <div class="accordion-inner">
           <strong>{{ _('Query example (first 5 results)') }}</strong>
           <p>
-          <code><a href="{{ datastore_root_url }}/datastore_search?resource_id={{resource_id}}&limit=5" target="_blank">{{ datastore_root_url }}/datastore_search?resource_id={{resource_id}}&limit=5</a></code>
+          <code><a href="{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', resource_id=resource_id, limit=5, qualified=True) }}" target="_blank">{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', resource_id=resource_id, limit=5, qualified=True) }}</a></code>
           </p>
 
           <strong>{{ _('Query example (results containing \'jones\')') }}</strong>
           <p>
-          <code><a href="{{ datastore_root_url }}/datastore_search?resource_id={{resource_id}}&q=jones"
-              target="_blank">{{ datastore_root_url }}/datastore_search?resource_id={{resource_id}}&q=jones</a></code>
+          <code><a href="{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', resource_id=resource_id, q='jones', qualified=True) }}" target="_blank">{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', resource_id=resource_id, q='jones', qualified=True) }}</a></code>
           </p>
 
           <strong>{{ _('Query example (via SQL statement)') }}</strong>
@@ -102,7 +101,7 @@ <h3>
     q: 'jones' // query for 'jones'
   };
   $.ajax({
-    url: '{{ datastore_root_url }}/datastore_search',
+    url: '{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', qualified=True) }}',
     data: data,
     dataType: 'jsonp',
     success: function(data) {
@@ -121,7 +120,7 @@ <h3>
         <div class="accordion-inner">
           <pre>
 import urllib
-url = '{{ datastore_root_url }}/datastore_search?resource_id={{resource_id}}&amp;limit=5&amp;q=title:jones'
+url = '{{ h.url_for(qualified=True, controller='api', action='action', ver=3, logic_function='datastore_search', resource_id=resource_id, limit=5) + '&q=title:jones' }}'  {# not urlencoding the ":" because its clearer #}
 fileobj = urllib.urlopen(url)
 print fileobj.read()
 </pre>

ckan/templates/package/resource_read.html

@@ -46,7 +46,7 @@
               </li>
             {% endif %}
             {% if 'datastore' in g.plugins %}
-              <li>{% snippet 'package/snippets/data_api_button.html', resource=res, datastore_root_url=c.datastore_api %}</li>
+              <li>{% snippet 'package/snippets/data_api_button.html', resource=res %}</li>
             {% endif %}
             {% endblock %}
           </ul>

ckan/templates/package/snippets/data_api_button.html

@@ -1,11 +1,10 @@
 {# Data API Help Button
 
 resource: the resource
-datastore_root_url: the root url of the datastore
 
 #}
 {% if resource.datastore_active %}
   {% set loading_text = _('Loading...') %}
-  {% set api_info_url = h.url_for(controller='api', action='snippet', ver=1, snippet_path='api_info.html', datastore_root_url=datastore_root_url, resource_id=resource.id) %}
-  <a class="btn btn-success" href="{{ api_info_url }}" data-module="api-info" data-module-template="{{ api_info_url }}" data-loading-text="{{ loading_text }}"><i class="fa fa-flask fa-lg"></i> {{ _('Data API') }}</a>
+  {% set api_info_url = h.url_for(controller='api', action='snippet', ver=1, snippet_path='api_info.html', resource_id=resource.id) %}
+  <a class="btn btn-success" href="{{ api_info_url }}" data-module="api-info" data-module-template="{{ api_info_url }}" data-loading-text="{{ loading_text }}"><i class="icon-beaker icon-large"></i> {{ _('Data API') }}</a>
 {% endif %}

ckan/tests/controllers/test_tags.py

@@ -124,7 +124,7 @@ def test_tag_read_redirects_to_dataset_search(self):
         tag_url = url_for(controller='tag', action='read', id='find-me')
         tag_response = app.get(tag_url, status=302)
         assert_equal(tag_response.headers['Location'],
-                     'http://localhost/dataset?tags=find-me')
+                     'http://test.ckan.net/dataset?tags=find-me')
 
     def test_tag_read_not_found(self):
         '''Attempting access to non-existing tag returns a 404'''

ckan/tests/legacy/functional/test_user.py

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-from routes import url_for
+from ckan.lib.helpers import url_for
 from nose.tools import assert_equal
 from ckan.common import config
 import hashlib

ckan/tests/test_none_root.py

@@ -0,0 +1,16 @@
+# encoding: utf-8
+from pylons import config
+import ckan.plugins as p
+import ckan.tests.helpers as helpers
+
+
+class TestNoneRootCKAN():
+    @helpers.change_config(u'ckan.root_path', u'/data/{{LANG}}')
+    def test_resource_url(self):
+        app = helpers._get_test_app()
+
+        p.load(u'example_theme_v15_fanstatic')
+        content = app.get(u'/en/base.html')
+        assert u'example_theme.min.css' in content
+        assert u'href="/data/fanstatic/example_theme' in content
+        p.unload(u'example_theme_v15_fanstatic')

ckanext/example_igroupform/tests/test_controllers.py

@@ -136,7 +136,7 @@ def test_save(self):
         response = submit_and_follow(app, form, env, 'save')
         # check correct redirect
         assert_equal(response.req.url,
-                     'http://localhost/%s/saved' % custom_group_type)
+                     'http://test.ckan.net/%s/saved' % custom_group_type)
         # check saved ok
         group = model.Group.by_name(u'saved')
         assert_equal(group.title, u'')
@@ -172,7 +172,7 @@ def test_save(self):
         response = submit_and_follow(app, form, env, 'save')
         # check correct redirect
         assert_equal(response.req.url,
-                     'http://localhost/%s/saved' % group_type)
+                     'http://test.ckan.net/%s/saved' % group_type)
         # check saved ok
         group = model.Group.by_name(u'saved')
         assert_equal(group.title, u'')