Merge branch 'release-v2.5.4' into release-v2.5-latest

CHANGELOG.rst

@@ -7,6 +7,24 @@
 Changelog
 ---------
 
+v2.5.4 2017-02-22
+=================
+
+
+ * Fix DataPusher being fired multiple times (#3245)
+ * Use the url_for() helper for datapusher URLs (#2866)
+ * Resource creation date use datetime.utcnow() (#3447)
+ * Fix locale error when using fix ckan.root_path
+ * `render_markdown` breaks links with ampersands
+ * Check group name and id during package creation
+ * Use utcnow() on dashboard_mark_activities_old (#3373)
+ * Fix encoding error on DataStore exception
+ * Datastore doesn't add site_url to resource created via API (#3189)
+ * Fix memberships after user deletion (#3265)
+ * Remove idle database connection (#3260)
+ * Fix package_owner_org_update action when called via the API (#2661)
+
+
 v2.5.3 2016-11-02
 =================
 
@@ -108,6 +126,21 @@ v2.5.0 2015-12-17
 
 Cancelled release
 
+v2.4.5 2017-02-22
+=================
+
+ * Use the url_for() helper for datapusher URLs (#2866)
+ * Resource creation date use datetime.utcnow() (#3447)
+ * Fix locale error when using fix ckan.root_path
+ * `render_markdown` breaks links with ampersands
+ * Check group name and id during package creation
+ * Use utcnow() on dashboard_mark_activities_old (#3373)
+ * Fix encoding error on DataStore exception
+ * Datastore doesn't add site_url to resource created via API (#3189)
+ * Fix memberships after user deletion (#3265)
+ * Remove idle database connection (#3260)
+ * Fix package_owner_org_update action when called via the API (#2661)
+
 v2.4.4 2016-11-02
 =================
 

ckan/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.5.3'
+__version__ = '2.5.4'
 
 __description__ = 'CKAN Software'
 __long_description__ = \

ckan/config/environment.py

@@ -231,6 +231,11 @@ def genshi_lookup_attr(cls, obj, key):
     # load all CKAN plugins
     p.load_all(config)
 
+    # issue #3260: remove idle transaction
+    # Session that was used for getting all config params nor committed,
+    # neither removed and we have idle connection as result
+    model.Session.commit()
+
 
 # A mapping of config settings that can be overridden by env vars.
 # Note: Do not remove the following lines, they are used in the docs

ckan/controllers/util.py

@@ -15,7 +15,7 @@ def redirect(self):
         if not url:
             base.abort(400, _('Missing Value') + ': url')
 
-        if h.url_is_local(url):
+        if h.url_is_local(url) and '\r' not in url and '\n' not in url:
             return base.redirect(url)
         else:
             base.abort(403, _('Redirecting to external site is not allowed.'))

ckan/lib/cli.py

@@ -872,12 +872,9 @@ def remove(self):
             return
         username = self.args[1]
 
-        user = model.User.by_name(unicode(username))
-        if not user:
-            print 'Error: user "%s" not found!' % username
-            return
-        user.delete()
-        model.repo.commit_and_remove()
+        p.toolkit.get_action('user_delete')(
+            {'model': model, 'ignore_auth': True},
+            {'id': username})
         print('Deleted user: %s' % username)
 
 

ckan/lib/helpers.py

@@ -13,7 +13,6 @@
 import pytz
 import tzlocal
 import urllib
-import urlparse
 import pprint
 import copy
 import urlparse
@@ -27,7 +26,7 @@
 from webhelpers.text import truncate
 import webhelpers.date as date
 from markdown import markdown
-from bleach import clean as clean_html
+from bleach import clean as clean_html, ALLOWED_TAGS, ALLOWED_ATTRIBUTES
 from pylons import url as _pylons_default_url
 from pylons.decorators.cache import beaker_cache
 from pylons import config
@@ -45,11 +44,21 @@
 import ckan.logic as logic
 import ckan.lib.uploader as uploader
 import ckan.authz as authz
-
 from ckan.common import (
     _, ungettext, g, c, request, session, json, OrderedDict
 )
 
+
+MARKDOWN_TAGS = set([
+    'del', 'dd', 'dl', 'dt', 'h1', 'h2',
+    'h3', 'img', 'kbd', 'p', 'pre', 's',
+    'sup', 'sub', 'strike', 'br', 'hr'
+]).union(ALLOWED_TAGS)
+
+MARKDOWN_ATTRIBUTES = copy.deepcopy(ALLOWED_ATTRIBUTES)
+MARKDOWN_ATTRIBUTES.setdefault('img', []).extend(['src', 'alt', 'title'])
+
+
 get_available_locales = i18n.get_available_locales
 get_locales_dict = i18n.get_locales_dict
 
@@ -291,7 +300,7 @@ def _add_i18n_to_url(url_to_amend, **kw):
         if default_locale:
             root_path = re.sub('/{{LANG}}', '', root_path)
         else:
-            root_path = re.sub('{{LANG}}', locale, root_path)
+            root_path = re.sub('{{LANG}}', str(locale), root_path)
         # make sure we don't have a trailing / on the root
         if root_path[-1] == '/':
             root_path = root_path[:-1]
@@ -1727,7 +1736,9 @@ def render_markdown(data, auto_link=True, allow_html=False):
         data = markdown(data.strip())
     else:
         data = RE_MD_HTML_TAGS.sub('', data.strip())
-        data = markdown(clean_html(data, strip=True))
+        data = clean_html(
+            markdown(data), strip=True,
+            tags=MARKDOWN_TAGS, attributes=MARKDOWN_ATTRIBUTES)
     # tags can be added by tag:... or tag:"...." and a link will be made
     # from it
     if auto_link:

ckan/logic/action/create.py

@@ -195,6 +195,7 @@ def package_create(context, data_dict):
     context_org_update = context.copy()
     context_org_update['ignore_auth'] = True
     context_org_update['defer_commit'] = True
+    context_org_update['add_revision'] = False
     _get_action('package_owner_org_update')(context_org_update,
                                             {'id': pkg.id,
                                              'organization_id': pkg.owner_org})

ckan/logic/action/delete.py

@@ -48,7 +48,7 @@ def user_delete(context, data_dict):
     user.delete()
 
     user_memberships = model.Session.query(model.Member).filter(
-        model.Member.table_id == user_id).all()
+        model.Member.table_id == user.id).all()
 
     for membership in user_memberships:
         membership.delete()

ckan/logic/action/get.py

@@ -1917,23 +1917,23 @@ def package_search(context, data_dict):
         fq = data_dict.get('fq', '')
         if not context.get('ignore_capacity_check', False):
             fq = ' '.join(p for p in fq.split() if 'capacity:' not in p)
-            data_dict['fq'] = fq + ' capacity:"public"'
+            data_dict['fq'] = 'capacity:"public" ' + fq
 
         # Solr doesn't need 'include_drafts`, so pop it.
         include_drafts = data_dict.pop('include_drafts', False)
         fq = data_dict.get('fq', '')
         if include_drafts:
             user_id = authz.get_user_id_for_username(user, allow_none=True)
             if authz.is_sysadmin(user):
-                data_dict['fq'] = fq + ' +state:(active OR draft)'
+                data_dict['fq'] = '+state:(active OR draft) ' + fq
             elif user_id:
                 # Query to return all active datasets, and all draft datasets
                 # for this user.
-                data_dict['fq'] = fq + \
-                    ' ((creator_user_id:{0} AND +state:(draft OR active))' \
-                    ' OR state:active)'.format(user_id)
+                u_fq = ' ((creator_user_id:{0} AND +state:(draft OR active))' \
+                       ' OR state:active) '.format(user_id)
+                data_dict['fq'] = u_fq + ' ' + fq
         elif not authz.is_sysadmin(user):
-            data_dict['fq'] = fq + ' +state:active'
+            data_dict['fq'] = '+state:active ' + fq
 
         # Pop these ones as Solr does not need them
         extras = data_dict.pop('extras', None)

ckan/logic/action/update.py

@@ -354,6 +354,7 @@ def package_update(context, data_dict):
     context_org_update = context.copy()
     context_org_update['ignore_auth'] = True
     context_org_update['defer_commit'] = True
+    context_org_update['add_revision'] = False
     _get_action('package_owner_org_update')(context_org_update,
                                             {'id': pkg.id,
                                              'organization_id': pkg.owner_org})
@@ -1044,7 +1045,7 @@ def dashboard_mark_activities_old(context, data_dict):
     model = context['model']
     user_id = model.User.get(context['user']).id
     model.Dashboard.get(user_id).activity_stream_last_viewed = (
-            datetime.datetime.now())
+            datetime.datetime.utcnow())
     if not context.get('defer_commit'):
         model.repo.commit()
 
@@ -1082,6 +1083,7 @@ def package_owner_org_update(context, data_dict):
     :type id: string
     '''
     model = context['model']
+    user = context['user']
     name_or_id = data_dict.get('id')
     organization_id = data_dict.get('organization_id')
 
@@ -1101,10 +1103,17 @@ def package_owner_org_update(context, data_dict):
         org = None
         pkg.owner_org = None
 
+    if context.get('add_revision', True):
+        rev = model.repo.new_revision()
+        rev.author = user
+        if 'message' in context:
+            rev.message = context['message']
+        else:
+            rev.message = _(u'REST API: Update object %s') % pkg.get("name")
 
     members = model.Session.query(model.Member) \
-            .filter(model.Member.table_id == pkg.id) \
-            .filter(model.Member.capacity == 'organization')
+        .filter(model.Member.table_id == pkg.id) \
+        .filter(model.Member.capacity == 'organization')
 
     need_update = True
     for member_obj in members:

ckan/logic/auth/create.py

@@ -194,10 +194,8 @@ def _check_group_auth(context, data_dict):
     for group_blob in group_blobs:
         # group_blob might be a dict or a group_ref
         if isinstance(group_blob, dict):
-            if api_version == '1':
-                id = group_blob.get('name')
-            else:
-                id = group_blob.get('id')
+            # use group id by default, but we can accept name as well
+            id = group_blob.get('id') or group_blob.get('name')
             if not id:
                 continue
         else:

ckan/model/dashboard.py

@@ -19,8 +19,8 @@ class Dashboard(object):
 
     def __init__(self, user_id):
         self.user_id = user_id
-        self.activity_stream_last_viewed = datetime.datetime.now()
-        self.email_last_sent = datetime.datetime.now()
+        self.activity_stream_last_viewed = datetime.datetime.utcnow()
+        self.email_last_sent = datetime.datetime.utcnow()
 
     @classmethod
     def get(cls, user_id):

ckan/model/resource.py

@@ -47,7 +47,7 @@
     Column('mimetype', types.UnicodeText),
     Column('mimetype_inner', types.UnicodeText),
     Column('size', types.BigInteger),
-    Column('created', types.DateTime, default=datetime.datetime.now),
+    Column('created', types.DateTime, default=datetime.datetime.utcnow),
     Column('last_modified', types.DateTime),
     Column('cache_url', types.UnicodeText),
     Column('cache_last_updated', types.DateTime),

ckan/tests/legacy/functional/api/test_activity.py

@@ -19,6 +19,7 @@
 from nose import SkipTest
 from ckan.common import json
 import ckan.tests.legacy as tests
+from ckan.tests.helpers import call_action
 
 
 ##def package_update(context, data_dict):
@@ -188,6 +189,7 @@ def setup_class(self):
                 'name': sysadmin_user.name,
                 }
         normal_user = model.User.get('annafan')
+
         self.normal_user = {
                 'id': normal_user.id,
                 'apikey': normal_user.apikey,
@@ -274,7 +276,7 @@ def record_details(self, user_id, package_id=None, group_ids=None,
         details['recently changed datasets stream'] = \
                 self.recently_changed_datasets_stream(apikey)
 
-        details['time'] = datetime.datetime.now()
+        details['time'] = datetime.datetime.utcnow()
         return details
 
     def _create_package(self, user, name=None):
@@ -289,10 +291,17 @@ def _create_package(self, user, name=None):
 
         # Create a new package.
         request_data = make_package(name)
+
         before = self.record_details(user_id=user_id,
                 group_ids=[group['name'] for group in request_data['groups']],
                 apikey=apikey)
         extra_environ = {'Authorization': str(user['apikey'])}
+
+        call_action('member_create',
+                    capacity='admin',
+                    object=user['id'],
+                    object_type='user',
+                    id='roger')
         response = self.app.post('/api/action/package_create',
                 json.dumps(request_data), extra_environ=extra_environ)
         response_dict = json.loads(response.body)
@@ -1402,7 +1411,7 @@ def test_create_user(self):
         a new user is created.
 
         """
-        before = datetime.datetime.now()
+        before = datetime.datetime.utcnow()
 
         # Create a new user.
         user_dict = {'name': 'testuser',

ckan/tests/lib/test_helpers.py

@@ -1,5 +1,6 @@
 import nose
 import i18n
+from babel import Locale
 
 import ckan.lib.helpers as h
 import ckan.exceptions
@@ -74,6 +75,14 @@ def test_url_for_with_locale(self):
                                   locale='de')
         eq_(generated_url, url)
 
+    @helpers.change_config('ckan.site_url', 'http://example.com')
+    @helpers.change_config('ckan.root_path', '/foo/{{LANG}}')
+    def test_url_for_with_locale_object(self):
+        url = '/foo/de/dataset/my_dataset'
+        generated_url = h.url_for('/dataset/my_dataset',
+                                  locale=Locale('de'))
+        eq_(generated_url, url)
+
     @helpers.change_config('ckan.site_url', 'http://example.com')
     def test_url_for_not_qualified(self):
         url = '/dataset/my_dataset'
@@ -155,6 +164,45 @@ def test_render_naughty_markdown(self):
         output = u'<ul>\n<li>[Foo (<a href="http://foo.bar" target="_blank" rel="nofollow">http://foo.bar</a>) * Bar] (<a href="http://foo.bar" target="_blank" rel="nofollow">http://foo.bar</a>)</li>\n</ul>'
         eq_(h.render_markdown(data), output)
 
+    def test_render_markdown_with_js(self):
+        data = u'[text](javascript: alert(1))'
+        output = u'<p><a>text</a></p>'
+        eq_(h.render_markdown(data), output)
+
+    def test_event_attributes(self):
+        data = u'<p onclick="some.script"><img onmouseover="some.script" src="image.png" /> and text</p>'
+        output = u'<p>and text</p>'
+        eq_(h.render_markdown(data), output)
+
+    def test_ampersand_in_links(self):
+        data = u'[link](/url?a=1&b=2)'
+        output = u'<p><a href="/url?a=1&amp;b=2">link</a></p>'
+        eq_(h.render_markdown(data), output)
+
+        data = u'http://example.com/page?a=1&b=2'
+        output = u'<p><a href="http://example.com/page?a=1&amp;b=2" target="_blank" rel="nofollow">http://example.com/page?a=1&amp;b=2</a></p>'
+        eq_(h.render_markdown(data), output)
+
+    def test_tags_h1(self):
+        data = u'#heading'
+        output = u'<h1>heading</h1>'
+        eq_(h.render_markdown(data), output)
+
+    def test_tags_h2(self):
+        data = u'##heading'
+        output = u'<h2>heading</h2>'
+        eq_(h.render_markdown(data), output)
+
+    def test_tags_h3(self):
+        data = u'###heading'
+        output = u'<h3>heading</h3>'
+        eq_(h.render_markdown(data), output)
+
+    def test_tags_img(self):
+        data = u'![image](/image.png)'
+        output = u'<p><img alt="image" src="/image.png"></p>'
+        eq_(h.render_markdown(data), output)
+
 
 class TestHelpersRemoveLineBreaks(object):