Merge branch 'master' into 3196-i18n

bin/travis-install-dependencies

@@ -29,6 +29,7 @@ sudo -E -u postgres ./bin/postgres_init/1_create_ckan_db.sh
 sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh
 
 export PIP_USE_MIRRORS=true
+pip install -r requirement-setuptools.txt --allow-all-external
 pip install -r requirements.txt --allow-all-external
 pip install -r dev-requirements.txt --allow-all-external
 

circle.yml

@@ -23,6 +23,7 @@ dependencies:
            && chmod +x ~/.local/bin/circleci-matrix"
 
     override:
+        - pip install -r requirement-setuptools.txt
         - pip install -r requirements.txt
         - pip install -r dev-requirements.txt
         - python setup.py develop

ckan/controllers/group.py

@@ -382,13 +382,16 @@ def bulk_process(self, id):
         data_dict = {'id': id, 'type': group_type}
 
         try:
+            self._check_access('bulk_update_public', context, {'org_id': id})
             # Do not query for the group datasets when dictizing, as they will
             # be ignored and get requested on the controller anyway
             data_dict['include_datasets'] = False
             c.group_dict = self._action('group_show')(context, data_dict)
             c.group = context['group']
-        except (NotFound, NotAuthorized):
+        except NotFound:
             abort(404, _('Group not found'))
+        except NotAuthorized:
+            abort(403, _('User %r not authorized to edit %s') % (c.user, id))
 
         if not c.group_dict['is_organization']:
             # FIXME: better error
@@ -634,14 +637,21 @@ def members(self, id):
                    'user': c.user}
 
         try:
+            data_dict = {'id': id}
+            check_access('group_edit_permissions', context, data_dict)
             c.members = self._action('member_list')(
                 context, {'id': id, 'object_type': 'user'}
             )
-            data_dict = {'id': id}
             data_dict['include_datasets'] = False
             c.group_dict = self._action('group_show')(context, data_dict)
-        except (NotFound, NotAuthorized):
+        except NotFound:
             abort(404, _('Group not found'))
+        except NotAuthorized:
+            abort(
+                403,
+                _('User %r not authorized to edit members of %s') % (
+                    c.user, id))
+
         return self._render_template('group/members.html', group_type)
 
     def member_new(self, id):

ckan/lib/navl/dictization_functions.py

@@ -47,9 +47,17 @@ class State(object):
 
 class DictizationError(Exception):
     def __str__(self):
+        return unicode(self).encode('utf8')
+
+    def __unicode__(self):
+        if hasattr(self, 'error') and self.error:
+            return u'{}: {}'.format(self.__class__.__name__, repr(self.error))
+        return self.__class__.__name__
+
+    def __repr__(self):
         if hasattr(self, 'error') and self.error:
-            return repr(self.error)
-        return ''
+            return '<{} {}>'.format(self.__class__.__name__, repr(self.error))
+        return '<{}>'.format(self.__class__.__name__)
 
 
 class Invalid(DictizationError):

ckan/lib/navl/validators.py

@@ -117,3 +117,9 @@ def convert_int(value, context):
     except ValueError:
         raise Invalid(_('Please enter an integer value'))
 
+def unicode_only(value):
+    '''Accept only unicode values'''
+
+    if not isinstance(value, unicode):
+        raise Invalid(_('Must be a Unicode string value'))
+    return value

ckan/logic/auth/update.py

@@ -153,14 +153,15 @@ def group_edit_permissions(context, data_dict):
     user = context['user']
     group = logic_auth.get_group_object(context, data_dict)
 
-    authorized = authz.has_user_permission_for_group_or_org(group.id,
-                                                                user,
-                                                                'update')
+    authorized = authz.has_user_permission_for_group_or_org(
+        group.id, user, 'update')
 
     if not authorized:
-        return {'success': False,
-                'msg': _('User %s not authorized to edit permissions of group %s') %
-                        (str(user), group.id)}
+        return {
+            'success': False,
+            'msg': _('User %s not authorized to'
+                     ' edit permissions of group %s') %
+            (str(user), group.id)}
     else:
         return {'success': True}
 

ckan/templates/organization/members.html

@@ -10,7 +10,9 @@
 {% endblock %}
 
 {% block primary_content_inner %}
-  <h3 class="page-heading">{{ _('{0} members'.format(c.members|length)) }}</h3>
+  {% set count = c.members|length %}
+  {% set members_count = ungettext('{count} member', '{count} members', count).format(count=count) %}
+  <h3 class="page-heading">{{ members_count }}</h3>
   <table class="table table-header table-hover table-bordered">
     <thead>
       <tr>

ckan/templates/package/resource_read.html

@@ -37,7 +37,7 @@
                     <i class="fa fa-eye"></i> {{ _('View') }}
                   {% elif  res.resource_type == 'api' %}
                     <i class="fa fa-key"></i> {{ _('API Endpoint') }}
-                  {% elif not res.has_views or not res.can_be_previewed %}
+                  {% elif (not res.has_views or not res.can_be_previewed) and not res.url_type == 'upload' %}
                     <i class="fa fa-external-link"></i> {{ _('Go to resource') }}
                   {% else %}
                     <i class="fa fa-arrow-circle-o-down"></i> {{ _('Download') }}

ckan/templates/package/snippets/resource_item.html

@@ -39,7 +39,7 @@
       {% if res.url and h.is_url(res.url) %}
       <li>
         <a href="{{ res.url }}" class="resource-url-analytics" target="_blank">
-          {% if res.has_views %}
+          {% if res.has_views or res.url_type == 'upload' %}
             <i class="fa fa-arrow-circle-o-down"></i>
             {{ _('Download') }}
           {% else %}

ckan/tests/controllers/test_group.py

@@ -282,9 +282,12 @@ def test_membership_list(self):
 
         group = self._create_group(user_one['name'], other_users)
 
+        env = {'REMOTE_USER': user_one['name'].encode('ascii')}
+
         member_list_url = url_for(controller='group', action='members',
                                   id=group['id'])
-        member_list_response = app.get(member_list_url)
+        member_list_response = app.get(
+            member_list_url, extra_environ=env)
 
         assert_true('2 members' in member_list_response)
 
@@ -375,7 +378,7 @@ def test_remove_member(self):
         env = {'REMOTE_USER': user_one['name'].encode('ascii')}
         remove_response = app.post(remove_url, extra_environ=env, status=302)
         # redirected to member list after removal
-        remove_response = remove_response.follow()
+        remove_response = remove_response.follow(extra_environ=env)
 
         assert_true('Group member has been deleted.' in remove_response)
         assert_true('1 members' in remove_response)

ckan/tests/helpers.py

@@ -179,7 +179,8 @@ class FunctionalTestBase(object):
     Allows configuration changes by overriding _apply_config_changes and
     resetting the CKAN config after your test class has run. It creates a
     webtest.TestApp at self.app for your class to use to make HTTP requests
-    to the CKAN web UI or API.
+    to the CKAN web UI or API. Also loads plugins defined by
+    _load_plugins in the class definition.
 
     If you're overriding methods that this class provides, like setup_class()
     and teardown_class(), make sure to use super() to call this class's methods
@@ -196,10 +197,13 @@ def _get_test_app(cls):  # leading _ because nose is terrible
 
     @classmethod
     def setup_class(cls):
+        import ckan.plugins as p
         # Make a copy of the Pylons config, so we can restore it in teardown.
         cls._original_config = dict(config)
         cls._apply_config_changes(config)
         cls._get_test_app()
+        for plugin in getattr(cls, '_load_plugins', []):
+            p.load(plugin)
 
     @classmethod
     def _apply_config_changes(cls, cfg):
@@ -214,6 +218,9 @@ def setup(self):
 
     @classmethod
     def teardown_class(cls):
+        import ckan.plugins as p
+        for plugin in reversed(getattr(cls, '_load_plugins', [])):
+            p.unload(plugin)
         # Restore the Pylons config to its original values, in case any tests
         # changed any config settings.
         config.clear()

ckan/tests/lib/navl/test_dictization_functions.py

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 import nose
-from ckan.lib.navl.dictization_functions import validate
+from ckan.lib.navl.dictization_functions import validate, Invalid
 
 
 eq_ = nose.tools.eq_
@@ -49,3 +49,44 @@ def my_validator(key, data, errors, context):
         context = {}
 
         data, errors = validate(data_dict, schema, context)
+
+
+class TestDictizationError(object):
+
+    def test_str_error(self):
+        err_obj = Invalid('Some ascii error')
+        eq_(str(err_obj), "Invalid: 'Some ascii error'")
+
+    def test_unicode_error(self):
+        err_obj = Invalid('Some ascii error')
+        eq_(unicode(err_obj), u"Invalid: 'Some ascii error'")
+
+    def test_repr_error(self):
+        err_obj = Invalid('Some ascii error')
+        eq_(repr(err_obj), "<Invalid 'Some ascii error'>")
+
+    # Error msgs should be ascii, but let's just see what happens for unicode
+
+    def test_str_unicode_error(self):
+        err_obj = Invalid(u'Some unicode \xa3 error')
+        eq_(str(err_obj), "Invalid: u'Some unicode \\xa3 error'")
+
+    def test_unicode_unicode_error(self):
+        err_obj = Invalid(u'Some unicode \xa3 error')
+        eq_(unicode(err_obj), "Invalid: u'Some unicode \\xa3 error'")
+
+    def test_repr_unicode_error(self):
+        err_obj = Invalid(u'Some unicode \xa3 error')
+        eq_(repr(err_obj), "<Invalid u'Some unicode \\xa3 error'>")
+
+    def test_str_blank(self):
+        err_obj = Invalid('')
+        eq_(str(err_obj), "Invalid")
+
+    def test_unicode_blank(self):
+        err_obj = Invalid('')
+        eq_(unicode(err_obj), u"Invalid")
+
+    def test_repr_blank(self):
+        err_obj = Invalid('')
+        eq_(repr(err_obj), "<Invalid>")

ckanext/datapusher/logic/action.py

@@ -94,7 +94,7 @@ def datapusher_submit(context, data_dict):
         if existing_task.get('state') == 'pending':
             updated = datetime.datetime.strptime(
                 existing_task['last_updated'], '%Y-%m-%dT%H:%M:%S.%f')
-            time_since_last_updated = datetime.datetime.now() - updated
+            time_since_last_updated = datetime.datetime.utcnow() - updated
             if time_since_last_updated > assume_task_stale_after:
                 # it's been a while since the job was last updated - it's more
                 # likely something went wrong with it and the state wasn't

ckanext/datapusher/tests/test.py

@@ -68,8 +68,7 @@ def setup_class(cls):
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
-        engine = db._get_engine(
-            {'connection_url': config['ckan.datastore.write_url']})
+        engine = db.get_write_engine()
         cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
         set_url_type(
             model.Package.get('annakarenina').resources, cls.sysadmin_user)

ckanext/datapusher/tests/test_interfaces.py

@@ -61,8 +61,7 @@ def setup_class(cls):
 
         cls.sysadmin_user = factories.User(name='testsysadmin', sysadmin=True)
         cls.normal_user = factories.User(name='annafan')
-        engine = db._get_engine(
-            {'connection_url': config['ckan.datastore.write_url']})
+        engine = db.get_write_engine()
         cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
 
     @classmethod