Merge branch 'master' into streaming-responses

bin/travis-install-dependencies

@@ -29,6 +29,7 @@ sudo -E -u postgres ./bin/postgres_init/1_create_ckan_db.sh
 sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh
 
 export PIP_USE_MIRRORS=true
+pip install -r requirement-setuptools.txt --allow-all-external
 pip install -r requirements.txt --allow-all-external
 pip install -r dev-requirements.txt --allow-all-external
 

circle.yml

@@ -23,6 +23,7 @@ dependencies:
            && chmod +x ~/.local/bin/circleci-matrix"
 
     override:
+        - pip install -r requirement-setuptools.txt
         - pip install -r requirements.txt
         - pip install -r dev-requirements.txt
         - python setup.py develop

ckan/controllers/group.py

@@ -382,13 +382,16 @@ def bulk_process(self, id):
         data_dict = {'id': id, 'type': group_type}
 
         try:
+            self._check_access('bulk_update_public', context, {'org_id': id})
             # Do not query for the group datasets when dictizing, as they will
             # be ignored and get requested on the controller anyway
             data_dict['include_datasets'] = False
             c.group_dict = self._action('group_show')(context, data_dict)
             c.group = context['group']
-        except (NotFound, NotAuthorized):
+        except NotFound:
             abort(404, _('Group not found'))
+        except NotAuthorized:
+            abort(403, _('User %r not authorized to edit %s') % (c.user, id))
 
         if not c.group_dict['is_organization']:
             # FIXME: better error
@@ -634,14 +637,21 @@ def members(self, id):
                    'user': c.user}
 
         try:
+            data_dict = {'id': id}
+            check_access('group_edit_permissions', context, data_dict)
             c.members = self._action('member_list')(
                 context, {'id': id, 'object_type': 'user'}
             )
-            data_dict = {'id': id}
             data_dict['include_datasets'] = False
             c.group_dict = self._action('group_show')(context, data_dict)
-        except (NotFound, NotAuthorized):
+        except NotFound:
             abort(404, _('Group not found'))
+        except NotAuthorized:
+            abort(
+                403,
+                _('User %r not authorized to edit members of %s') % (
+                    c.user, id))
+
         return self._render_template('group/members.html', group_type)
 
     def member_new(self, id):

ckan/lib/lazyjson.py

@@ -0,0 +1,55 @@
+# encoding: utf-8
+
+
+from simplejson import loads, RawJSON, dumps
+
+
+class LazyJSONObject(RawJSON):
+    u'''
+    An object that behaves like a dict returned from json.loads
+    but when passed to simplejson.dumps will render original
+    string passed when possible. Accepts and produces only
+    unicode strings containing a single JSON object.
+    '''
+    def __init__(self, json_string):
+        assert isinstance(json_string, unicode), json_string
+        self._json_string = json_string
+        self._json_dict = None
+
+    def _loads(self):
+        if not self._json_dict:
+            self._json_dict = loads(self._json_string)
+            self._json_string = None
+        return self._json_dict
+
+    def __nonzero__(self):
+        return True
+
+    def __repr__(self):
+        if self._json_string:
+            return u'<LazyJSONObject %r>' % self._json_string
+        return u'<LazyJSONObject %r>' % self._json_dict
+
+    @property
+    def encoded_json(self):
+        if self._json_string:
+            return self._json_string
+        return dumps(
+            self._json_dict,
+            ensure_ascii=False,
+            separators=(u',', u':'))
+
+
+def _loads_method(name):
+    def method(self, *args, **kwargs):
+        return getattr(self._loads(), name)(*args, **kwargs)
+    return method
+
+
+for fn in [u'__contains__', u'__delitem__', u'__eq__', u'__ge__',
+           u'__getitem__', u'__gt__', u'__iter__', u'__le__', u'__len__',
+           u'__lt__', u'__ne__', u'__setitem__', u'clear', u'copy',
+           u'fromkeys', u'get', u'has_key', u'items', u'iteritems',
+           u'iterkeys', u'itervalues', u'keys', u'pop', u'popitem',
+           u'setdefault', u'update', u'values']:
+    setattr(LazyJSONObject, fn, _loads_method(fn))

ckan/lib/navl/validators.py

@@ -117,3 +117,9 @@ def convert_int(value, context):
     except ValueError:
         raise Invalid(_('Please enter an integer value'))
 
+def unicode_only(value):
+    '''Accept only unicode values'''
+
+    if not isinstance(value, unicode):
+        raise Invalid(_('Must be a Unicode string value'))
+    return value

ckan/logic/auth/update.py

@@ -153,14 +153,15 @@ def group_edit_permissions(context, data_dict):
     user = context['user']
     group = logic_auth.get_group_object(context, data_dict)
 
-    authorized = authz.has_user_permission_for_group_or_org(group.id,
-                                                                user,
-                                                                'update')
+    authorized = authz.has_user_permission_for_group_or_org(
+        group.id, user, 'update')
 
     if not authorized:
-        return {'success': False,
-                'msg': _('User %s not authorized to edit permissions of group %s') %
-                        (str(user), group.id)}
+        return {
+            'success': False,
+            'msg': _('User %s not authorized to'
+                     ' edit permissions of group %s') %
+            (str(user), group.id)}
     else:
         return {'success': True}
 

ckan/logic/schema.py

@@ -70,6 +70,7 @@
     extra_key_not_in_root_schema,
     empty_if_not_sysadmin,
     package_id_does_not_exist,
+    email_validator
     )
 
 
@@ -146,9 +147,9 @@ def default_create_package_schema():
         'name': [not_empty, unicode, name_validator, package_name_validator],
         'title': [if_empty_same_as("name"), unicode],
         'author': [ignore_missing, unicode],
-        'author_email': [ignore_missing, unicode],
+        'author_email': [ignore_missing, unicode, email_validator],
         'maintainer': [ignore_missing, unicode],
-        'maintainer_email': [ignore_missing, unicode],
+        'maintainer_email': [ignore_missing, unicode, email_validator],
         'license_id': [ignore_missing, unicode],
         'notes': [ignore_missing, unicode],
         'url': [ignore_missing, unicode],  # , URL(add_http=False)],

ckan/logic/validators.py

@@ -831,3 +831,17 @@ def empty_if_not_sysadmin(key, data, errors, context):
         return
 
     empty(key, data, errors, context)
+
+#pattern from https://html.spec.whatwg.org/#e-mail-state-(type=email)
+email_pattern = re.compile(r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9]"\
+                       "(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]"\
+                       "(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$")
+
+
+def email_validator(value, context):
+    '''Validate email input '''
+
+    if value:
+        if not email_pattern.match(value):
+            raise Invalid(_('Email {email} is not a valid format').format(email=value))
+    return value

ckan/public/base/javascript/modules/resource-view-filters.js

@@ -6,7 +6,7 @@ this.ckan.module('resource-view-filters', function (jQuery) {
         resourceId = self.options.resourceId,
         fields = self.options.fields,
         dropdownTemplate = self.options.dropdownTemplate,
-        addFilterTemplate = '<a href="#">' + self._('Add Filter') + '</a>';
+        addFilterTemplate = '<a href="#">' + self._('Add Filter') + '</a>',
         filtersDiv = $('<div></div>');
 
     var filters = ckan.views.filters.get();

ckan/public/base/less/forms.less

@@ -170,7 +170,7 @@ textarea {
 
 @media (min-width: 980px) {
   .form-horizontal .info-block {
-    padding: 6px 0 6px 25px;
+    padding: 0 0 6px 25px;
   }
   .form-horizontal .info-inline {
     float: right;

ckan/public/base/less/media.less

@@ -28,6 +28,11 @@
   .media-grid {
     margin-left:-27px;
   }
+  .module-content {
+    .wide .media-grid {
+      margin-left:-25px;
+    }
+  } 
 }
 
 .media-item {

ckan/templates/package/resource_read.html

@@ -37,7 +37,7 @@
                     <i class="fa fa-eye"></i> {{ _('View') }}
                   {% elif  res.resource_type == 'api' %}
                     <i class="fa fa-key"></i> {{ _('API Endpoint') }}
-                  {% elif not res.has_views or not res.can_be_previewed %}
+                  {% elif (not res.has_views or not res.can_be_previewed) and not res.url_type == 'upload' %}
                     <i class="fa fa-external-link"></i> {{ _('Go to resource') }}
                   {% else %}
                     <i class="fa fa-arrow-circle-o-down"></i> {{ _('Download') }}

ckan/templates/package/snippets/resource_item.html

@@ -39,7 +39,7 @@
       {% if res.url and h.is_url(res.url) %}
       <li>
         <a href="{{ res.url }}" class="resource-url-analytics" target="_blank">
-          {% if res.has_views %}
+          {% if res.has_views or res.url_type == 'upload' %}
             <i class="fa fa-arrow-circle-o-down"></i>
             {{ _('Download') }}
           {% else %}

ckan/tests/controllers/test_group.py

@@ -282,9 +282,12 @@ def test_membership_list(self):
 
         group = self._create_group(user_one['name'], other_users)
 
+        env = {'REMOTE_USER': user_one['name'].encode('ascii')}
+
         member_list_url = url_for(controller='group', action='members',
                                   id=group['id'])
-        member_list_response = app.get(member_list_url)
+        member_list_response = app.get(
+            member_list_url, extra_environ=env)
 
         assert_true('2 members' in member_list_response)
 
@@ -375,7 +378,7 @@ def test_remove_member(self):
         env = {'REMOTE_USER': user_one['name'].encode('ascii')}
         remove_response = app.post(remove_url, extra_environ=env, status=302)
         # redirected to member list after removal
-        remove_response = remove_response.follow()
+        remove_response = remove_response.follow(extra_environ=env)
 
         assert_true('Group member has been deleted.' in remove_response)
         assert_true('1 members' in remove_response)

ckan/tests/helpers.py

@@ -179,7 +179,8 @@ class FunctionalTestBase(object):
     Allows configuration changes by overriding _apply_config_changes and
     resetting the CKAN config after your test class has run. It creates a
     webtest.TestApp at self.app for your class to use to make HTTP requests
-    to the CKAN web UI or API.
+    to the CKAN web UI or API. Also loads plugins defined by
+    _load_plugins in the class definition.
 
     If you're overriding methods that this class provides, like setup_class()
     and teardown_class(), make sure to use super() to call this class's methods
@@ -196,10 +197,13 @@ def _get_test_app(cls):  # leading _ because nose is terrible
 
     @classmethod
     def setup_class(cls):
+        import ckan.plugins as p
         # Make a copy of the Pylons config, so we can restore it in teardown.
         cls._original_config = dict(config)
         cls._apply_config_changes(config)
         cls._get_test_app()
+        for plugin in getattr(cls, '_load_plugins', []):
+            p.load(plugin)
 
     @classmethod
     def _apply_config_changes(cls, cfg):
@@ -214,6 +218,9 @@ def setup(self):
 
     @classmethod
     def teardown_class(cls):
+        import ckan.plugins as p
+        for plugin in reversed(getattr(cls, '_load_plugins', [])):
+            p.unload(plugin)
         # Restore the Pylons config to its original values, in case any tests
         # changed any config settings.
         config.clear()

ckan/tests/legacy/functional/api/test_activity.py

@@ -139,9 +139,9 @@ def make_package(name=None):
         'name': name,
         'title': 'My Test Package',
         'author': 'test author',
-        'author_email': 'test_author@test_author.com',
+        'author_email': 'test_author@testauthor.com',
         'maintainer': 'test maintainer',
-        'maintainer_email': 'test_maintainer@test_maintainer.com',
+        'maintainer_email': 'test_maintainer@testmaintainer.com',
         'notes': 'some test notes',
         'url': 'www.example.com',
         }

ckanext/datapusher/tests/test.py

@@ -68,8 +68,7 @@ def setup_class(cls):
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
-        engine = db._get_engine(
-            {'connection_url': config['ckan.datastore.write_url']})
+        engine = db.get_write_engine()
         cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
         set_url_type(
             model.Package.get('annakarenina').resources, cls.sysadmin_user)

ckanext/datapusher/tests/test_interfaces.py

@@ -61,8 +61,7 @@ def setup_class(cls):
 
         cls.sysadmin_user = factories.User(name='testsysadmin', sysadmin=True)
         cls.normal_user = factories.User(name='annafan')
-        engine = db._get_engine(
-            {'connection_url': config['ckan.datastore.write_url']})
+        engine = db.get_write_engine()
         cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
 
     @classmethod

ckanext/datastore/controller.py

@@ -28,7 +28,7 @@
 boolean_validator = get_validator('boolean_validator')
 
 DUMP_FORMATS = 'csv', 'tsv', 'json', 'xml'
-PAGINATE_BY = 10000
+PAGINATE_BY = 32000
 
 
 class DatastoreController(BaseController):
@@ -100,16 +100,22 @@ def dictionary(self, id, resource_id):
 
 
 def dump_to(resource_id, output, fmt, offset, limit, options):
+    if fmt == 'csv':
+        writer_factory = csv_writer
+        records_format = 'csv'
+    elif fmt == 'tsv':
+        writer_factory = tsv_writer
+        records_format = 'tsv'
+    elif fmt == 'json':
+        writer_factory = json_writer
+        records_format = 'lists'
+    elif fmt == 'xml':
+        writer_factory = xml_writer
+        records_format = 'objects'
+
     def start_writer(fields):
         bom = options.get(u'bom', False)
-        if fmt == 'csv':
-            return csv_writer(output, fields, resource_id, bom)
-        if fmt == 'tsv':
-            return tsv_writer(output, fields, resource_id, bom)
-        if fmt == 'json':
-            return json_writer(output, fields, resource_id, bom)
-        if fmt == 'xml':
-            return xml_writer(output, fields, resource_id, bom)
+        return writer_factory(output, fields, resource_id, bom)
 
     def result_page(offs, lim):
         return get_action('datastore_search')(None, {
@@ -118,6 +124,8 @@ def result_page(offs, lim):
                 PAGINATE_BY if limit is None
                 else min(PAGINATE_BY, lim),
             'offset': offs,
+            'records_format': records_format,
+            'include_total': 'false',  # XXX: default() is broken
             })
 
     result = result_page(offset, limit)
@@ -128,13 +136,20 @@ def result_page(offs, lim):
             if limit is not None and limit <= 0:
                 break
 
-            for record in result['records']:
-                wr.writerow([record[column] for column in columns])
+            records = result['records']
+
+            wr.write_records(records)
 
-            if len(result['records']) < PAGINATE_BY:
+            if records_format == 'objects' or records_format == 'lists':
+                if len(records) < PAGINATE_BY:
+                    break
+            elif not records:
                 break
+
             offset += PAGINATE_BY
             if limit is not None:
                 limit -= PAGINATE_BY
+                if limit <= 0:
+                    break
 
             result = result_page(offset, limit)