Merge remote-tracking branch 'origin/master' into alembic

CHANGELOG.rst

@@ -10,6 +10,16 @@ Changelog
 v.2.9.0 TBA
 ==================
 
+Minor changes:
+
+ * For navl schemas, the 'default' validator no longer applies the default when
+   the value is False, 0, [] or {} (#4448)
+
+Bugfixes:
+
+ * Action function "datastore_search" would calculate the total, even if you
+   set include_total=False (#4448)
+
 Deprecations:
  * ``c.action`` and ``c.controller`` variables should be avoided.
    ``ckan.plugins.toolkit.get_endpoint`` can be used instead. This function

ckan/lib/cli.py

@@ -118,6 +118,7 @@ def user_add(args):
 
     # Required
     while '@' not in data_dict.get('email', ''):
+        print('Error: Invalid email address')
         data_dict['email'] = input('Email address: ').strip()
 
     if 'password' not in data_dict:

ckan/lib/helpers.py

@@ -69,8 +69,17 @@
     'home': 'home.index',
     'about': 'home.about',
     'search': 'dataset.search',
+    'dataset_read': 'dataset.read',
+    'dataset_activity': 'dataset.activity',
+    'dataset_groups': 'dataset.groups',
     'group_index': 'group.index',
-    'organizations_index': 'organization.index'
+    'group_about': 'group.about',
+    'group_read': 'group.read',
+    'group_activity': 'group.activity',
+    'organizations_index': 'organization.index',
+    'organization_activity': 'organization.activity',
+    'organization_read': 'organization.read',
+    'organization_about': 'organization.about',
 }
 
 
@@ -2280,7 +2289,8 @@ def resource_view_get_fields(resource):
 
     data = {
         'resource_id': resource['id'],
-        'limit': 0
+        'limit': 0,
+        'include_total': False,
     }
     result = logic.get_action('datastore_search')({}, data)
 

ckan/lib/navl/validators.py

@@ -74,11 +74,13 @@ def ignore(key, data, errors, context):
     raise StopOnError
 
 def default(default_value):
+    '''When key is missing or value is an empty string or None, replace it with
+    a default value'''
 
     def callable(key, data, errors, context):
 
         value = data.get(key)
-        if not value or value is missing:
+        if value is None or value == '' or value is missing:
             data[key] = default_value
 
     return callable

ckan/templates/package/base.html

@@ -12,8 +12,8 @@
     {% if pkg.organization %}
       {% set organization = h.get_translated(pkg.organization, 'title') or pkg.organization.name %}
       {% set group_type = pkg.organization.type %}
-      <li>{% link_for _('Organizations'), controller='organization', action='index', named_route=group_type + '_index' %}</li>
-      <li>{% link_for organization|truncate(30), controller='organization', action='read', id=pkg.organization.name, named_route=group_type + '_read' %}</li>
+      <li>{% link_for _('Organizations'), named_route='organization.index' %}</li>
+      <li>{% link_for organization|truncate(30), named_route='organization.read', id=pkg.organization.name %}</li>
     {% else %}
       <li>{% link_for _('Datasets'), named_route='dataset.search' %}</li>
     {% endif %}

ckan/tests/legacy/lib/test_navl.py

@@ -186,21 +186,6 @@ def test_basic_errors():
 
     assert errors == {('__junk',): [u"The input field [('4', 1, '30')] was not expected."], ('1',): [u'Missing value'], ('__extras',): [u'The input field __extras was not expected.']}, errors
 
-def test_default():
-    schema = {
-        "__junk": [ignore],
-        "__extras": [ignore, default("weee")],
-        "__before": [ignore],
-        "__after": [ignore],
-        "0": [default("default")],
-        "1": [default("default")],
-    }
-
-    converted_data, errors = validate_flattened(data, schema)
-
-    assert not errors
-    assert converted_data == {('1',): 'default', ('0',): '0 value'}, converted_data
-
 
 def test_flatten():
 

ckan/tests/lib/navl/test_validators.py

@@ -8,6 +8,10 @@
 import nose.tools
 
 import ckan.tests.factories as factories
+import ckan.lib.navl.validators as validators
+
+
+eq_ = nose.tools.eq_
 
 
 def returns_None(function):
@@ -222,7 +226,6 @@ def test_ignore_missing_with_value_missing(self):
 
         '''
         import ckan.lib.navl.dictization_functions as df
-        import ckan.lib.navl.validators as validators
 
         for value in (None, df.missing, 'skip'):
 
@@ -251,8 +254,6 @@ def test_ignore_missing_with_a_value(self):
         nothing.
 
         '''
-        import ckan.lib.navl.validators as validators
-
         key = ('key to be validated',)
         data = factories.validator_data_dict()
         data[key] = 'value to be validated'
@@ -265,3 +266,32 @@ def test_ignore_missing_with_a_value(self):
         def call_validator(*args, **kwargs):
             return validators.ignore_missing(*args, **kwargs)
         call_validator(key=key, data=data, errors=errors, context={})
+
+
+class TestDefault(object):
+    def test_key_doesnt_exist(self):
+        dict_ = {}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': 'default_value'})
+
+    def test_value_is_none(self):
+        dict_ = {'key': None}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': 'default_value'})
+
+    def test_value_is_empty_string(self):
+        dict_ = {'key': ''}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': 'default_value'})
+
+    def test_value_is_false(self):
+        # False is a consciously set value, so should not be changed to the
+        # default
+        dict_ = {'key': False}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': False})
+
+    def test_already_has_a_value(self):
+        dict_ = {'key': 'original'}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': 'original'})

ckan/tests/lib/test_helpers.py

@@ -563,6 +563,51 @@ def test_legacy_pylon_routes(self):
             '<li><a href="/group/">Groups</a></li>'
             '<li><a href="/about">About</a></li>'))
 
+    def test_dataset_navigation_legacy_routes(self):
+        dataset_name = 'test-dataset'
+        eq_(
+            h.build_nav_icon('dataset_read', 'Datasets', id=dataset_name),
+            '<li><a href="/dataset/test-dataset">Datasets</a></li>'
+        )
+        eq_(
+            h.build_nav_icon('dataset_groups', 'Groups', id=dataset_name),
+            '<li><a href="/dataset/groups/test-dataset">Groups</a></li>'
+        )
+        eq_(
+            h.build_nav_icon('dataset_activity', 'Activity Stream', id=dataset_name),
+            '<li><a href="/dataset/activity/test-dataset">Activity Stream</a></li>'
+        )
+
+    def test_group_navigation_legacy_routes(self):
+        group_name = 'test-group'
+        eq_(
+            h.build_nav_icon('group_read', 'Datasets', id=group_name),
+            '<li><a href="/group/test-group">Datasets</a></li>'
+        )
+        eq_(
+            h.build_nav_icon('group_activity', 'Activity Stream', id=group_name),
+            '<li><a href="/group/activity/test-group">Activity Stream</a></li>'
+        )
+        eq_(
+            h.build_nav_icon('group_about', 'About', id=group_name),
+            '<li><a href="/group/about/test-group">About</a></li>'
+        )
+
+    def test_organization_navigation_legacy_routes(self):
+        org_name = 'test-org'
+        eq_(
+            h.build_nav_icon('organization_read', 'Datasets', id=org_name),
+            '<li><a href="/organization/test-org">Datasets</a></li>'
+        )
+        eq_(
+            h.build_nav_icon('organization_activity', 'Activity Stream', id=org_name),
+            '<li><a href="/organization/activity/test-org">Activity Stream</a></li>'
+        )
+        eq_(
+            h.build_nav_icon('organization_about', 'About', id=org_name),
+            '<li><a href="/organization/about/test-org">About</a></li>'
+        )
+
 
 class TestHelperException(helpers.FunctionalTestBase):
 

ckanext/datastore/controller.py

@@ -143,7 +143,7 @@ def result_page(offs, lim):
             'offset': offs,
             'sort': '_id',
             'records_format': records_format,
-            'include_total': 'false',  # XXX: default() is broken
+            'include_total': False,
         })
 
     result = result_page(offset, limit)

ckanext/datastore/tests/test_search.py

@@ -100,6 +100,23 @@ def test_all_params_work_with_fields_with_whitespaces(self):
         result_years = [r['the year'] for r in result['records']]
         assert_equals(result_years, [2013])
 
+    def test_search_without_total(self):
+        resource = factories.Resource()
+        data = {
+            'resource_id': resource['id'],
+            'force': True,
+            'records': [
+                {'the year': 2014},
+                {'the year': 2013},
+            ],
+        }
+        result = helpers.call_action('datastore_create', **data)
+        search_data = {
+            'resource_id': resource['id'],
+            'include_total': False
+        }
+        result = helpers.call_action('datastore_search', **search_data)
+        assert 'total' not in result
 
 
 class TestDatastoreSearch(DatastoreLegacyTestBase):