Handle utf-8 decoding

ckan/config/middleware/common_middleware.py

@@ -4,6 +4,7 @@
 import hashlib
 import cgi
 
+import six
 from six.moves.urllib.parse import unquote
 
 import sqlalchemy as sa
@@ -74,7 +75,7 @@ def __call__(self, environ, start_response):
             data = {}
             for part in parts:
                 k, v = part.split('=')
-                data[k] = unquote(v).decode("utf8")
+                data[k] = six.ensure_text(unquote(v))
             start_response('200 OK', [('Content-Type', 'text/html')])
             # we want a unique anonomized key for each user so that we do
             # not count multiple clicks from the same user.

ckan/controllers/feed.py

@@ -23,6 +23,7 @@
 # TODO fix imports
 import logging
 
+import six
 from six import text_type
 from six.moves.urllib.parse import urlparse
 
@@ -550,11 +551,11 @@ def add_item_elements(self, handler, item):
 
         if(item['updated']):
             handler.addQuickElement(u'updated',
-                                    dfunc(item['updated']).decode('utf-8'))
+                                    six.ensure_text(dfunc(item['updated'])))
 
         if(item['published']):
             handler.addQuickElement(u'published',
-                                    dfunc(item['published']).decode('utf-8'))
+                                    six.ensure_text(dfunc(item['published'])))
 
     def add_root_elements(self, handler):
         """

ckan/i18n/check_po_files.py

@@ -76,7 +76,6 @@ def check_po_files(paths):
                     msgid, msgstr.encode('ascii', 'replace')))
 
 
-
 def check_po_file(path):
     errors = []
 

ckan/lib/app_globals.py

@@ -139,7 +139,7 @@ def get_config_value(key, default=''):
         # sort encodeings if needed
         if isinstance(config_value, str) and six.PY2:
             try:
-                config_value = config_value.decode('utf-8')
+                config_value = six.ensure_text(config_value)
             except UnicodeDecodeError:
                 config_value = config_value.decode('latin-1')
         # we want to store the config the first time we get here so we can

ckan/lib/lazyjson.py

@@ -23,7 +23,7 @@ def _loads(self):
             self._json_string = None
         return self._json_dict
 
-    def __nonzero__(self):
+    def __bool__(self):
         return True
 
     def __repr__(self):
@@ -50,7 +50,7 @@ def method(self, *args, **kwargs):
 for fn in [u'__contains__', u'__delitem__', u'__eq__', u'__ge__',
            u'__getitem__', u'__gt__', u'__iter__', u'__le__', u'__len__',
            u'__lt__', u'__ne__', u'__setitem__', u'clear', u'copy',
-           u'fromkeys', u'get',  u'items', u'iteritems',
+           u'fromkeys', u'get', u'items', u'iteritems',
            u'iterkeys', u'itervalues', u'keys', u'pop', u'popitem',
            u'setdefault', u'update', u'values']:
     setattr(LazyJSONObject, fn, _loads_method(fn))

ckan/lib/navl/dictization_functions.py

@@ -33,7 +33,7 @@ def __oct__(self):
     def __hex__(self):
         raise Invalid(_('Missing value'))
 
-    def __nonzero__(self):
+    def __bool__(self):
         return False
 
 

ckan/lib/navl/validators.py

@@ -163,7 +163,7 @@ def unicode_safe(value):
         # bytes only arrive when core ckan or plugins call
         # actions from Python code
         try:
-            return value.decode(u'utf8')
+            return six.ensure_text(value)
         except UnicodeDecodeError:
             return value.decode(u'cp1252')
     try:

ckan/lib/webassets_tools.py

@@ -17,6 +17,7 @@
 
 yaml.warnings({'YAMLLoadWarning': False})
 
+
 def create_library(name, path):
     """Create WebAssets library(set of Bundles).
     """

ckan/logic/action/create.py

@@ -941,7 +941,6 @@ def user_create(context, data_dict):
     session = context['session']
 
     _check_access('user_create', context, data_dict)
-
     data, errors = _validate(data_dict, schema, context)
 
     if errors:

ckan/logic/action/delete.py

@@ -5,6 +5,7 @@
 import logging
 
 import sqlalchemy as sqla
+import six
 
 import ckan.lib.jobs as jobs
 import ckan.logic
@@ -380,7 +381,7 @@ def _group_or_org_delete(context, data_dict, is_org=False):
         activity_type = 'deleted group'
 
     activity_dict = {
-        'user_id': model.User.by_name(user.decode('utf8')).id,
+        'user_id': model.User.by_name(six.ensure_text(user)).id,
         'object_id': group.id,
         'activity_type': activity_type,
         'data': {

ckan/logic/validators.py

@@ -607,7 +607,6 @@ def user_passwords_match(key, data, errors, context):
 def user_password_not_empty(key, data, errors, context):
     '''Only check if password is present if the user is created via action API.
        If not, user_both_passwords_entered will handle the validation'''
-
     # sysadmin may provide password_hash directly for importing users
     if (data.get(('password_hash',), missing) is not missing and
             authz.is_sysadmin(context.get('user'))):

ckan/model/license.py

@@ -41,7 +41,7 @@ def __init__(self, data):
                 if six.PY2:
                     # Convert str to unicode
                     # (keeps Pylons and SQLAlchemy happy).
-                    value = value.decode('utf8')
+                    value = six.ensure_text(value)
                 self._data[key] = value
 
     def __getattr__(self, name):
@@ -53,7 +53,12 @@ def __getattr__(self, name):
             log.warn('license.is_osi_compliant is deprecated - use '
                      'osd_conformance instead.')
             return self._data['osd_conformance'] == 'approved'
-        return self._data[name]
+        try:
+            return self._data[name]
+        except KeyError as e:
+            # Python3 strictly requires `AttributeError` for correct
+            # behavior of `hasattr`
+            raise AttributeError(*e.args)
 
     @maintain.deprecated("License.__getitem__() is deprecated and will be "
                          "removed in a future version of CKAN. Instead, "
@@ -217,7 +222,7 @@ def __getitem__(self, key):
             else:
                 return value
         else:
-            raise KeyError()
+            raise KeyError(key)
 
     def copy(self):
         ''' create a dict of the license used by the licenses api '''

ckan/model/user.py

@@ -102,7 +102,7 @@ def _set_password(self, password):
         hashed_password = pbkdf2_sha512.encrypt(password)
 
         if not isinstance(hashed_password, text_type):
-            hashed_password = hashed_password.decode('utf-8')
+            hashed_password = six.ensure_text(hashed_password)
         self._password = hashed_password
 
     def _get_password(self):

ckan/tests/config/test_sessions.py

@@ -9,6 +9,7 @@
 from ckan.lib.base import render as pylons_render
 from ckan.tests.helpers import body_contains
 
+
 @pytest.mark.ckan_config(u"ckan.plugins", u"test_flash_plugin")
 class TestWithFlashPlugin:
     # @pytest.mark.skipif(six.PY3, reason=u"There is no pylons app in Py3")
@@ -31,7 +32,7 @@ def test_flash_populated_in_pylons_action_redirect_to_flask(self, app):
         """
         res = app.get(u"/pylons_add_flash_message_redirect_view").follow()
 
-        assert u"This is a success message populated by Pylons" in res.body
+        assert body_contains(res, u"This is a success message populated by Pylons")
 
     @pytest.mark.skipif(six.PY3, reason=u"There is no pylons app in Py3")
     def test_flash_populated_in_flask_view_redirect_to_pylons(self, app):
@@ -40,7 +41,7 @@ def test_flash_populated_in_flask_view_redirect_to_pylons(self, app):
         """
         res = app.get(u"/flask_add_flash_message_redirect_pylons").follow()
 
-        assert u"This is a success message populated by Flask" in res.body
+        assert body_contains(res, u"This is a success message populated by Flask")
 
 
 class FlashMessagePlugin(p.SingletonPlugin):

ckan/tests/controllers/test_feed.py

@@ -30,23 +30,23 @@ def test_general_atom_feed_works(self, app):
         offset = url_for(u"feeds.general")
         res = app.get(offset)
 
-        assert u"<title>{0}</title>".format(dataset["title"]) in res.body
+        assert helpers.body_contains(res, u"<title>{0}</title>".format(dataset["title"]))
 
     def test_group_atom_feed_works(self, app):
         group = factories.Group()
         dataset = factories.Dataset(groups=[{"id": group["id"]}])
         offset = url_for(u"feeds.group", id=group["name"])
         res = app.get(offset)
 
-        assert u"<title>{0}</title>".format(dataset["title"]) in res.body
+        assert helpers.body_contains(res, u"<title>{0}</title>".format(dataset["title"]))
 
     def test_organization_atom_feed_works(self, app):
         group = factories.Organization()
         dataset = factories.Dataset(owner_org=group["id"])
         offset = url_for(u"feeds.organization", id=group["name"])
         res = app.get(offset)
 
-        assert u"<title>{0}</title>".format(dataset["title"]) in res.body
+        assert helpers.body_contains(res, u"<title>{0}</title>".format(dataset["title"]))
 
     def test_custom_atom_feed_works(self, app):
         dataset1 = factories.Dataset(
@@ -63,9 +63,9 @@ def test_custom_atom_feed_works(self, app):
 
         res = app.get(offset, params=params)
 
-        assert u"<title>{0}</title>".format(dataset1["title"]) in res.body
+        assert helpers.body_contains(res, u"<title>{0}</title>".format(dataset1["title"]))
 
-        assert u'<title">{0}</title>'.format(dataset2["title"]) not in res.body
+        assert not helpers.body_contains(u'<title">{0}</title>'.format(dataset2["title"]))
 
 
 @pytest.mark.ckan_config("ckan.plugins", "test_feed_plugin")
@@ -75,9 +75,10 @@ def test_custom_class_used(self, app):
         offset = url_for(u"feeds.general")
         res = app.get(offset)
 
-        assert (
-            'xmlns:georss="http://www.georss.org/georss"' in res.body
-        ), res.body
+        assert helpers.body_contains(
+            res,
+            'xmlns:georss="http://www.georss.org/georss"'
+        )
 
     def test_additional_fields_added(self, app):
         metadata = {
@@ -93,10 +94,10 @@ def test_additional_fields_added(self, app):
         offset = url_for(u"feeds.general")
         res = app.get(offset)
 
-        assert (
+        assert helpers.body_contains(
+            res,
             "<georss:box>-2373790.000000 2937940.000000 -1681290.000000 3567770.000000</georss:box>"
-            in res.body
-        ), res.body
+        )
 
 
 class MockFeedPlugin(plugins.SingletonPlugin):

ckan/tests/controllers/test_package.py

@@ -1274,7 +1274,8 @@ def test_search_basic(self, app):
         offset = url_for("dataset.search")
         page = app.get(offset)
 
-        assert dataset1["name"] in page.body.decode("utf8")
+        assert helpers.body_contains(page, dataset1["name"])
+
 
     def test_search_language_toggle(self, app):
         dataset1 = factories.Dataset()
@@ -1283,8 +1284,8 @@ def test_search_language_toggle(self, app):
             offset = url_for("dataset.search", q=dataset1["name"])
         page = app.get(offset)
 
-        assert dataset1["name"] in page.body.decode("utf8")
-        assert ("q=" + dataset1["name"]) in page.body.decode("utf8")
+        assert helpers.body_contains(page, dataset1["name"])
+        assert helpers.body_contains(page, "q=" + dataset1["name"])
 
     def test_search_sort_by_blank(self, app):
         factories.Dataset()

ckan/tests/legacy/functional/api/test_api.py

@@ -6,7 +6,7 @@
     ControllerTestCase,
     Api3TestCase,
 )
-
+from ckan.tests.helpers import body_contains
 
 class ApiTestCase(ApiTestCase, ControllerTestCase):
     def test_get_api(self, app):
@@ -44,7 +44,8 @@ def test_sideeffect_action_is_not_get_able(self, app):
         res = app.get(
             offset, params=data_dict, status=[400], expect_errors=True
         )
-        assert (
+        assert body_contains(
+            res,
             "Bad request - JSON Error: Invalid request."
-            " Please use POST method for your request" in res.body
+            " Please use POST method for your request"
         )

ckan/tests/legacy/functional/api/test_package_search.py

@@ -9,7 +9,7 @@
 from ckan.tests.legacy.functional.api.base import ApiTestCase, Api3TestCase
 from ckan.tests.legacy import setup_test_search_index, CreateTestData
 from ckan.tests.legacy import TestController as ControllerTestCase
-
+from ckan.tests.helpers import body_contains
 
 class PackageSearchApiTestCase(ApiTestCase, ControllerTestCase):
     @pytest.fixture(autouse=True)
@@ -99,12 +99,12 @@ def test_12_all_packages_no_q(self):
     def test_12_filter_by_openness(self):
         offset = self.base_url + "?filter_by_openness=1"
         res = self.app.get(offset, status=400)  # feature dropped in #1360
-        assert "'filter_by_openness'" in res.body, res.body
+        assert body_contains(res, "'filter_by_openness'")
 
     def test_12_filter_by_downloadable(self):
         offset = self.base_url + "?filter_by_downloadable=1"
         res = self.app.get(offset, status=400)  # feature dropped in #1360
-        assert "'filter_by_downloadable'" in res.body, res.body
+        assert body_contains(res, "'filter_by_downloadable'")
 
 
 class LegacyOptionsTestCase(ApiTestCase, ControllerTestCase):
@@ -140,8 +140,8 @@ def test_08_all_fields_syntax_error(self):
             self.base_url + "?all_fields=should_be_boolean"
         )  # invalid all_fields value
         res = self.app.get(offset, status=400)
-        assert "boolean" in res.body
-        assert "all_fields" in res.body
+        assert body_contains(res, "boolean")
+        assert body_contains(res, "all_fields")
         self.assert_json_response(res, "boolean")
 
     def test_09_just_tags(self):
@@ -222,14 +222,15 @@ def test_11_pagination_validation_error(self):
             + "?fl=*&q=tags:russian&start=should_be_integer&rows=1&sort=name asc"
         )  # invalid offset value
         res = self.app.get(offset, status=409)
-        assert "Validation Error" in res.body
+        assert body_contains(res, "Validation Error")
 
     def test_12_v1_or_v2_syntax(self):
         offset = self.base_url + "?all_fields=1"
         res = self.app.get(offset, status=400)
-        assert (
-            "Invalid search parameters: ['all_fields']" in res.body
-        ), res.body
+        assert body_contains(
+            res,
+            "Invalid search parameters: ['all_fields']"
+        )
 
     def test_13_just_groups(self):
         offset = self.base_url + "?q=groups:roger"

ckan/tests/legacy/functional/test_package.py

@@ -14,6 +14,7 @@
 from ckan.logic.action import get, update
 from ckan import plugins
 from ckan.lib.search.common import SolrSettings
+from ckan.tests.helpers import body_contains
 
 existing_extra_html = (
     '<label class="field_opt" for="Package-%(package_id)s-extras-%(key)s">%(capitalized_key)s</label>',
@@ -489,7 +490,7 @@ def test_change_locale(self, env_user, app):
         res = app.get(offset, extra_environ=env_user)
 
         res = app.get("/de/dataset/new", extra_environ=env_user)
-        assert "Datensatz" in res.body, res.body
+        assert body_contains(res, "Datensatz")
 
 
 class TestNonActivePackages: