From d1a09457dcb47f38d7861e017c7c5f7c91273ff2 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 24 Dec 2019 14:43:02 +0200 Subject: [PATCH] Handle utf-8 decoding --- ckan/config/middleware/common_middleware.py | 3 +- ckan/controllers/feed.py | 5 ++- ckan/i18n/check_po_files.py | 1 - ckan/lib/app_globals.py | 2 +- ckan/lib/lazyjson.py | 4 +- ckan/lib/navl/dictization_functions.py | 2 +- ckan/lib/navl/validators.py | 2 +- ckan/lib/webassets_tools.py | 1 + ckan/logic/action/create.py | 1 - ckan/logic/action/delete.py | 3 +- ckan/logic/validators.py | 1 - ckan/model/license.py | 11 +++-- ckan/model/user.py | 2 +- ckan/tests/config/test_sessions.py | 5 ++- ckan/tests/controllers/test_feed.py | 23 ++++++----- ckan/tests/controllers/test_package.py | 7 ++-- ckan/tests/legacy/functional/api/test_api.py | 7 ++-- .../functional/api/test_package_search.py | 19 +++++---- ckan/tests/legacy/functional/test_package.py | 3 +- ckan/tests/legacy/html_check.py | 8 ++-- ckan/tests/legacy/logic/test_action.py | 41 +++++++++++-------- ckan/tests/legacy/test_coding_standards.py | 1 - ckan/tests/lib/test_cli.py | 4 ++ ckan/tests/lib/test_helpers.py | 4 +- ckan/tests/model/test_user.py | 4 +- ckan/tests/test_common.py | 1 + ckan/views/__init__.py | 4 +- ckan/views/api.py | 8 ++-- ckanext/datastore/backend/postgres.py | 6 +-- ckanext/datastore/logic/action.py | 3 +- ckanext/datastore/tests/test_dump.py | 13 +++--- .../tests/test_routes.py | 5 +-- .../tests/test_controllers.py | 1 + ckanext/example_ivalidators/plugin.py | 5 ++- 34 files changed, 119 insertions(+), 91 deletions(-) diff --git a/ckan/config/middleware/common_middleware.py b/ckan/config/middleware/common_middleware.py index 8b66795cf13..fcee0bff501 100644 --- a/ckan/config/middleware/common_middleware.py +++ b/ckan/config/middleware/common_middleware.py @@ -4,6 +4,7 @@ import hashlib import cgi +import six from six.moves.urllib.parse import unquote import sqlalchemy as sa @@ -74,7 +75,7 @@ def __call__(self, environ, start_response): data = {} for part in parts: k, v = part.split('=') - data[k] = unquote(v).decode("utf8") + data[k] = six.ensure_text(unquote(v)) start_response('200 OK', [('Content-Type', 'text/html')]) # we want a unique anonomized key for each user so that we do # not count multiple clicks from the same user. diff --git a/ckan/controllers/feed.py b/ckan/controllers/feed.py index 793bb750914..cb72eaf12d2 100644 --- a/ckan/controllers/feed.py +++ b/ckan/controllers/feed.py @@ -23,6 +23,7 @@ # TODO fix imports import logging +import six from six import text_type from six.moves.urllib.parse import urlparse @@ -550,11 +551,11 @@ def add_item_elements(self, handler, item): if(item['updated']): handler.addQuickElement(u'updated', - dfunc(item['updated']).decode('utf-8')) + six.ensure_text(dfunc(item['updated']))) if(item['published']): handler.addQuickElement(u'published', - dfunc(item['published']).decode('utf-8')) + six.ensure_text(dfunc(item['published']))) def add_root_elements(self, handler): """ diff --git a/ckan/i18n/check_po_files.py b/ckan/i18n/check_po_files.py index de447232cd9..7eeb9d8cf2c 100755 --- a/ckan/i18n/check_po_files.py +++ b/ckan/i18n/check_po_files.py @@ -76,7 +76,6 @@ def check_po_files(paths): msgid, msgstr.encode('ascii', 'replace'))) - def check_po_file(path): errors = [] diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index bdb7ab65428..b4ce6fbcf4c 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -139,7 +139,7 @@ def get_config_value(key, default=''): # sort encodeings if needed if isinstance(config_value, str) and six.PY2: try: - config_value = config_value.decode('utf-8') + config_value = six.ensure_text(config_value) except UnicodeDecodeError: config_value = config_value.decode('latin-1') # we want to store the config the first time we get here so we can diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index a96603b30b2..a3a2965d379 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -23,7 +23,7 @@ def _loads(self): self._json_string = None return self._json_dict - def __nonzero__(self): + def __bool__(self): return True def __repr__(self): @@ -50,7 +50,7 @@ def method(self, *args, **kwargs): for fn in [u'__contains__', u'__delitem__', u'__eq__', u'__ge__', u'__getitem__', u'__gt__', u'__iter__', u'__le__', u'__len__', u'__lt__', u'__ne__', u'__setitem__', u'clear', u'copy', - u'fromkeys', u'get', u'items', u'iteritems', + u'fromkeys', u'get', u'items', u'iteritems', u'iterkeys', u'itervalues', u'keys', u'pop', u'popitem', u'setdefault', u'update', u'values']: setattr(LazyJSONObject, fn, _loads_method(fn)) diff --git a/ckan/lib/navl/dictization_functions.py b/ckan/lib/navl/dictization_functions.py index e9a79b1d927..75d457ced13 100644 --- a/ckan/lib/navl/dictization_functions.py +++ b/ckan/lib/navl/dictization_functions.py @@ -33,7 +33,7 @@ def __oct__(self): def __hex__(self): raise Invalid(_('Missing value')) - def __nonzero__(self): + def __bool__(self): return False diff --git a/ckan/lib/navl/validators.py b/ckan/lib/navl/validators.py index 18c5e8386cb..a37a85639bc 100644 --- a/ckan/lib/navl/validators.py +++ b/ckan/lib/navl/validators.py @@ -163,7 +163,7 @@ def unicode_safe(value): # bytes only arrive when core ckan or plugins call # actions from Python code try: - return value.decode(u'utf8') + return six.ensure_text(value) except UnicodeDecodeError: return value.decode(u'cp1252') try: diff --git a/ckan/lib/webassets_tools.py b/ckan/lib/webassets_tools.py index e55865f113f..b1626063e76 100644 --- a/ckan/lib/webassets_tools.py +++ b/ckan/lib/webassets_tools.py @@ -17,6 +17,7 @@ yaml.warnings({'YAMLLoadWarning': False}) + def create_library(name, path): """Create WebAssets library(set of Bundles). """ diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 98b03373524..2c0038f3f6e 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -941,7 +941,6 @@ def user_create(context, data_dict): session = context['session'] _check_access('user_create', context, data_dict) - data, errors = _validate(data_dict, schema, context) if errors: diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index bcd65b75e79..ede9b2e6a3d 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -5,6 +5,7 @@ import logging import sqlalchemy as sqla +import six import ckan.lib.jobs as jobs import ckan.logic @@ -380,7 +381,7 @@ def _group_or_org_delete(context, data_dict, is_org=False): activity_type = 'deleted group' activity_dict = { - 'user_id': model.User.by_name(user.decode('utf8')).id, + 'user_id': model.User.by_name(six.ensure_text(user)).id, 'object_id': group.id, 'activity_type': activity_type, 'data': { diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index a2aadadf844..e18771b7f50 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -607,7 +607,6 @@ def user_passwords_match(key, data, errors, context): def user_password_not_empty(key, data, errors, context): '''Only check if password is present if the user is created via action API. If not, user_both_passwords_entered will handle the validation''' - # sysadmin may provide password_hash directly for importing users if (data.get(('password_hash',), missing) is not missing and authz.is_sysadmin(context.get('user'))): diff --git a/ckan/model/license.py b/ckan/model/license.py index d12bf0302c8..7bfaf2d1b8c 100644 --- a/ckan/model/license.py +++ b/ckan/model/license.py @@ -41,7 +41,7 @@ def __init__(self, data): if six.PY2: # Convert str to unicode # (keeps Pylons and SQLAlchemy happy). - value = value.decode('utf8') + value = six.ensure_text(value) self._data[key] = value def __getattr__(self, name): @@ -53,7 +53,12 @@ def __getattr__(self, name): log.warn('license.is_osi_compliant is deprecated - use ' 'osd_conformance instead.') return self._data['osd_conformance'] == 'approved' - return self._data[name] + try: + return self._data[name] + except KeyError as e: + # Python3 strictly requires `AttributeError` for correct + # behavior of `hasattr` + raise AttributeError(*e.args) @maintain.deprecated("License.__getitem__() is deprecated and will be " "removed in a future version of CKAN. Instead, " @@ -217,7 +222,7 @@ def __getitem__(self, key): else: return value else: - raise KeyError() + raise KeyError(key) def copy(self): ''' create a dict of the license used by the licenses api ''' diff --git a/ckan/model/user.py b/ckan/model/user.py index 63c2952ae75..26e90513e44 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -102,7 +102,7 @@ def _set_password(self, password): hashed_password = pbkdf2_sha512.encrypt(password) if not isinstance(hashed_password, text_type): - hashed_password = hashed_password.decode('utf-8') + hashed_password = six.ensure_text(hashed_password) self._password = hashed_password def _get_password(self): diff --git a/ckan/tests/config/test_sessions.py b/ckan/tests/config/test_sessions.py index fcb18a304d5..c9d4b142055 100644 --- a/ckan/tests/config/test_sessions.py +++ b/ckan/tests/config/test_sessions.py @@ -9,6 +9,7 @@ from ckan.lib.base import render as pylons_render from ckan.tests.helpers import body_contains + @pytest.mark.ckan_config(u"ckan.plugins", u"test_flash_plugin") class TestWithFlashPlugin: # @pytest.mark.skipif(six.PY3, reason=u"There is no pylons app in Py3") @@ -31,7 +32,7 @@ def test_flash_populated_in_pylons_action_redirect_to_flask(self, app): """ res = app.get(u"/pylons_add_flash_message_redirect_view").follow() - assert u"This is a success message populated by Pylons" in res.body + assert body_contains(res, u"This is a success message populated by Pylons") @pytest.mark.skipif(six.PY3, reason=u"There is no pylons app in Py3") def test_flash_populated_in_flask_view_redirect_to_pylons(self, app): @@ -40,7 +41,7 @@ def test_flash_populated_in_flask_view_redirect_to_pylons(self, app): """ res = app.get(u"/flask_add_flash_message_redirect_pylons").follow() - assert u"This is a success message populated by Flask" in res.body + assert body_contains(res, u"This is a success message populated by Flask") class FlashMessagePlugin(p.SingletonPlugin): diff --git a/ckan/tests/controllers/test_feed.py b/ckan/tests/controllers/test_feed.py index cd738911f2a..cba418c04e4 100644 --- a/ckan/tests/controllers/test_feed.py +++ b/ckan/tests/controllers/test_feed.py @@ -30,7 +30,7 @@ def test_general_atom_feed_works(self, app): offset = url_for(u"feeds.general") res = app.get(offset) - assert u"{0}".format(dataset["title"]) in res.body + assert helpers.body_contains(res, u"{0}".format(dataset["title"])) def test_group_atom_feed_works(self, app): group = factories.Group() @@ -38,7 +38,7 @@ def test_group_atom_feed_works(self, app): offset = url_for(u"feeds.group", id=group["name"]) res = app.get(offset) - assert u"{0}".format(dataset["title"]) in res.body + assert helpers.body_contains(res, u"{0}".format(dataset["title"])) def test_organization_atom_feed_works(self, app): group = factories.Organization() @@ -46,7 +46,7 @@ def test_organization_atom_feed_works(self, app): offset = url_for(u"feeds.organization", id=group["name"]) res = app.get(offset) - assert u"{0}".format(dataset["title"]) in res.body + assert helpers.body_contains(res, u"{0}".format(dataset["title"])) def test_custom_atom_feed_works(self, app): dataset1 = factories.Dataset( @@ -63,9 +63,9 @@ def test_custom_atom_feed_works(self, app): res = app.get(offset, params=params) - assert u"{0}".format(dataset1["title"]) in res.body + assert helpers.body_contains(res, u"{0}".format(dataset1["title"])) - assert u'{0}'.format(dataset2["title"]) not in res.body + assert not helpers.body_contains(u'{0}'.format(dataset2["title"])) @pytest.mark.ckan_config("ckan.plugins", "test_feed_plugin") @@ -75,9 +75,10 @@ def test_custom_class_used(self, app): offset = url_for(u"feeds.general") res = app.get(offset) - assert ( - 'xmlns:georss="http://www.georss.org/georss"' in res.body - ), res.body + assert helpers.body_contains( + res, + 'xmlns:georss="http://www.georss.org/georss"' + ) def test_additional_fields_added(self, app): metadata = { @@ -93,10 +94,10 @@ def test_additional_fields_added(self, app): offset = url_for(u"feeds.general") res = app.get(offset) - assert ( + assert helpers.body_contains( + res, "-2373790.000000 2937940.000000 -1681290.000000 3567770.000000" - in res.body - ), res.body + ) class MockFeedPlugin(plugins.SingletonPlugin): diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index 58cc09f9bd0..2ed44bffeb2 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -1274,7 +1274,8 @@ def test_search_basic(self, app): offset = url_for("dataset.search") page = app.get(offset) - assert dataset1["name"] in page.body.decode("utf8") + assert helpers.body_contains(page, dataset1["name"]) + def test_search_language_toggle(self, app): dataset1 = factories.Dataset() @@ -1283,8 +1284,8 @@ def test_search_language_toggle(self, app): offset = url_for("dataset.search", q=dataset1["name"]) page = app.get(offset) - assert dataset1["name"] in page.body.decode("utf8") - assert ("q=" + dataset1["name"]) in page.body.decode("utf8") + assert helpers.body_contains(page, dataset1["name"]) + assert helpers.body_contains(page, "q=" + dataset1["name"]) def test_search_sort_by_blank(self, app): factories.Dataset() diff --git a/ckan/tests/legacy/functional/api/test_api.py b/ckan/tests/legacy/functional/api/test_api.py index d0e9c310bb8..918434dcdbc 100644 --- a/ckan/tests/legacy/functional/api/test_api.py +++ b/ckan/tests/legacy/functional/api/test_api.py @@ -6,7 +6,7 @@ ControllerTestCase, Api3TestCase, ) - +from ckan.tests.helpers import body_contains class ApiTestCase(ApiTestCase, ControllerTestCase): def test_get_api(self, app): @@ -44,7 +44,8 @@ def test_sideeffect_action_is_not_get_able(self, app): res = app.get( offset, params=data_dict, status=[400], expect_errors=True ) - assert ( + assert body_contains( + res, "Bad request - JSON Error: Invalid request." - " Please use POST method for your request" in res.body + " Please use POST method for your request" ) diff --git a/ckan/tests/legacy/functional/api/test_package_search.py b/ckan/tests/legacy/functional/api/test_package_search.py index 2d7ecb94afd..9d640d280dd 100644 --- a/ckan/tests/legacy/functional/api/test_package_search.py +++ b/ckan/tests/legacy/functional/api/test_package_search.py @@ -9,7 +9,7 @@ from ckan.tests.legacy.functional.api.base import ApiTestCase, Api3TestCase from ckan.tests.legacy import setup_test_search_index, CreateTestData from ckan.tests.legacy import TestController as ControllerTestCase - +from ckan.tests.helpers import body_contains class PackageSearchApiTestCase(ApiTestCase, ControllerTestCase): @pytest.fixture(autouse=True) @@ -99,12 +99,12 @@ def test_12_all_packages_no_q(self): def test_12_filter_by_openness(self): offset = self.base_url + "?filter_by_openness=1" res = self.app.get(offset, status=400) # feature dropped in #1360 - assert "'filter_by_openness'" in res.body, res.body + assert body_contains(res, "'filter_by_openness'") def test_12_filter_by_downloadable(self): offset = self.base_url + "?filter_by_downloadable=1" res = self.app.get(offset, status=400) # feature dropped in #1360 - assert "'filter_by_downloadable'" in res.body, res.body + assert body_contains(res, "'filter_by_downloadable'") class LegacyOptionsTestCase(ApiTestCase, ControllerTestCase): @@ -140,8 +140,8 @@ def test_08_all_fields_syntax_error(self): self.base_url + "?all_fields=should_be_boolean" ) # invalid all_fields value res = self.app.get(offset, status=400) - assert "boolean" in res.body - assert "all_fields" in res.body + assert body_contains(res, "boolean") + assert body_contains(res, "all_fields") self.assert_json_response(res, "boolean") def test_09_just_tags(self): @@ -222,14 +222,15 @@ def test_11_pagination_validation_error(self): + "?fl=*&q=tags:russian&start=should_be_integer&rows=1&sort=name asc" ) # invalid offset value res = self.app.get(offset, status=409) - assert "Validation Error" in res.body + assert body_contains(res, "Validation Error") def test_12_v1_or_v2_syntax(self): offset = self.base_url + "?all_fields=1" res = self.app.get(offset, status=400) - assert ( - "Invalid search parameters: ['all_fields']" in res.body - ), res.body + assert body_contains( + res, + "Invalid search parameters: ['all_fields']" + ) def test_13_just_groups(self): offset = self.base_url + "?q=groups:roger" diff --git a/ckan/tests/legacy/functional/test_package.py b/ckan/tests/legacy/functional/test_package.py index 7c1465999d5..f68c5a12e9d 100644 --- a/ckan/tests/legacy/functional/test_package.py +++ b/ckan/tests/legacy/functional/test_package.py @@ -14,6 +14,7 @@ from ckan.logic.action import get, update from ckan import plugins from ckan.lib.search.common import SolrSettings +from ckan.tests.helpers import body_contains existing_extra_html = ( '', @@ -489,7 +490,7 @@ def test_change_locale(self, env_user, app): res = app.get(offset, extra_environ=env_user) res = app.get("/de/dataset/new", extra_environ=env_user) - assert "Datensatz" in res.body, res.body + assert body_contains(res, "Datensatz") class TestNonActivePackages: diff --git a/ckan/tests/legacy/html_check.py b/ckan/tests/legacy/html_check.py index c5b93ce1536..d8cb43def9e 100644 --- a/ckan/tests/legacy/html_check.py +++ b/ckan/tests/legacy/html_check.py @@ -2,7 +2,7 @@ import re from html.parser import HTMLParser - +import six from six import string_types, text_type import webtest @@ -34,7 +34,7 @@ def sidebar(self, html): def strip_tags(self, res): """Call strip_tags on a TestResponse object to strip any and all HTML and normalise whitespace.""" if not isinstance(res, string_types): - res = res.body.decode("utf-8") + res = six.ensure_text(res.body) return Stripper().strip(res) def check_named_element(self, html, tag_name, *html_to_find): @@ -71,9 +71,9 @@ def _get_html_from_res(self, html): if isinstance(html, text_type): html_str = html elif isinstance(html, str): - html_str = html.decode("utf8") + html_str = six.ensure_text(html) elif isinstance(html, webtest.app.TestResponse): - html_str = html.body.decode("utf-8") + html_str = six.ensure_text(html.body) else: raise TypeError return html_str # always unicode diff --git a/ckan/tests/legacy/logic/test_action.py b/ckan/tests/legacy/logic/test_action.py index 6816759da87..9e21a801ad2 100644 --- a/ckan/tests/legacy/logic/test_action.py +++ b/ckan/tests/legacy/logic/test_action.py @@ -3,6 +3,7 @@ import json from ckan.common import config import pytest +import six import ckan from ckan.lib.create_test_data import CreateTestData from ckan.lib.dictization.model_dictize import resource_dictize @@ -12,7 +13,7 @@ from ckan.logic import get_action, NotAuthorized from ckan.logic.action import get_domain_object from ckan.tests.legacy import call_action_api - +from ckan.tests.helpers import body_contains import ckan.tests.factories as factories from ckan.plugins import SingletonPlugin, implements, IPackageController @@ -223,12 +224,12 @@ def test_01_package_list(self, app): anna_id = model.Package.by_name(u"annakarenina").id resource = {"package_id": anna_id, "url": "http://new_url"} - api_key = model.User.get("testsysadmin").apikey.encode("utf8") + api_key = six.ensure_text(model.User.get("testsysadmin").apikey) postparams = "%s=1" % json.dumps(resource) res = app.post( "/api/action/resource_create", params=postparams, - extra_environ={"Authorization": api_key}, + extra_environ={"Authorization": str(api_key)}, ) resource = json.loads(res.body)["result"] @@ -243,13 +244,13 @@ def test_01_package_list(self, app): "url": "new_url", "created": "bad_date", } - api_key = model.User.get("testsysadmin").apikey.encode("utf8") + api_key = six.ensure_text(model.User.get("testsysadmin").apikey) postparams = "%s=1" % json.dumps(resource) res = app.post( "/api/action/resource_create", params=postparams, - extra_environ={"Authorization": api_key}, + extra_environ={"Authorization": str(api_key)}, status=StatusCodes.STATUS_409_CONFLICT, ) @@ -765,19 +766,21 @@ def test_01_package_list(self, app): res = app.post( "/api/action/package_list", params=postparams, status=400 ) - assert ( - "Bad request - JSON Error: Request data JSON decoded to 'not a dict' but it needs to be a dictionary." - in res.body - ), res.body + + assert body_contains( + res, + "Bad request - JSON Error: Request data JSON decoded to " + "'not a dict' but it needs to be a dictionary." + ) # def test_31_bad_request_format_not_json(self): postparams = "=1" res = app.post( "/api/action/package_list", params=postparams, status=400 ) - assert ( - "Bad request - JSON Error: Error decoding JSON data." in res.body - ), res.body + assert body_contains( + res, "Bad request - JSON Error: Error decoding JSON data." + ) # def test_32_get_domain_object(self): anna = model.Package.by_name(u"annakarenina") @@ -998,7 +1001,7 @@ def test_1_update_single(self, app): assert json.loads(res.body)["success"] # sort the result since the order is not important and is implementation # dependent - assert sorted(json.loads(res.body)["result"]) == sorted( + assert sorted(json.loads(res.body)["result"], key=dict.items) == sorted( [ { u"lang_code": u"fr", @@ -1010,7 +1013,7 @@ def test_1_update_single(self, app): u"term": u"moo", u"term_translation": u"moomoo", }, - ] + ], key=dict.items ), json.loads(res.body) def test_2_update_many(self, app): @@ -1057,7 +1060,7 @@ def test_2_update_many(self, app): # sort the result since the order is not important and is implementation # dependent - assert sorted(json.loads(res.body)["result"]) == sorted( + assert sorted(json.loads(res.body)["result"], key=dict.items) == sorted( [ { u"lang_code": u"fr", @@ -1069,7 +1072,7 @@ def test_2_update_many(self, app): u"term": u"many", u"term_translation": u"manymoomoo", }, - ] + ], key=dict.items ), json.loads(res.body) @@ -1187,10 +1190,12 @@ def test_before_index(self, app): def test_before_view(self, app): res = app.get("/dataset/annakarenina") - assert "string_not_found_in_rest_of_template" in res.body + assert body_contains(res, "string_not_found_in_rest_of_template") res = app.get("/dataset?q=") - assert res.body.count("string_not_found_in_rest_of_template") == 2 + assert six.ensure_str(res.body).count( + "string_not_found_in_rest_of_template" + ) == 2 class TestBulkActions(object): diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py index 5ed3c761f0a..fde261ea96b 100644 --- a/ckan/tests/legacy/test_coding_standards.py +++ b/ckan/tests/legacy/test_coding_standards.py @@ -375,7 +375,6 @@ class TestPep8(object): "ckan/poo.py", "ckan/rating.py", "ckan/templates_legacy/home/__init__.py", - "ckan/tests/legacy/__init__.py", "ckan/tests/legacy/ckantestplugin/ckantestplugin/__init__.py", "ckan/tests/legacy/ckantestplugin/setup.py", "ckan/tests/legacy/functional/api/base.py", diff --git a/ckan/tests/lib/test_cli.py b/ckan/tests/lib/test_cli.py index 629d341621e..cca9a065a7a 100644 --- a/ckan/tests/lib/test_cli.py +++ b/ckan/tests/lib/test_cli.py @@ -243,6 +243,7 @@ def test_show_missing_id(self): assert code != 0 assert stderr + @pytest.mark.skipif(six.PY3, reason=u"") class TestJobsCancel(helpers.RQTestBase): """ @@ -279,6 +280,7 @@ def test_cancel_missing_id(self): assert code != 0 assert stderr + @pytest.mark.skipif(six.PY3, reason=u"") class TestJobsClear(helpers.RQTestBase): """ @@ -316,6 +318,7 @@ def test_clear_specific_queues(self): all_jobs = self.all_jobs() assert set(all_jobs) == {job1, job2} + @pytest.mark.skipif(six.PY3, reason=u"") class TestJobsTest(helpers.RQTestBase): """ @@ -346,6 +349,7 @@ def test_test_specific_queues(self): u"q2", } + @pytest.mark.skipif(six.PY3, reason=u"") class TestJobsWorker(helpers.RQTestBase): """ diff --git a/ckan/tests/lib/test_helpers.py b/ckan/tests/lib/test_helpers.py index 12abbb3477b..8e9fe0d79f7 100644 --- a/ckan/tests/lib/test_helpers.py +++ b/ckan/tests/lib/test_helpers.py @@ -616,7 +616,7 @@ def test_helper_existing_helper_as_attribute(self, app): res = app.get("/helper_as_attribute") - assert "My lang is: en" in res.body + assert helpers.body_contains(res, "My lang is: en") def test_helper_existing_helper_as_item(self, app): """Calling an existing helper on `h` doesn't raises a @@ -624,7 +624,7 @@ def test_helper_existing_helper_as_item(self, app): res = app.get("/helper_as_item") - assert "My lang is: en" in res.body + assert helpers.body_contains(res, "My lang is: en") class TestHelpersPlugin(p.SingletonPlugin): diff --git a/ckan/tests/model/test_user.py b/ckan/tests/model/test_user.py index 838b3318983..c028b15b5e1 100644 --- a/ckan/tests/model/test_user.py +++ b/ckan/tests/model/test_user.py @@ -4,6 +4,8 @@ import hashlib import pytest +import six + from passlib.hash import pbkdf2_sha512 from six import text_type @@ -26,7 +28,7 @@ def _set_password(password): hashed_password = salt.hexdigest() + hash.hexdigest() if not isinstance(hashed_password, text_type): - hashed_password = hashed_password.decode("utf-8") + hashed_password = six.ensure_text(hashed_password) return hashed_password diff --git a/ckan/tests/test_common.py b/ckan/tests/test_common.py index 82f6c0cd31e..957a3dff76b 100644 --- a/ckan/tests/test_common.py +++ b/ckan/tests/test_common.py @@ -20,6 +20,7 @@ else: pylons = None + def test_del_works(): my_conf = CKANConfig() my_conf[u"test_key_1"] = u"Test value 1" diff --git a/ckan/views/__init__.py b/ckan/views/__init__.py index d5563b37392..5be7bb6421e 100644 --- a/ckan/views/__init__.py +++ b/ckan/views/__init__.py @@ -161,7 +161,7 @@ def _identify_user_default(): g.user = request.environ.get(u'REMOTE_USER', u'') if g.user: if six.PY2: - g.user = g.user.decode(u'utf8') + g.user = six.ensure_text(g.user) g.userobj = model.User.by_name(g.user) if g.userobj is None or not g.userobj.is_active(): @@ -200,7 +200,7 @@ def _get_user_for_apikey(): apikey = u'' if not apikey: return None - apikey = apikey.decode(u'utf8', u'ignore') + apikey = six.ensure_text(apikey, errors="ignore") log.debug(u'Received API Key: %s' % apikey) query = model.Session.query(model.User) user = query.filter_by(apikey=apikey).first() diff --git a/ckan/views/api.py b/ckan/views/api.py index e74e6a82846..e2b731aedfe 100644 --- a/ckan/views/api.py +++ b/ckan/views/api.py @@ -173,10 +173,12 @@ def mixed(multi_dict): request_data = {} if request.method in [u'POST', u'PUT'] and request.form: - if (len(request.form.values()) == 1 and - request.form.values()[0] in [u'1', u'']): + values = list(request.form.values()) + if (len(values) == 1 and + values[0] in [u'1', u'']): try: - request_data = json.loads(request.form.keys()[0]) + keys = list(request.form.keys()) + request_data = json.loads(keys[0]) except ValueError as e: raise ValueError( u'Error decoding JSON data. ' diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index 7b5716481ce..055a1a6c97e 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -285,7 +285,7 @@ def _get_fields(connection, resource_id): for field in all_fields.cursor.description: if not field[0].startswith('_'): fields.append({ - 'id': field[0].decode('utf-8'), + 'id': six.ensure_text(field[0]), 'type': _get_type(connection, field[1]) }) return fields @@ -1388,7 +1388,7 @@ def format_results(context, results, data_dict, rows_max): result_fields = [] for field in results.cursor.description: result_fields.append({ - 'id': field[0].decode('utf-8'), + 'id': six.ensure_text(field[0]), 'type': _get_type(context['connection'], field[1]) }) @@ -2085,5 +2085,5 @@ def _programming_error_summary(pe): ValidationError to send back to API users ''' # first line only, after the '(ProgrammingError)' text - message = pe.args[0].split('\n')[0].decode('utf8') + message = six.ensure_text(pe.args[0].split('\n')[0]) return message.split(u') ', 1)[-1] diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 1568a81d5b8..65cd4b82309 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -4,6 +4,7 @@ import json import sqlalchemy +import six from six import text_type import ckan.lib.search as search @@ -201,7 +202,7 @@ def datastore_run_triggers(context, data_dict): try: results = connection.execute(sql) except sqlalchemy.exc.DatabaseError as err: - message = err.args[0].split('\n')[0].decode('utf8') + message = six.ensure_text(err.args[0].split('\n')[0]) raise p.toolkit.ValidationError({ u'records': [message.split(u') ', 1)[-1]]}) return results.rowcount diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 4eb2021648c..f11638dab0f 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -3,6 +3,7 @@ import mock import json import pytest +import six import ckan.tests.helpers as helpers import ckan.tests.factories as factories @@ -60,7 +61,7 @@ def test_all_fields_types(self, app): helpers.call_action("datastore_create", **data) response = app.get("/datastore/dump/{0}".format(str(resource["id"]))) - content = response.body.decode("utf-8") + content = six.ensure_text(response.body) expected = ( u"_id,b\xfck,author,published" u",characters,random_letters,nested" ) @@ -108,7 +109,7 @@ def test_dump_limit(self, app): response = app.get( "/datastore/dump/{0}?limit=1".format(str(resource["id"])) ) - content = response.body.decode("utf-8") + content = six.ensure_text(response.body) expected_content = u"_id,book\r\n" u"1,annakarenina\n" assert content == expected_content @@ -150,7 +151,7 @@ def test_dump_q_and_fields(self, app): resource["id"] ) ) - content = response.body.decode("utf-8") + content = six.ensure_text(response.body) expected_content = u"nested,author\r\n" u'"{""a"": ""b""}",tolstoy\n' assert content == expected_content @@ -193,7 +194,7 @@ def test_dump_tsv(self, app): str(resource["id"]) ) ) - content = res.body.decode("utf-8") + content = six.ensure_text(res.body) expected_content = ( u"_id\tb\xfck\tauthor\tpublished\tcharacters\trandom_letters\t" @@ -242,7 +243,7 @@ def test_dump_json(self, app): str(resource["id"]) ) ) - content = res.body.decode("utf-8") + content = six.ensure_text(res.body) expected_content = ( u'{\n "fields": [{"type":"int","id":"_id"},{"type":"text",' u'"id":"b\xfck"},{"type":"text","id":"author"},{"type":"timestamp"' @@ -293,7 +294,7 @@ def test_dump_xml(self, app): str(resource["id"]) ) ) - content = res.body.decode("utf-8") + content = six.ensure_text(res.body) expected_content = ( u"\n" r'' diff --git a/ckanext/example_flask_iblueprint/tests/test_routes.py b/ckanext/example_flask_iblueprint/tests/test_routes.py index b01cd4b0628..344b15f38ff 100644 --- a/ckanext/example_flask_iblueprint/tests/test_routes.py +++ b/ckanext/example_flask_iblueprint/tests/test_routes.py @@ -29,9 +29,8 @@ def test_plugin_route_core_flask_override(self): u"""Test extension overrides flask core route.""" res = self.app.get(u"/") - assert ( + assert helpers.body_contains(res, u"Hello World, this is served from an extension, overriding the flask url." - in res.body ) def test_plugin_route_with_helper(self): @@ -41,7 +40,7 @@ def test_plugin_route_with_helper(self): """ res = self.app.get(u"/helper") - assert u"Hello World, helper here:

hi

" in res.body + assert helpers.body_contains(res, u"Hello World, helper here:

hi

") def test_plugin_route_with_non_existent_helper(self): u""" diff --git a/ckanext/example_igroupform/tests/test_controllers.py b/ckanext/example_igroupform/tests/test_controllers.py index e2870cf6459..bb430dd0b87 100644 --- a/ckanext/example_igroupform/tests/test_controllers.py +++ b/ckanext/example_igroupform/tests/test_controllers.py @@ -152,6 +152,7 @@ def test_custom_group_form(self, app): assert "My Custom Group Form!" in response + @pytest.mark.ckan_config("ckan.plugins", "example_igroupform_default_group_type") @pytest.mark.usefixtures("clean_db", "with_plugins") class TestGroupControllerNew_DefaultGroupType(object): diff --git a/ckanext/example_ivalidators/plugin.py b/ckanext/example_ivalidators/plugin.py index 2fe8e4e2b62..ebadeeed78f 100644 --- a/ckanext/example_ivalidators/plugin.py +++ b/ckanext/example_ivalidators/plugin.py @@ -1,5 +1,6 @@ # encoding: utf-8 +import six from six import text_type from ckan.plugins.toolkit import Invalid @@ -28,9 +29,9 @@ def negate(value): def unicode_please(value): - if isinstance(value, bytes): + if isinstance(value, six.binary_type): try: - return value.decode(u'utf8') + return six.ensure_text(value) except UnicodeDecodeError: return value.decode(u'cp1252') return text_type(value)