diff --git a/bin/running_stats.py b/bin/running_stats.py index a2612f35d0f..26c86684f2a 100644 --- a/bin/running_stats.py +++ b/bin/running_stats.py @@ -15,11 +15,11 @@ package.delete() package_stats.increment('deleted') else: - package_stats.increment('not deleted') + package_stats.increment('not deleted') print package_stats.report() > deleted: 30 > not deleted: 70 - + from running_stats import StatsList package_stats = StatsList() for package in packages: @@ -30,7 +30,7 @@ package_stats.add('not deleted' package.name) print package_stats.report() > deleted: 30 pollution-uk, flood-regions, river-quality, ... -> not deleted: 70 spending-bristol, ... +> not deleted: 70 spending-bristol, ... ''' @@ -40,11 +40,11 @@ class StatsCount(dict): # {category:count} _init_value = 0 report_value_limit = 150 - + def _init_category(self, category): - if not self.has_key(category): + if category not in self: self[category] = copy.deepcopy(self._init_value) - + def increment(self, category): self._init_category(category) self[category] += 1 diff --git a/ckan/i18n/check_po_files.py b/ckan/i18n/check_po_files.py index 6f652fa03cc..5cb2c90d0a2 100755 --- a/ckan/i18n/check_po_files.py +++ b/ckan/i18n/check_po_files.py @@ -9,10 +9,12 @@ ''' import polib import re -import paste.script.command import six +if six.PY2: + import paste.script.command + def simple_conv_specs(s): '''Return the simple Python string conversion specifiers in the string s. @@ -50,23 +52,28 @@ def replacement_fields(s): return sorted(repl_fields_re.findall(s)) -class CheckPoFiles(paste.script.command.Command): +if six.PY2: + class CheckPoFiles(paste.script.command.Command): + + usage = "[FILE] ..." + group_name = 'ckan' + summary = 'Check po files for common mistakes' + parser = paste.script.command.Command.standard_parser(verbose=True) + + def command(self): + check_po_files(self.args) - usage = "[FILE] ..." - group_name = 'ckan' - summary = 'Check po files for common mistakes' - parser = paste.script.command.Command.standard_parser(verbose=True) - def command(self): +def check_po_files(paths): + for path in paths: + print(u'Checking file {}'.format(path)) + errors = check_po_file(path) + if errors: + for msgid, msgstr in errors: + print("Format specifiers don't match:") + print(u' {0} -> {1}'.format( + msgid, msgstr.encode('ascii', 'replace'))) - for path in self.args: - print(u'Checking file {}'.format(path)) - errors = check_po_file(path) - if errors: - for msgid, msgstr in errors: - print("Format specifiers don't match:") - print(u' {0} -> {1}'.format( - msgid, msgstr.encode('ascii', 'replace'))) def check_po_file(path): diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 1d51c1e616b..3e161221ec8 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -157,7 +157,7 @@ def create_arbitrary(cls, package_dicts, relationships=[], for item in package_dicts: pkg_dict = {} for field in cls.pkg_core_fields: - if item.has_key(field): + if field in item: pkg_dict[field] = text_type(item[field]) if model.Package.by_name(pkg_dict['name']): log.warning('Cannot create package "%s" as it already exists.' % \ diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index ef07fcfa0e3..9c9c930deed 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -411,7 +411,7 @@ def tag_list_dictize(tag_list, context): # the same as its name, but the display_name might get changed later # (e.g. translated into another language by the multilingual # extension). - assert not dictized.has_key('display_name') + assert 'display_name' not in dictized dictized['display_name'] = dictized['name'] if context.get('for_view'): @@ -622,7 +622,7 @@ def make_api_2(package_id): def vocabulary_dictize(vocabulary, context, include_datasets=False): vocabulary_dict = d.table_dictize(vocabulary, context) - assert not vocabulary_dict.has_key('tags') + assert 'tags' not in vocabulary_dict vocabulary_dict['tags'] = [tag_dictize(tag, context, include_datasets) for tag in vocabulary.tags] diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py index 1bc3ea5e371..92193a17fe8 100644 --- a/ckan/lib/dictization/model_save.py +++ b/ckan/lib/dictization/model_save.py @@ -517,7 +517,7 @@ def activity_dict_save(activity_dict, context): user_id = activity_dict['user_id'] object_id = activity_dict['object_id'] activity_type = activity_dict['activity_type'] - if activity_dict.has_key('data'): + if 'data' in activity_dict: data = activity_dict['data'] else: data = None @@ -554,7 +554,7 @@ def vocabulary_dict_save(vocabulary_dict, context): vocabulary_obj = model.Vocabulary(vocabulary_name) session.add(vocabulary_obj) - if vocabulary_dict.has_key('tags'): + if 'tags' in vocabulary_dict: vocabulary_tag_list_save(vocabulary_dict['tags'], vocabulary_obj, context) @@ -567,10 +567,10 @@ def vocabulary_dict_update(vocabulary_dict, context): vocabulary_obj = model.vocabulary.Vocabulary.get(vocabulary_dict['id']) - if vocabulary_dict.has_key('name'): + if 'name' in vocabulary_dict: vocabulary_obj.name = vocabulary_dict['name'] - if vocabulary_dict.has_key('tags'): + if 'tags' in vocabulary_dict: vocabulary_tag_list_save(vocabulary_dict['tags'], vocabulary_obj, context) diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index a91ecfccbf7..a96603b30b2 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -50,7 +50,7 @@ def method(self, *args, **kwargs): for fn in [u'__contains__', u'__delitem__', u'__eq__', u'__ge__', u'__getitem__', u'__gt__', u'__iter__', u'__le__', u'__len__', u'__lt__', u'__ne__', u'__setitem__', u'clear', u'copy', - u'fromkeys', u'get', u'has_key', u'items', u'iteritems', + u'fromkeys', u'get', u'items', u'iteritems', u'iterkeys', u'itervalues', u'keys', u'pop', u'popitem', u'setdefault', u'update', u'values']: setattr(LazyJSONObject, fn, _loads_method(fn)) diff --git a/ckan/logic/action/__init__.py b/ckan/logic/action/__init__.py index 591f175ea89..54f251f85b6 100644 --- a/ckan/logic/action/__init__.py +++ b/ckan/logic/action/__init__.py @@ -21,9 +21,9 @@ def rename_keys(dict_, key_map, reverse=False, destructive=False): for key, mapping in key_map.items(): if reverse: key, mapping = (mapping, key) - if (not destructive) and new_dict.has_key(mapping): + if (not destructive) and mapping in new_dict: continue - if dict_.has_key(key): + if key in dict_: value = dict_[key] new_dict[mapping] = value del new_dict[key] diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 8870d6c7778..bcd65b75e79 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -594,7 +594,7 @@ def tag_delete(context, data_dict): ''' model = context['model'] - if not data_dict.has_key('id') or not data_dict['id']: + if 'id' not in data_dict or not data_dict['id']: raise ValidationError({'id': _('id not in data')}) tag_id_or_name = _get_or_bust(data_dict, 'id') @@ -614,7 +614,7 @@ def tag_delete(context, data_dict): def _unfollow(context, data_dict, schema, FollowerClass): model = context['model'] - if not context.has_key('user'): + if 'user' not in context: raise ckan.logic.NotAuthorized( _("You must be logged in to unfollow something.")) userobj = model.User.get(context['user']) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 4ee07af7106..f119e87c9bf 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -906,7 +906,7 @@ def vocabulary_update(context, data_dict): raise NotFound(_('Could not find vocabulary "%s"') % vocab_id) data_dict['id'] = vocab.id - if data_dict.has_key('name'): + if 'name' in data_dict: if data_dict['name'] == vocab.name: del data_dict['name'] diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 3f81daeaa10..a2aadadf844 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -308,7 +308,7 @@ def object_id_validator(key, activity_dict, errors, context): ''' activity_type = activity_dict[('activity_type',)] - if object_id_validators.has_key(activity_type): + if activity_type in object_id_validators: object_id = activity_dict[('object_id',)] return object_id_validators[activity_type](object_id, context) else: @@ -672,7 +672,7 @@ def tag_not_in_vocabulary(key, tag_dict, errors, context): tag_name = tag_dict[('name',)] if not tag_name: raise Invalid(_('No tag name')) - if tag_dict.has_key(('vocabulary_id',)): + if ('vocabulary_id',) in tag_dict: vocabulary_id = tag_dict[('vocabulary_id',)] else: vocabulary_id = None diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index 64148ad5ed4..b9ca4bf3ea4 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -397,9 +397,11 @@ def _add_resource(cls, path, name): absolute_path = os.path.join(this_dir, path) create_library(name, absolute_path) - # TODO: remove next two lines after dropping Fanstatic support - import ckan.lib.fanstatic_resources - ckan.lib.fanstatic_resources.create_library(name, absolute_path) + import six + if six.PY2: + # TODO: remove next two lines after dropping Fanstatic support + import ckan.lib.fanstatic_resources + ckan.lib.fanstatic_resources.create_library(name, absolute_path) @classmethod def _add_ckan_admin_tabs(cls, config, route_name, tab_label, diff --git a/ckan/templates/snippets/search_form.html b/ckan/templates/snippets/search_form.html index fed9bd48ddd..a7b1e07fefd 100644 --- a/ckan/templates/snippets/search_form.html +++ b/ckan/templates/snippets/search_form.html @@ -61,7 +61,7 @@

Error

{{ facets.titles.get(field) }}: {% for value in facets.fields[field] %} - {%- if facets.translated_fields and facets.translated_fields.has_key((field,value)) -%} + {%- if facets.translated_fields and (field,value) in facets.translated_fields -%} {{ facets.translated_fields[(field,value)] }} {%- else -%} {{ h.list_dict_filter(search_facets_items, 'name', 'display_name', value) }} diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index 28cbf97b206..e1c13863daf 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -4,6 +4,7 @@ import mock import pytest import wsgiref +import six from flask import Blueprint import ckan.lib.helpers as h @@ -13,8 +14,10 @@ from ckan.common import config, _ from ckan.config.middleware import AskAppDispatcherMiddleware from ckan.config.middleware.flask_app import CKANFlask -from ckan.config.middleware.pylons_app import CKANPylonsApp - +if six.PY2: + from ckan.config.middleware.pylons_app import CKANPylonsApp +else: + CKANPylonsApp = object _test_controller = u"ckan.tests.config.test_middleware:MockPylonsController" @@ -104,16 +107,17 @@ def flask_translated_view(): return _(u"Dataset") -class MockPylonsController(p.toolkit.BaseController): - def view(self): - return u"Hello World, this is served from a Pylons extension" +if six.PY2: + class MockPylonsController(p.toolkit.BaseController): + def view(self): + return u"Hello World, this is served from a Pylons extension" - def test_flask_url_for(self): - url = h.url_for(u"api.get_api", ver=3) - return u"This URL was generated by Flask: {0}".format(url) + def test_flask_url_for(self): + url = h.url_for(u"api.get_api", ver=3) + return u"This URL was generated by Flask: {0}".format(url) - def test_translation(self): - return _(u"Groups") + def test_translation(self): + return _(u"Groups") @pytest.fixture @@ -137,12 +141,13 @@ def test_view(): return app +@pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_flask_core_route_is_served_by_flask(patched_app): res = patched_app.get(u"/") - assert res.environ[u"ckan.app"] == u"flask_app" +@pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_flask_core_and_pylons_core_route_is_served_by_flask(patched_app): """ This should never happen in core, but just in case @@ -155,6 +160,7 @@ def test_flask_core_and_pylons_core_route_is_served_by_flask(patched_app): @pytest.mark.ckan_config(u"ckan.plugins", u"test_routing_plugin") class TestMiddlewareWithRoutingPlugin: + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_ask_around_pylons_extension_route_get_before_map( self, patched_app ): @@ -174,6 +180,7 @@ def test_ask_around_pylons_extension_route_get_before_map( (True, u"pylons_app", u"extension"), ] + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_ask_around_pylons_extension_route_post(self, patched_app): environ = { u"PATH_INFO": u"/from_pylons_extension_before_map_post_only", @@ -188,6 +195,7 @@ def test_ask_around_pylons_extension_route_post(self, patched_app): (True, u"pylons_app", u"extension"), ] + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_ask_around_pylons_extension_route_post_using_get( self, patched_app ): @@ -207,6 +215,7 @@ def test_ask_around_pylons_extension_route_post_using_get( (False, u"pylons_app"), ] + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_ask_around_pylons_extension_route_get_after_map( self, patched_app ): @@ -226,10 +235,12 @@ def test_ask_around_pylons_extension_route_get_after_map( (True, u"pylons_app", u"extension"), ] + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_flask_extension_route_is_served_by_flask(self, patched_app): res = patched_app.get(u"/simple_flask") assert res.environ[u"ckan.app"] == u"flask_app" + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_pylons_extension_route_is_served_by_pylons(self, patched_app): res = patched_app.get(u"/from_pylons_extension_before_map") @@ -251,7 +262,7 @@ def test_user_objects_in_g_normal_user(self, app): with app.flask_app.app_context(): app.get( u"/simple_flask", - extra_environ={u"REMOTE_USER": username.encode(u"ascii")}, + extra_environ={u"REMOTE_USER": username.encode(u"ascii") if six.PY2 else username}, ) assert flask.g.user == username assert flask.g.userobj == test_user_obj @@ -282,13 +293,14 @@ def test_user_objects_in_g_sysadmin(self, app): with app.flask_app.app_context(): app.get( u"/simple_flask", - extra_environ={u"REMOTE_USER": user[u"name"].encode(u"ascii")}, + extra_environ={u"REMOTE_USER": user[u"name"].encode(u"ascii") if six.PY2 else user[u"name"]}, ) assert flask.g.user == user[u"name"] assert flask.g.userobj == test_user_obj assert flask.g.author == user[u"name"] assert flask.g.remote_addr == u"Unknown IP Address" + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_user_objects_in_c_normal_user(self, app): """ A normal logged in user request will have expected user objects added @@ -299,7 +311,7 @@ def test_user_objects_in_c_normal_user(self, app): resp = app.get( u"/from_pylons_extension_before_map", - extra_environ={u"REMOTE_USER": username.encode(u"ascii")}, + extra_environ={u"REMOTE_USER": username.encode(u"ascii") if six.PY2 else username}, ) # tmpl_context available on response @@ -308,6 +320,7 @@ def test_user_objects_in_c_normal_user(self, app): assert resp.tmpl_context.author == username assert resp.tmpl_context.remote_addr == u"Unknown IP Address" + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_user_objects_in_c_anon_user(self, app): """An anon user request will have expected user objects added to request. @@ -324,6 +337,7 @@ def test_user_objects_in_c_anon_user(self, app): assert resp.tmpl_context.author == u"Unknown IP Address" assert resp.tmpl_context.remote_addr == u"Unknown IP Address" + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") @pytest.mark.usefixtures(u"clean_db") def test_user_objects_in_c_sysadmin(self, app): """A sysadmin user request will have expected user objects added to @@ -334,7 +348,7 @@ def test_user_objects_in_c_sysadmin(self, app): resp = app.get( u"/from_pylons_extension_before_map", - extra_environ={u"REMOTE_USER": username.encode(u"ascii")}, + extra_environ={u"REMOTE_USER": username.encode(u"ascii") if six.PY2 else username}, ) # tmpl_context available on response @@ -343,6 +357,7 @@ def test_user_objects_in_c_sysadmin(self, app): assert resp.tmpl_context.author == username assert resp.tmpl_context.remote_addr == u"Unknown IP Address" + @pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") @pytest.mark.ckan_config( u"ckan.use_pylons_response_cleanup_middleware", True ) @@ -380,6 +395,7 @@ def test_no_beaker_secret_crashes(make_app): make_app() +@pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") @pytest.mark.parametrize( u"rv,app_base", [ @@ -401,6 +417,7 @@ def test_can_handle_request_with_environ(monkeypatch, app, rv, app_base): assert handler.called_with(environ) +@pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_ask_around_is_called(monkeypatch, app): ask = mock.MagicMock() monkeypatch.setattr(AskAppDispatcherMiddleware, u"ask_around", ask) @@ -408,6 +425,7 @@ def test_ask_around_is_called(monkeypatch, app): assert ask.called +@pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_ask_around_is_called_with_args(monkeypatch, app): ckan_app = app.app @@ -423,6 +441,7 @@ def test_ask_around_is_called_with_args(monkeypatch, app): ask.assert_called_with(environ) +@pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_ask_around_flask_core_route_get(app): ckan_app = app.app @@ -434,6 +453,7 @@ def test_ask_around_flask_core_route_get(app): assert answers == [(True, u"flask_app", u"core"), (False, u"pylons_app")] +@pytest.mark.skipif(six.PY3, reason="Do not test AskAppDispatcherMiddleware in Py3") def test_ask_around_flask_core_route_post(app): ckan_app = app.app diff --git a/ckan/tests/config/test_sessions.py b/ckan/tests/config/test_sessions.py index 0902a148bb0..f6a36afaef3 100644 --- a/ckan/tests/config/test_sessions.py +++ b/ckan/tests/config/test_sessions.py @@ -1,12 +1,13 @@ # encoding: utf-8 import pytest +import six from flask import Blueprint, render_template import ckan.lib.helpers as h import ckan.plugins as p from ckan.lib.base import render as pylons_render - +from ckan.tests.helpers import body_contains @pytest.mark.ckan_config(u"ckan.plugins", u"test_flash_plugin") class TestWithFlashPlugin: @@ -17,8 +18,9 @@ def test_flash_populated_by_flask_redirect_to_flask(self, app): """ res = app.get(u"/flask_add_flash_message_redirect_to_flask").follow() - assert u"This is a success message populated by Flask" in res.body + assert body_contains(res, u"This is a success message populated by Flask") + @pytest.mark.skipif(six.PY3, reason=u"There is no pylons app in Py3") def test_flash_populated_in_pylons_action_redirect_to_flask(self, app): u""" Flash store is populated by pylons action is accessible by Flask view. @@ -27,6 +29,7 @@ def test_flash_populated_in_pylons_action_redirect_to_flask(self, app): assert u"This is a success message populated by Pylons" in res.body + @pytest.mark.skipif(six.PY3, reason=u"There is no pylons app in Py3") def test_flash_populated_in_flask_view_redirect_to_pylons(self, app): u""" Flash store is populated by flask view is accessible by pylons action. @@ -109,12 +112,13 @@ def before_map(self, _map): return _map -class PylonsAddFlashMessageController(p.toolkit.BaseController): - def flash_message_action(self): - u"""Pylons view to render flash messages in a template.""" - return pylons_render(u"tests/flash_messages.html") +if six.PY2: + class PylonsAddFlashMessageController(p.toolkit.BaseController): + def flash_message_action(self): + u"""Pylons view to render flash messages in a template.""" + return pylons_render(u"tests/flash_messages.html") - def add_flash_message_redirect(self): - # Adds a flash message and redirects to flask view - h.flash_success(u"This is a success message populated by Pylons") - return h.redirect_to(u"/flask_view_flash_message") + def add_flash_message_redirect(self): + # Adds a flash message and redirects to flask view + h.flash_success(u"This is a success message populated by Pylons") + return h.redirect_to(u"/flask_view_flash_message") diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 746801ce0da..5f4f4c0eb31 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -6,7 +6,10 @@ import json import re -import __builtin__ as builtins +try: + import builtins +except ImportError: + import __builtin__ as builtins import mock import pytest diff --git a/ckan/tests/controllers/test_feed.py b/ckan/tests/controllers/test_feed.py index 981096da7b9..cd738911f2a 100644 --- a/ckan/tests/controllers/test_feed.py +++ b/ckan/tests/controllers/test_feed.py @@ -1,13 +1,15 @@ # encoding: utf-8 import pytest +import six from ckan.lib.helpers import url_for import ckan.tests.helpers as helpers import ckan.tests.factories as factories import ckan.plugins as plugins -from webhelpers.feedgenerator import GeoAtom1Feed +if six.PY2: + from webhelpers.feedgenerator import GeoAtom1Feed @pytest.mark.usefixtures("clean_db") diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index a25b1337930..03661bb4e8a 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -2041,7 +2041,7 @@ def test_legacy_changed_package_activity(self, app): @mock.patch( "ckan.logic.validators.object_id_validators", dict( - object_id_validators.items() + list(object_id_validators.items()) + [("changed datastore", package_id_exists)] ), ) diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index a109d1b9fb3..cfa978bb79a 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -30,6 +30,7 @@ import nose.tools import mock import rq +import six from ckan.common import config import ckan.lib.jobs as jobs @@ -119,6 +120,14 @@ def call_action(action_name, context=None, **kwargs): context = {} context.setdefault("user", "127.0.0.1") context.setdefault("ignore_auth", True) + + if six.PY3: + from ckan.lib.helpers import _get_auto_flask_context + _auto_flask_context = _get_auto_flask_context() + if _auto_flask_context: + _auto_flask_context.push() + + return logic.get_action(action_name)(context=context, data_dict=kwargs) @@ -159,6 +168,11 @@ def call_auth(auth_name, context, **kwargs): return logic.check_access(auth_name, context, data_dict=kwargs) +def body_contains(res, content): + body = res.body if six.PY2 else res.body.decode() + return content in body + + class CKANTestApp(webtest.TestApp): """A wrapper around webtest.TestApp @@ -170,7 +184,10 @@ class CKANTestApp(webtest.TestApp): @property def flask_app(self): if not self._flask_app: - self._flask_app = self.app.apps["flask_app"]._wsgi_app + if six.PY2: + self._flask_app = self.app.apps["flask_app"]._wsgi_app + else: + self._flask_app = self.app._wsgi_app return self._flask_app def post(self, url, *args, **kwargs): @@ -196,7 +213,10 @@ def test_dataset_search(self, app): """ config["ckan.legacy_templates"] = False config["testing"] = True - app = ckan.config.middleware.make_app(config["global_conf"], **config) + if six.PY2: + app = ckan.config.middleware.make_app(config["global_conf"], **config) + else: + app = ckan.config.middleware.make_app(config, **config) app = CKANTestApp(app) return app diff --git a/ckan/tests/i18n/test_check_po_files.py b/ckan/tests/i18n/test_check_po_files.py index 82d38046d5c..dff583bf937 100644 --- a/ckan/tests/i18n/test_check_po_files.py +++ b/ckan/tests/i18n/test_check_po_files.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from ckan.i18n.check_po_files import ( +from ckan.cli.translation import ( check_po_file, simple_conv_specs, mapping_keys, diff --git a/ckan/tests/legacy/__init__.py b/ckan/tests/legacy/__init__.py index 0da06b7d2af..c8e8ade227d 100644 --- a/ckan/tests/legacy/__init__.py +++ b/ckan/tests/legacy/__init__.py @@ -16,12 +16,8 @@ from nose.plugins.skip import SkipTest from ckan.common import config -from pylons.test import pylonsapp -from paste.script.appinstall import SetupCommand from six import text_type -import paste.fixture -import paste.script.appinstall from ckan.lib.create_test_data import CreateTestData from ckan.lib import search @@ -55,18 +51,6 @@ here_dir = os.path.dirname(os.path.abspath(__file__)) conf_dir = os.path.dirname(os.path.dirname(here_dir)) -# Invoke websetup with the current config file -# SetupCommand('setup-app').run([config['__file__']]) - -# monkey patch paste.fixtures.TestRespose -# webtest (successor library) already has this -# http://pythonpaste.org/webtest/#parsing-the-body -def _getjson(self): - return json.loads(self.body) - - -paste.fixture.TestResponse.json = property(_getjson) - # Check config is correct for sqlite if model.engine_is_sqlite(): assert ( @@ -259,12 +243,6 @@ def teardown(self): class WsgiAppCase(BaseCase): - # wsgiapp = pylonsapp - # assert wsgiapp, 'You need to run nose with --with-pylons' - # Either that, or this file got imported somehow before the tests started - # running, meaning the pylonsapp wasn't setup yet (which is done in - # pylons.test.py:begin()) - # app = paste.fixture.TestApp(wsgiapp) app = helpers._get_test_app() @@ -422,7 +400,6 @@ def call_action_api(app, action, apikey=None, status=200, **kwargs): assert error_dict['message'] == 'Access Denied' :param app: the test app to post to - :type app: paste.fixture.TestApp :param action: the action to post to, e.g. 'package_create' :type action: string @@ -439,10 +416,6 @@ def call_action_api(app, action, apikey=None, status=200, **kwargs): :param **kwargs: any other keyword arguments passed to this function will be posted to the API as params - :raises paste.fixture.AppError: if the HTTP status code of the response - from the CKAN API is different from the status param passed to this - function - :returns: the 'result' or 'error' dictionary from the CKAN API response :rtype: dictionary diff --git a/ckan/tests/legacy/functional/test_package.py b/ckan/tests/legacy/functional/test_package.py index 4530fc4308d..7c1465999d5 100644 --- a/ckan/tests/legacy/functional/test_package.py +++ b/ckan/tests/legacy/functional/test_package.py @@ -9,7 +9,6 @@ from ckan.tests.legacy import url_for import ckan.tests.legacy as tests from ckan.tests.legacy.html_check import HtmlCheckMethods -from base import FunctionalTestCase import ckan.model as model from ckan.lib.create_test_data import CreateTestData from ckan.logic.action import get, update @@ -64,7 +63,7 @@ def _check_package_read(self, res, **params): assert license.title in main_div, (license.title, main_div_str) tag_names = list(params["tags"]) self.check_named_element(main_div, "ul", *tag_names) - if params.has_key("state"): + if "state" in params: assert "State: %s" % params["state"] in main_div.replace( "", "" ), main_div_str @@ -117,7 +116,7 @@ def escape_for_html_body(self, unescaped_str): return unescaped_str.replace("<", "<") def check_form_filled_correctly(self, res, **params): - if params.has_key("pkg"): + if "pkg" in params: for key, value in params["pkg"].as_dict().items(): if key == "license": key = "license_id" @@ -141,7 +140,7 @@ def check_form_filled_correctly(self, res, **params): tags = params["tags"] for tag in tags: self.check_tag(main_res, prefix + "tag_string", tag) - if params.has_key("state"): + if "state" in params: self.check_tag_and_data(main_res, "selected", str(params["state"])) if isinstance(params["extras"], dict): extras = [] @@ -336,7 +335,7 @@ def test_edit_2_not_groups(self, app): fv = app.get( self.offset, extra_environ=self.extra_environ_admin ).forms["dataset-edit"] - assert not fv.fields.has_key(prefix + "groups") + assert prefix + "groups" not in fv.fields def test_redirect_after_edit_using_param(self, app): return_url = "http://random.site.com/dataset/?test=param" diff --git a/ckan/tests/legacy/html_check.py b/ckan/tests/legacy/html_check.py index 17c0d12300e..c5b93ce1536 100644 --- a/ckan/tests/legacy/html_check.py +++ b/ckan/tests/legacy/html_check.py @@ -1,12 +1,10 @@ # encoding: utf-8 import re -import sgmllib +from html.parser import HTMLParser from six import string_types, text_type - -import paste.fixture import webtest @@ -70,9 +68,7 @@ def check_tag(self, html, *html_to_find): self._check_html(self.tag_re, html, html_to_find) def _get_html_from_res(self, html): - if isinstance(html, paste.fixture.TestResponse): - html_str = html.body.decode("utf8") - elif isinstance(html, text_type): + if isinstance(html, text_type): html_str = html elif isinstance(html, str): html_str = html.decode("utf8") @@ -124,12 +120,9 @@ def _check_html(self, regex_compiled, html, html_to_find): ) -class Stripper(sgmllib.SGMLParser): +class Stripper(HTMLParser): """A simple helper class to cleanly strip HTML from a response.""" - def __init__(self): - sgmllib.SGMLParser.__init__(self) - def strip(self, html): self.str = u"" self.feed(html) diff --git a/ckan/tests/logic/action/test_create.py b/ckan/tests/logic/action/test_create.py index 98293ec3f1a..00415f69f47 100644 --- a/ckan/tests/logic/action/test_create.py +++ b/ckan/tests/logic/action/test_create.py @@ -2,7 +2,10 @@ """Unit tests for ckan/logic/action/create.py. """ -import __builtin__ as builtins +try: + import builtins +except ImportError: + import __builtin__ as builtins import cgi import mock diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py index 982eee266c7..7e9a11265bb 100644 --- a/ckan/tests/logic/action/test_update.py +++ b/ckan/tests/logic/action/test_update.py @@ -1,6 +1,10 @@ # encoding: utf-8 """Unit tests for ckan/logic/action/update.py.""" -import __builtin__ as builtins +try: + import builtins +except ImportError: + import __builtin__ as builtins + import datetime import mock diff --git a/ckan/tests/pytest_ckan/ckan_setup.py b/ckan/tests/pytest_ckan/ckan_setup.py index a9ac91caa4f..619a5808bd3 100644 --- a/ckan/tests/pytest_ckan/ckan_setup.py +++ b/ckan/tests/pytest_ckan/ckan_setup.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import six + from ckan.config.middleware import make_app from ckan.cli import load_config @@ -23,7 +25,10 @@ def pytest_sessionstart(session): global _tests_test_request_context app = make_app(conf.global_conf, **conf.local_conf) - flask_app = app.apps['flask_app']._wsgi_app + try: + flask_app = app.apps['flask_app']._wsgi_app + except AttributeError: + flask_app = app._wsgi_app _tests_test_request_context = flask_app.test_request_context() diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index 796e135a326..7db01992811 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -16,6 +16,7 @@ import re import subprocess import sys +import six from six import text_type from six.moves import xrange @@ -24,7 +25,10 @@ sys.getfilesystemencoding() or sys.getdefaultencoding() ) -HERE = os.path.abspath(os.path.dirname(__file__.decode(FILESYSTEM_ENCODING))) +if six.PY2: + HERE = os.path.abspath(os.path.dirname(__file__.decode(FILESYSTEM_ENCODING))) +else: + HERE = os.path.abspath(os.path.dirname(__file__)) PROJECT_ROOT = os.path.normpath(os.path.join(HERE, u"..", u"..")) diff --git a/ckan/tests/test_common.py b/ckan/tests/test_common.py index c30896a23a6..0c165d39c90 100644 --- a/ckan/tests/test_common.py +++ b/ckan/tests/test_common.py @@ -1,9 +1,7 @@ # encoding: utf-8 import flask -import pylons import pytest -from nose.tools import eq_, assert_not_equal as neq_, assert_raises import six from six import text_type @@ -34,7 +32,10 @@ def test_get_item_works(): def test_repr_works(): my_conf = CKANConfig() my_conf[u"test_key_1"] = u"Test value 1" - assert repr(my_conf) == u"{u'test_key_1': u'Test value 1'}" + if six.PY3: + assert repr(my_conf) == u"{'test_key_1': 'Test value 1'}" + else: + assert repr(my_conf) == u"{u'test_key_1': u'Test value 1'}" def test_len_works(): @@ -51,9 +52,8 @@ def test_keys_works(): assert sorted(my_conf.keys()) == [u"test_key_1", u"test_key_2"] +# @pytest.mark.usefixtures("ckan_config") def test_clear_works(): - # Keep a copy of the original Pylons config - _original_pylons_config = pylons.config.copy() my_conf = CKANConfig() my_conf[u"test_key_1"] = u"Test value 1" my_conf[u"test_key_2"] = u"Test value 2" @@ -62,9 +62,6 @@ def test_clear_works(): my_conf.clear() assert len(my_conf.keys()) == 0 - # Restore Pylons config - pylons.config.update(_original_pylons_config) - def test_for_in_works(): my_conf = CKANConfig() @@ -102,6 +99,7 @@ def test_true_if_not_empty(): assert my_conf +@pytest.mark.skipif(six.PY3, reason="Do not test pylons in Py3") @pytest.mark.ckan_config(u"ckan.site_title", u"Example title") def test_setting_a_key_sets_it_on_pylons_config(): assert pylons.config[u"ckan.site_title"] == u"Example title" @@ -124,7 +122,7 @@ def test_setting_a_key_does_not_set_it_on_flask_config_if_outside_app_context( @pytest.mark.ckan_config(u"ckan.site_title", u"Example title") -def test_deleting_a_key_deletes_it_on_pylons_config(): +def test_deleting_a_key_deletes_it_on_ckan_config(): del ckan_config[u"ckan.site_title"] assert u"ckan.site_title" not in ckan_config @@ -142,6 +140,7 @@ def test_deleting_a_key_delets_it_on_flask_config( # END-CONFIG-OVERRIDE +@pytest.mark.skipif(six.PY3, reason="Do not test pylons in Py3") @pytest.mark.ckan_config(u"ckan.site_title", u"Example title") def test_update_works_on_pylons_config(): ckan_config.update( @@ -164,6 +163,7 @@ def test_update_works_on_flask_config(app): assert flask.current_app.config[u"ckan.new_key"] == u"test" +@pytest.mark.skipif(six.PY3, reason="Do not test pylons in Py3") def test_config_option_update_action_works_on_pylons(reset_db): params = {u"ckan.site_title": u"Example title action"} helpers.call_action(u"config_option_update", {}, **params) @@ -171,11 +171,11 @@ def test_config_option_update_action_works_on_pylons(reset_db): reset_db() -def test_config_option_update_action_works_on_flask(app, reset_db): +def test_config_option_update_action_works_on_flask(app, reset_db, ckan_config): with app.flask_app.app_context(): params = {u"ckan.site_title": u"Example title action"} helpers.call_action(u"config_option_update", {}, **params) - assert pylons.config[u"ckan.site_title"] == u"Example title action" + assert ckan_config[u"ckan.site_title"] == u"Example title action" reset_db() diff --git a/ckanext/example_iuploader/test/test_plugin.py b/ckanext/example_iuploader/test/test_plugin.py index 785d1bb1bae..cc1209db8df 100644 --- a/ckanext/example_iuploader/test/test_plugin.py +++ b/ckanext/example_iuploader/test/test_plugin.py @@ -1,6 +1,9 @@ # encoding: utf-8 -import __builtin__ as builtins +try: + import builtins +except ImportError: + import __builtin__ as builtins import paste.fileapp import flask diff --git a/doc/extensions/testing-extensions.rst b/doc/extensions/testing-extensions.rst index 1e2d3d9b7a1..f3c64f35ba3 100644 --- a/doc/extensions/testing-extensions.rst +++ b/doc/extensions/testing-extensions.rst @@ -47,11 +47,6 @@ Some notes on how these tests work: * Pytest has lots of useful functions for testing, see the `pytest documentation `_. -* We're using a :class:`paste.fixture.TestApp` object to simulate sending HTTP - requests to the CKAN API or frontend. - See `Testing Applications with Paste `_ - for some documentation of this. - * We're calling :func:`ckan.tests.call_action_api` to post (simulated) HTTP requests to the CKAN API. This is a convenience function that CKAN provides for its own tests.