From 3e979fcf0ae835ccdef977413b5829a8fbcac59a Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Jan 2020 19:06:23 +0200 Subject: [PATCH 01/10] Green legacy and logic --- ckan/cli/search_index.py | 56 ++++++------ ckan/config/middleware/common_middleware.py | 6 +- ckan/config/middleware/flask_app.py | 4 + ckan/lib/dictization/__init__.py | 3 +- ckan/lib/hash.py | 3 +- ckan/lib/helpers.py | 7 +- ckan/lib/navl/validators.py | 2 +- ckan/lib/search/__init__.py | 2 +- ckan/lib/search/query.py | 2 +- ckan/tests/controllers/test_api.py | 2 +- ckan/tests/helpers.py | 4 +- .../api/test_email_notifications.py | 69 ++++++-------- ckan/tests/legacy/functional/api/test_user.py | 1 + ckan/tests/legacy/functional/test_group.py | 4 +- ckan/tests/legacy/functional/test_package.py | 5 +- .../functional/test_preview_interface.py | 28 +++--- ckan/tests/legacy/functional/test_tracking.py | 13 ++- ckan/tests/legacy/functional/test_user.py | 1 - ckan/tests/legacy/lib/test_dictization.py | 17 ++-- .../legacy/lib/test_dictization_schema.py | 2 +- ckan/tests/legacy/lib/test_helpers.py | 11 ++- ckan/tests/legacy/lib/test_navl.py | 2 +- ckan/tests/legacy/logic/test_action.py | 2 +- ckan/tests/legacy/logic/test_auth.py | 4 +- ckan/tests/legacy/logic/test_init.py | 2 +- ckan/tests/legacy/logic/test_member.py | 1 + ckan/tests/legacy/misc/__init__.py | 0 .../legacy/misc/test_mock_mail_server.py | 39 -------- ckan/tests/legacy/mock_mail_server.py | 89 ------------------- ckan/tests/legacy/test_plugins.py | 2 +- ckan/tests/lib/test_mailer.py | 1 - ckan/tests/test_coding_standards.py | 2 - ckan/views/__init__.py | 4 +- 33 files changed, 126 insertions(+), 264 deletions(-) delete mode 100644 ckan/tests/legacy/misc/__init__.py delete mode 100644 ckan/tests/legacy/misc/test_mock_mail_server.py delete mode 100644 ckan/tests/legacy/mock_mail_server.py diff --git a/ckan/cli/search_index.py b/ckan/cli/search_index.py index bc23b38c9c8..9f164f01a1a 100644 --- a/ckan/cli/search_index.py +++ b/ckan/cli/search_index.py @@ -5,7 +5,7 @@ import click import sqlalchemy as sa -from ckan.cli import error_shout +import ckan.plugins.toolkit as tk @click.group(name=u'search-index', short_help=u'Search index commands') @@ -28,20 +28,17 @@ def search_index(): u'ensures that changes are immediately available on the' u'search, but slows significantly the process. Default' u'is false.') -@click.pass_context -def rebuild(ctx, verbose, force, refresh, only_missing, quiet, commit_each): +def rebuild(verbose, force, refresh, only_missing, quiet, commit_each): u''' Rebuild search index ''' - flask_app = ctx.obj.app.apps['flask_app']._wsgi_app from ckan.lib.search import rebuild, commit try: - with flask_app.test_request_context(): - rebuild(only_missing=only_missing, - force=force, - refresh=refresh, - defer_commit=(not commit_each), - quiet=quiet) + rebuild(only_missing=only_missing, + force=force, + refresh=refresh, + defer_commit=(not commit_each), + quiet=quiet) except Exception as e: - error_shout(e) + tk.error_shout(e) if not commit_each: commit() @@ -74,11 +71,10 @@ def clear(dataset_name): @search_index.command(name=u'rebuild-fast', short_help=u'Reindex with multiprocessing') -@click.pass_context -def rebuild_fast(ctx): - conf = ctx.obj.config - flask_app = ctx.obj.app.apps['flask_app']._wsgi_app - db_url = conf['sqlalchemy.url'] +def rebuild_fast(): + from ckan.lib.search import commit + + db_url = tk.config['sqlalchemy.url'] engine = sa.create_engine(db_url) package_ids = [] result = engine.execute(u"select id from package where state = 'active';") @@ -86,9 +82,8 @@ def rebuild_fast(ctx): package_ids.append(row[0]) def start(ids): - from ckan.lib.search import rebuild, commit + from ckan.lib.search import rebuild rebuild(package_ids=ids) - commit() def chunks(l, n): u""" Yield n successive chunks from l.""" @@ -98,15 +93,16 @@ def chunks(l, n): yield l[n * newn - newn:] processes = [] - with flask_app.test_request_context(): - try: - for chunk in chunks(package_ids, mp.cpu_count()): - process = mp.Process(target=start, args=(chunk,)) - processes.append(process) - process.daemon = True - process.start() - - for process in processes: - process.join() - except Exception as e: - click.echo(e.message) + + try: + for chunk in chunks(package_ids, mp.cpu_count()): + process = mp.Process(target=start, args=(chunk,)) + processes.append(process) + process.daemon = True + process.start() + + for process in processes: + process.join() + commit() + except Exception as e: + click.echo(e.message) diff --git a/ckan/config/middleware/common_middleware.py b/ckan/config/middleware/common_middleware.py index fcee0bff501..abba216ad52 100644 --- a/ckan/config/middleware/common_middleware.py +++ b/ckan/config/middleware/common_middleware.py @@ -70,12 +70,12 @@ def __call__(self, environ, start_response): if path == '/_tracking' and method == 'POST': # do the tracking # get the post data - payload = environ['wsgi.input'].read() + payload = six.ensure_str(environ['wsgi.input'].read()) parts = payload.split('&') data = {} for part in parts: k, v = part.split('=') - data[k] = six.ensure_text(unquote(v)) + data[k] = unquote(v) start_response('200 OK', [('Content-Type', 'text/html')]) # we want a unique anonomized key for each user so that we do # not count multiple clicks from the same user. @@ -85,7 +85,7 @@ def __call__(self, environ, start_response): environ.get('HTTP_ACCEPT_LANGUAGE', ''), environ.get('HTTP_ACCEPT_ENCODING', ''), ]) - key = hashlib.md5(key).hexdigest() + key = hashlib.md5(six.ensure_binary(key)).hexdigest() # store key/data here sql = '''INSERT INTO tracking_raw (user_key, url, tracking_type) diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py index 341d0e4080f..05c4ad267e9 100644 --- a/ckan/config/middleware/flask_app.py +++ b/ckan/config/middleware/flask_app.py @@ -30,6 +30,7 @@ from ckan.lib import helpers from ckan.lib import jinja_extensions from ckan.common import config, g, request, ungettext +from ckan.config.middleware.common_middleware import TrackingMiddleware import ckan.lib.app_globals as app_globals import ckan.lib.plugins as lib_plugins import ckan.plugins.toolkit as toolkit @@ -300,6 +301,9 @@ def ungettext_alias(): if six.PY3: app = I18nMiddleware(app) + if asbool(config.get('ckan.tracking_enabled', 'false')): + app = TrackingMiddleware(app, config) + # Add a reference to the actual Flask app so it's easier to access app._wsgi_app = flask_app diff --git a/ckan/lib/dictization/__init__.py b/ckan/lib/dictization/__init__.py index 6e1f219b6bc..ca9d3acefbc 100644 --- a/ckan/lib/dictization/__init__.py +++ b/ckan/lib/dictization/__init__.py @@ -24,6 +24,7 @@ # and saving dictized objects. If a specialised use is needed please do NOT extend # these functions. Copy code from here as needed. +legacy_dict_sort = lambda x: (len(x), dict.items(x)) def table_dictize(obj, context, **kw): '''Get any model object and represent it as a dict''' @@ -72,7 +73,7 @@ def table_dictize(obj, context, **kw): return result_dict -def obj_list_dictize(obj_list, context, sort_key=lambda x:x): +def obj_list_dictize(obj_list, context, sort_key=legacy_dict_sort): '''Get a list of model object and represent it as a list of dicts''' result_list = [] diff --git a/ckan/lib/hash.py b/ckan/lib/hash.py index 87a15ab7348..0dbecf37adc 100644 --- a/ckan/lib/hash.py +++ b/ckan/lib/hash.py @@ -2,6 +2,7 @@ import hmac import hashlib +import six from ckan.common import config, request @@ -12,7 +13,7 @@ def get_message_hash(value): if not secret: # avoid getting config value at module scope since config may # not be read in yet - secret = config['beaker.session.secret'] + secret = six.ensure_binary(config['beaker.session.secret']) return hmac.new(secret, value.encode('utf8'), hashlib.sha1).hexdigest() def get_redirect(): diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index c6328acda06..4c6048add6b 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1666,8 +1666,7 @@ def date_str_to_datetime(date_str): @core_helper def parse_rfc_2822_date(date_str, assume_utc=True): - ''' - Parse a date string of the form specified in RFC 2822, and return a + '''Parse a date string of the form specified in RFC 2822, and return a datetime. RFC 2822 is the date format used in HTTP headers. It should contain @@ -1684,6 +1683,10 @@ def parse_rfc_2822_date(date_str, assume_utc=True): datetime is 'aware', ie - it has an associated tz_info object. Returns None if the string cannot be parsed as a valid datetime. + + Note: in Python3, `email.utils` always assume UTC if there is no + timezone, so `assume_utc` has no sense in this version. + ''' time_tuple = email.utils.parsedate_tz(date_str) diff --git a/ckan/lib/navl/validators.py b/ckan/lib/navl/validators.py index d35134314a7..67f1605487c 100644 --- a/ckan/lib/navl/validators.py +++ b/ckan/lib/navl/validators.py @@ -65,7 +65,7 @@ def empty(key, data, errors, context): key_name = key[-1] if key_name == '__junk': # for junked fields, the field name is contained in the value - key_name = value.keys() + key_name = list(value.keys()) errors[key].append(_( 'The input field %(name)s was not expected.') % {"name": key_name}) diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index 60af1b0f07b..c06520cb667 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -158,7 +158,7 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, log.info('Indexing just package %r...', pkg_dict['name']) package_index.remove_dict(pkg_dict) package_index.insert_dict(pkg_dict) - elif package_ids: + elif package_ids is not None: for package_id in package_ids: pkg_dict = logic.get_action('package_show')(context, {'id': package_id}) diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 7f5be9f7e9b..4c33fad72e8 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -399,7 +399,7 @@ def run(self, query, permission_labels=None, **kwargs): for result in self.results: extra_keys = filter(lambda x: x.startswith('extras_'), result.keys()) extras = {} - for extra_key in extra_keys: + for extra_key in list(extra_keys): value = result.pop(extra_key) extras[extra_key[len('extras_'):]] = value if extra_keys: diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 47922a0c78b..706e58a506a 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -53,7 +53,7 @@ def test_resource_create_upload_file(self, app, monkeypatch): env = {"REMOTE_USER": six.ensure_str(user["name"])} content = six.ensure_binary('upload-content') - upload_content = six.StringIO(content) + upload_content = six.BytesIO(content) postparams = {"name": "test-flask-upload", "package_id": pkg["id"], "upload": (upload_content, "test-upload.txt")} monkeypatch.setattr(builtins, 'open', mock_open_if_open_fails) diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index b728e9d0e49..077ce2faf0a 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -175,7 +175,7 @@ def body_contains(res, content): class CKANResponse(Response): @property def body(self): - return self.data + return six.ensure_str(self.data) def __contains__(self, segment): return body_contains(self, segment) @@ -610,7 +610,7 @@ def get_smtp_messages(self): return self._msgs def clear_smtp_messages(self): - self.msgs = [] + self._msgs = [] def sendmail( self, from_addr, to_addrs, msg, mail_options=(), rcpt_options=() diff --git a/ckan/tests/legacy/functional/api/test_email_notifications.py b/ckan/tests/legacy/functional/api/test_email_notifications.py index 9f46b59a3fd..eb9ee963a9b 100644 --- a/ckan/tests/legacy/functional/api/test_email_notifications.py +++ b/ckan/tests/legacy/functional/api/test_email_notifications.py @@ -5,18 +5,14 @@ import ckan.model as model import ckan.tests.legacy as tests import ckan.tests.helpers as helpers -import ckan.tests.legacy.mock_mail_server as mock_mail_server from ckan.tests.legacy import TestController as ControllerTestCase -class TestEmailNotifications( - mock_mail_server.SmtpServerHarness, ControllerTestCase -): +class TestEmailNotifications(ControllerTestCase): @classmethod def setup_class(cls): model.repo.rebuild_db() - mock_mail_server.SmtpServerHarness.setup_class() tests.CreateTestData.create() cls.app = helpers._get_test_app() joeadmin = model.User.get("joeadmin") @@ -50,7 +46,7 @@ def check_email(self, email, address, name, subject): assert encoded_subject in email[3] # TODO: Check that body contains link to dashboard and email prefs. - def test_00_send_email_notifications_not_logged_in(self): + def test_00_send_email_notifications_not_logged_in(self, mail_server): """Not-logged-in users shouldn't be able to send email notifications. """ @@ -76,7 +72,7 @@ def test_00_send_email_notifications_not_logged_in(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - self.clear_smtp_messages() + mail_server.clear_smtp_messages() # No notification emails should be sent to anyone at this point. tests.call_action_api( @@ -84,7 +80,7 @@ def test_00_send_email_notifications_not_logged_in(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 0 + assert len(mail_server.get_smtp_messages()) == 0 # def test_02_one_new_activity(self): """A user with one new activity should get one email.""" @@ -114,8 +110,8 @@ def test_00_send_email_notifications_not_logged_in(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 1 - email = self.get_smtp_messages()[0] + assert len(mail_server.get_smtp_messages()) == 1 + email = mail_server.get_smtp_messages()[0] self.check_email( email, "sara@sararollins.com", @@ -123,8 +119,6 @@ def test_00_send_email_notifications_not_logged_in(self): "1 new activity from CKAN", ) - self.clear_smtp_messages() - # def test_03_multiple_new_activities(self): """Test that a user with multiple new activities gets just one email. @@ -142,13 +136,15 @@ def test_00_send_email_notifications_not_logged_in(self): # Run the email notifier job, it should send one notification email # to Sara. + mail_server.clear_smtp_messages() + tests.call_action_api( self.app, "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 1 - email = self.get_smtp_messages()[0] + assert len(mail_server.get_smtp_messages()) == 1 + email = mail_server.get_smtp_messages()[0] self.check_email( email, "sara@sararollins.com", @@ -156,7 +152,7 @@ def test_00_send_email_notifications_not_logged_in(self): "3 new activities from CKAN", ) - self.clear_smtp_messages() + mail_server.clear_smtp_messages() # def test_04_no_repeat_email_notifications(self): """Test that a user does not get a second email notification for the @@ -170,7 +166,7 @@ def test_00_send_email_notifications_not_logged_in(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 0 + assert len(mail_server.get_smtp_messages()) == 0 # def test_05_no_email_if_seen_on_dashboard(self): """Test that emails are not sent for activities already seen on dash. @@ -211,7 +207,7 @@ def test_00_send_email_notifications_not_logged_in(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 0 + assert len(mail_server.get_smtp_messages()) == 0 # def test_05_no_email_notifications_when_disabled_site_wide(self): """Users should not get email notifications when the feature is @@ -226,15 +222,12 @@ def test_00_send_email_notifications_not_logged_in(self): # It's just easier to separate these tests into their own test class. -class TestEmailNotificationsUserPreference( - mock_mail_server.SmtpServerHarness, ControllerTestCase -): +class TestEmailNotificationsUserPreference(ControllerTestCase): """Tests for the email notifications (on/off) user preference.""" @classmethod def setup_class(cls): model.repo.rebuild_db() - mock_mail_server.SmtpServerHarness.setup_class() tests.CreateTestData.create() cls.app = helpers._get_test_app() joeadmin = model.User.get("joeadmin") @@ -255,7 +248,7 @@ def setup_class(cls): fullname="Sara Rollins", ) - def test_00_email_notifications_disabled_by_default(self): + def test_00_email_notifications_disabled_by_default(self, mail_server): """Email notifications should be disabled for new users.""" assert self.sara["activity_streams_email_notifications"] is False assert ( @@ -302,7 +295,7 @@ def test_00_email_notifications_disabled_by_default(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 0 + assert len(mail_server.get_smtp_messages()) == 0 # def test_02_enable_email_notifications(self): """Users should be able to turn email notifications on.""" @@ -349,7 +342,7 @@ def test_00_email_notifications_disabled_by_default(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 0 + assert len(mail_server.get_smtp_messages()) == 0 # Enable email notifications for Sara. self.sara["activity_streams_email_notifications"] = True @@ -360,7 +353,7 @@ def test_00_email_notifications_disabled_by_default(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 0, ( + assert len(mail_server.get_smtp_messages()) == 0, ( "After a user enables " "email notifications she should _not_ get emails about activities " "that happened before she enabled them, even if those activities " @@ -392,8 +385,8 @@ def test_00_email_notifications_disabled_by_default(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 1 - self.clear_smtp_messages() + assert len(mail_server.get_smtp_messages()) == 1 + mail_server.clear_smtp_messages() # def test_03_disable_email_notifications(self): """Users should be able to turn email notifications off.""" @@ -423,12 +416,10 @@ def test_00_email_notifications_disabled_by_default(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 0 + assert len(mail_server.get_smtp_messages()) == 0 -class TestEmailNotificationsIniSetting( - mock_mail_server.SmtpServerHarness, ControllerTestCase -): +class TestEmailNotificationsIniSetting(ControllerTestCase): """Tests for the ckan.activity_streams_email_notifications config setting. """ @@ -440,7 +431,6 @@ def setup_class(cls): cls.app = helpers._get_test_app() model.repo.rebuild_db() - mock_mail_server.SmtpServerHarness.setup_class() tests.CreateTestData.create() joeadmin = model.User.get("joeadmin") @@ -452,7 +442,7 @@ def setup_class(cls): } @helpers.change_config("ckan.activity_streams_email_notifications", False) - def test_00_send_email_notifications_feature_disabled(self): + def test_00_send_email_notifications_feature_disabled(self, mail_server): """Send_email_notifications API should error when feature disabled.""" # Register a new user. @@ -517,16 +507,14 @@ def test_00_send_email_notifications_feature_disabled(self): ) @helpers.change_config("ckan.activity_streams_email_notifications", False) - def test_01_no_emails_sent_if_turned_off(self): + def test_01_no_emails_sent_if_turned_off(self, mail_server): """No emails should be sent if the feature is disabled site-wide.""" # No emails should have been sent by the last test. - assert len(self.get_smtp_messages()) == 0 + assert len(mail_server.get_smtp_messages()) == 0 -class TestEmailNotificationsSinceIniSetting( - mock_mail_server.SmtpServerHarness, ControllerTestCase -): +class TestEmailNotificationsSinceIniSetting(ControllerTestCase): """Tests for the ckan.email_notifications_since config setting.""" @classmethod @@ -534,7 +522,6 @@ def setup_class(cls): cls.app = helpers._get_test_app() model.repo.rebuild_db() - mock_mail_server.SmtpServerHarness.setup_class() tests.CreateTestData.create() joeadmin = model.User.get("joeadmin") @@ -548,7 +535,7 @@ def setup_class(cls): # Don't send email notifications for activities older than 1 # microsecond @helpers.change_config("ckan.email_notifications_since", ".000001") - def test_00_email_notifications_since(self): + def test_00_email_notifications_since(self, mail_server): """No emails should be sent for activities older than email_notifications_since. @@ -614,4 +601,4 @@ def test_00_email_notifications_since(self): "send_email_notifications", apikey=self.testsysadmin["apikey"], ) - assert len(self.get_smtp_messages()) == 0 + assert len(mail_server.get_smtp_messages()) == 0 diff --git a/ckan/tests/legacy/functional/api/test_user.py b/ckan/tests/legacy/functional/api/test_user.py index 5d15474bdff..4ca82751e2a 100644 --- a/ckan/tests/legacy/functional/api/test_user.py +++ b/ckan/tests/legacy/functional/api/test_user.py @@ -169,6 +169,7 @@ def test_user_create_api_disabled(self, app): assert res_dict["success"] is False +@pytest.mark.usefixtures("with_request_context") class TestUserActions(object): @pytest.fixture(autouse=True) def initial_data(self, clean_db): diff --git a/ckan/tests/legacy/functional/test_group.py b/ckan/tests/legacy/functional/test_group.py index 5f61d181d32..9c6d555f42e 100644 --- a/ckan/tests/legacy/functional/test_group.py +++ b/ckan/tests/legacy/functional/test_group.py @@ -11,7 +11,7 @@ import ckan.tests.factories as factories -@pytest.mark.usefixtures("clean_db", "clean_index") +@pytest.mark.usefixtures("clean_db", "clean_index", "with_request_context") def test_sorting(): testsysadmin = factories.Sysadmin(name=u"testsysadmin") @@ -86,7 +86,7 @@ def test_read_non_existent(app): app.get(offset, status=404) -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") @mock.patch("ckan.lib.mailer.mail_user") def test_member_new_invites_user_if_received_email(_mail_user, app): user = CreateTestData.create_user("a_user", sysadmin=True) diff --git a/ckan/tests/legacy/functional/test_package.py b/ckan/tests/legacy/functional/test_package.py index f1d6a4691e8..7cd9a952652 100644 --- a/ckan/tests/legacy/functional/test_package.py +++ b/ckan/tests/legacy/functional/test_package.py @@ -255,8 +255,8 @@ def check_link(res, controller, id): "%s:%s" % (controller, id.replace('"', """)), ) - check_link(res.data, "dataset", "pkg-1") - check_link(res.data, "group", "test-group-1") + check_link(res.body, "dataset", "pkg-1") + check_link(res.body, "group", "test-group-1") assert "decoy" not in res, res assert 'decoy"' not in res, res @@ -271,6 +271,7 @@ def test_read_plugin_hook(self, app): assert plugin.calls["read"] == 1, plugin.calls assert plugin.calls["after_show"] == 1, plugin.calls + @pytest.mark.usefixtures("with_request_context") def test_resource_list(self, app): # TODO restore this test. It doesn't make much sense with the # present resource list design. diff --git a/ckan/tests/legacy/functional/test_preview_interface.py b/ckan/tests/legacy/functional/test_preview_interface.py index 0ea429b3df2..20057e1e273 100644 --- a/ckan/tests/legacy/functional/test_preview_interface.py +++ b/ckan/tests/legacy/functional/test_preview_interface.py @@ -1,5 +1,6 @@ # encoding: utf-8 +import pytest import ckan.lib.helpers as h import ckan.logic as l import ckan.model as model @@ -9,32 +10,27 @@ import ckan.lib.dictization.model_dictize as model_dictize +@pytest.mark.ckan_config("ckan.plugins", "test_resource_preview test_json_resource_preview") +@pytest.mark.usefixtures("with_plugins", "with_request_context") class TestPluggablePreviews(base.FunctionalTestCase): - @classmethod - def setup_class(cls): + def setup_method(self): model.repo.rebuild_db() - plugins.load("test_resource_preview", "test_json_resource_preview") - cls.plugin = plugins.get_plugin("test_resource_preview") create_test_data.CreateTestData.create() - cls.package = model.Package.get("annakarenina") - cls.resource = cls.package.resources[0] - cls.url = h.url_for( - "resource.read", id=cls.package.name, resource_id=cls.resource.id + self.package = model.Package.get("annakarenina") + self.resource = self.package.resources[0] + self.url = h.url_for( + "resource.read", id=self.package.name, resource_id=self.resource.id ) - cls.preview_url = h.url_for( + self.preview_url = h.url_for( "resource.datapreview", - id=cls.package.id, - resource_id=cls.resource.id, + id=self.package.id, + resource_id=self.resource.id, ) - @classmethod - def teardown_class(cls): - - plugins.unload("test_resource_preview", "test_json_resource_preview") - def test_hook(self): + self.plugin = plugins.get_plugin("test_resource_preview") testpackage = self.package resource_dict = model_dictize.resource_dictize( self.resource, {"model": model} diff --git a/ckan/tests/legacy/functional/test_tracking.py b/ckan/tests/legacy/functional/test_tracking.py index c3104c388ae..55f0793d516 100644 --- a/ckan/tests/legacy/functional/test_tracking.py +++ b/ckan/tests/legacy/functional/test_tracking.py @@ -82,13 +82,13 @@ def _update_tracking_summary(self): # FIXME: Can this be done as more of a functional test where we # actually test calling the command and passing the args? By calling # the method directly, we're not testing the command-line parsing. - import ckan.lib.cli + import ckan.cli.tracking as tracking import ckan.model date = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime( "%Y-%m-%d" ) - ckan.lib.cli.Tracking("Tracking").update_all( + tracking.update_all( engine=ckan.model.meta.engine, start_date=date ) @@ -99,9 +99,8 @@ def _rebuild_search_index(self): line. """ - import ckan.lib.cli - - ckan.lib.cli.SearchIndexCommand("SearchIndexCommand").rebuild() + from ckan.lib.search import rebuild + rebuild() def test_package_with_0_views(self, app): sysadmin_user, apikey = self._create_sysadmin(app) @@ -574,11 +573,11 @@ def _export_tracking_summary(self): # FIXME: Can this be done as more of a functional test where we # actually test calling the command and passing the args? By calling # the method directly, we're not testing the command-line parsing. - import ckan.lib.cli + from ckan.cli.tracking import export_tracking import ckan.model f = tempfile.NamedTemporaryFile() - ckan.lib.cli.Tracking("Tracking").export_tracking( + export_tracking( engine=ckan.model.meta.engine, output_filename=f.name ) lines = [line for line in csv.DictReader(open(f.name, "r"))] diff --git a/ckan/tests/legacy/functional/test_user.py b/ckan/tests/legacy/functional/test_user.py index ba1539f2769..77bb5711067 100644 --- a/ckan/tests/legacy/functional/test_user.py +++ b/ckan/tests/legacy/functional/test_user.py @@ -11,7 +11,6 @@ @pytest.fixture(autouse=True) def initial_data(clean_db): - # SmtpServerHarness.setup_class() CreateTestData.create() # make 3 changes, authored by annafan diff --git a/ckan/tests/legacy/lib/test_dictization.py b/ckan/tests/legacy/lib/test_dictization.py index bbf7042d4a4..d5d9e5cfa94 100644 --- a/ckan/tests/legacy/lib/test_dictization.py +++ b/ckan/tests/legacy/lib/test_dictization.py @@ -45,7 +45,7 @@ def remove_changable_columns(self, dict, remove_package_id=False): if not remove_package_id: ids_to_keep.append("package_id") - for key, value in dict.items(): + for key, value in list(dict.items()): if key.endswith("id") and key not in ids_to_keep: dict.pop(key) if key == "created": @@ -92,15 +92,14 @@ def test_04_package_to_api1_with_relationship(self): create.create_family_test_data() pkg = model.Session.query(model.Package).filter_by(name="homer").one() - as_dict = pkg.as_dict() + as_dict = dict(pkg.as_dict()) as_dict["license_title"] = None as_dict["num_tags"] = 0 as_dict["num_resources"] = 0 dictize = package_to_api1(pkg, context) - as_dict["relationships"].sort(key=lambda x: x.items()) - dictize["relationships"].sort(key=lambda x: x.items()) - + as_dict["relationships"].sort(key=lambda x: list(x.items())) + dictize["relationships"].sort(key=lambda x: list(x.items())) # the is_dict method doesn't care about organizations del dictize["organization"] as_dict_string = pformat(as_dict) @@ -149,8 +148,8 @@ def test_06_package_to_api2_with_relationship(self): as_dict["num_resources"] = 0 dictize = package_to_api2(pkg, context) - as_dict["relationships"].sort(key=lambda x: x.items()) - dictize["relationships"].sort(key=lambda x: x.items()) + as_dict["relationships"].sort(key=lambda x: list(x.items())) + dictize["relationships"].sort(key=lambda x: list(x.items())) # the is_dict method doesn't care about organizations del dictize["organization"] @@ -193,6 +192,7 @@ def test_07_table_simple_save(self): == anna_dictized ), self.remove_changable_columns(table_dictize(pkg, context)) + @pytest.mark.usefixtures("with_request_context") def test_08_package_save(self): context = { @@ -478,6 +478,7 @@ def test_21_package_dictization_with_deleted_group(self): assert "test-group-2" not in [g["name"] for g in result["groups"]] assert "test-group-1" in [g["name"] for g in result["groups"]] + @pytest.mark.usefixtures("with_request_context") def test_22_user_dictize_as_sysadmin(self): """Sysadmins should be allowed to see certain sensitive data.""" context = { @@ -502,6 +503,7 @@ def test_22_user_dictize_as_sysadmin(self): assert "password" not in user_dict assert "reset_key" not in user_dict + @pytest.mark.usefixtures("with_request_context") def test_23_user_dictize_as_same_user(self): """User should be able to see their own sensitive data.""" context = {"model": model, "session": model.Session, "user": "tester"} @@ -522,6 +524,7 @@ def test_23_user_dictize_as_same_user(self): assert "password" not in user_dict assert "reset_key" not in user_dict + @pytest.mark.usefixtures("with_request_context") def test_24_user_dictize_as_other_user(self): """User should not be able to see other's sensitive data.""" context = {"model": model, "session": model.Session, "user": "annafan"} diff --git a/ckan/tests/legacy/lib/test_dictization_schema.py b/ckan/tests/legacy/lib/test_dictization_schema.py index 0f2c94df20b..e4646e54d39 100644 --- a/ckan/tests/legacy/lib/test_dictization_schema.py +++ b/ckan/tests/legacy/lib/test_dictization_schema.py @@ -22,7 +22,7 @@ def initial_data(self, clean_db, clean_index): self.context = {"model": model, "session": model.Session} def remove_changable_columns(self, dict): - for key, value in dict.items(): + for key, value in list(dict.items()): if key.endswith("id") and key != "license_id": dict.pop(key) if key == "created": diff --git a/ckan/tests/legacy/lib/test_helpers.py b/ckan/tests/legacy/lib/test_helpers.py index c71524fdb9c..0c5d088aa03 100644 --- a/ckan/tests/legacy/lib/test_helpers.py +++ b/ckan/tests/legacy/lib/test_helpers.py @@ -2,6 +2,8 @@ import datetime import pytest +import six + from ckan.common import config import ckan.lib.helpers as h @@ -63,7 +65,7 @@ def test_gravatar(self): # Hash the email address import hashlib - email_hash = hashlib.md5(email).hexdigest() + email_hash = hashlib.md5(six.ensure_binary(email)).hexdigest() res = h.linked_gravatar(email_hash, 200, default="mm") for e in expected: assert e in res, (e, res) @@ -81,7 +83,7 @@ def test_gravatar_config_set_default(self): # Hash the email address import hashlib - email_hash = hashlib.md5(email).hexdigest() + email_hash = hashlib.md5(six.ensure_binary(email)).hexdigest() res = h.linked_gravatar(email_hash, 200) for e in expected: assert e in res, (e, res) @@ -98,7 +100,7 @@ def test_gravatar_encodes_url_correctly(self): # Hash the email address import hashlib - email_hash = hashlib.md5(email).hexdigest() + email_hash = hashlib.md5(six.ensure_binary(email)).hexdigest() res = h.linked_gravatar(email_hash, 200, default=default) for e in expected: assert e in res, (e, res) @@ -112,6 +114,7 @@ def test_parse_rfc_2822_no_timezone_specified(self): dt = h.parse_rfc_2822_date("Tue, 15 Nov 1994 12:45:26") assert dt.isoformat() == "1994-11-15T12:45:26+00:00" + @pytest.mark.skipif(six.PY3, reason="`email.utils` always assumes UTC") def test_parse_rfc_2822_no_timezone_specified_assuming_local(self): """ Parse "Tue, 15 Nov 1994 12:45:26" successfully. @@ -150,7 +153,7 @@ def test_escape_js(self): assert output_str == expected_str - @pytest.mark.usefixtures("clean_db") + @pytest.mark.usefixtures("clean_db", "with_request_context") def test_get_pkg_dict_extra(self): from ckan.lib.create_test_data import CreateTestData diff --git a/ckan/tests/legacy/lib/test_navl.py b/ckan/tests/legacy/lib/test_navl.py index 6bc771478fd..bfc298c56ce 100644 --- a/ckan/tests/legacy/lib/test_navl.py +++ b/ckan/tests/legacy/lib/test_navl.py @@ -334,7 +334,7 @@ def validate_flattened(data, schema, context=None): assert isinstance(data, dict) converted_data, errors = _validate(data, schema, context) - for key, value in errors.items(): + for key, value in list(errors.items()): if not value: errors.pop(key) diff --git a/ckan/tests/legacy/logic/test_action.py b/ckan/tests/legacy/logic/test_action.py index 609b150b2b3..b5e4fb88bac 100644 --- a/ckan/tests/legacy/logic/test_action.py +++ b/ckan/tests/legacy/logic/test_action.py @@ -1133,7 +1133,7 @@ def test_before_view(self, app): class TestBulkActions(object): @pytest.fixture(autouse=True) - def initial_data(self, clean_db, clean_index, app): + def initial_data(self, clean_db, clean_index, app, with_request_context): factories.Sysadmin(apikey=u"sysadmin") data_dict = "%s=1" % json.dumps({"name": "org"}) diff --git a/ckan/tests/legacy/logic/test_auth.py b/ckan/tests/legacy/logic/test_auth.py index 1a52950020c..f1dceb16b6f 100644 --- a/ckan/tests/legacy/logic/test_auth.py +++ b/ckan/tests/legacy/logic/test_auth.py @@ -72,7 +72,7 @@ def create(name): return create -@pytest.mark.usefixtures("auth_config") +@pytest.mark.usefixtures("clean_db", "auth_config", "with_request_context") def test_only_sysadmins_can_delete_users(): user = factories.User() sysadmin = factories.Sysadmin() @@ -85,7 +85,7 @@ def test_only_sysadmins_can_delete_users(): assert call_auth("user_delete", context=context, id=user["id"]) -@pytest.mark.usefixtures("auth_config") +@pytest.mark.usefixtures("clean_db", "auth_config", "with_request_context") def test_auth_deleted_users_are_always_unauthorized(): always_success = lambda x, y: {"success": True} authz._AuthFunctions._build() diff --git a/ckan/tests/legacy/logic/test_init.py b/ckan/tests/legacy/logic/test_init.py index 00271ad5e14..a281921ff8a 100644 --- a/ckan/tests/legacy/logic/test_init.py +++ b/ckan/tests/legacy/logic/test_init.py @@ -14,7 +14,7 @@ def test_model_name_to_class(self): logic.model_name_to_class(model, "inexistent_model_name") -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestCheckAccess(object): def test_check_access_auth_user_obj_is_set(self): diff --git a/ckan/tests/legacy/logic/test_member.py b/ckan/tests/legacy/logic/test_member.py index 8ba262b3e37..d77e65e9eb3 100644 --- a/ckan/tests/legacy/logic/test_member.py +++ b/ckan/tests/legacy/logic/test_member.py @@ -6,6 +6,7 @@ import pytest +@pytest.mark.usefixtures("with_request_context") class TestMemberLogic(object): @pytest.fixture(autouse=True) def setup_method(self, clean_db): diff --git a/ckan/tests/legacy/misc/__init__.py b/ckan/tests/legacy/misc/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ckan/tests/legacy/misc/test_mock_mail_server.py b/ckan/tests/legacy/misc/test_mock_mail_server.py deleted file mode 100644 index ef03d0d2c28..00000000000 --- a/ckan/tests/legacy/misc/test_mock_mail_server.py +++ /dev/null @@ -1,39 +0,0 @@ -# encoding: utf-8 - -import time - -from ckan.common import config -import hashlib - -from ckan.tests.legacy.mock_mail_server import SmtpServerHarness -from ckan.lib.mailer import mail_recipient - - -class TestMockMailServer(SmtpServerHarness): - @classmethod - def setup_class(cls): - smtp_server = config.get("smtp.test_server") - if smtp_server: - host, port = smtp_server.split(":") - port = int(port) + int( - str(hashlib.md5(cls.__name__).hexdigest())[0], 16 - ) - config["smtp.test_server"] = "%s:%s" % (host, port) - SmtpServerHarness.setup_class() - - def test_basic(self): - msgs = self.get_smtp_messages() - assert msgs == [] - - test_email = { - "recipient_name": "Bob", - "recipient_email": "bob@bob.net", - "subject": "Meeting", - "body": "The meeting is cancelled.", - "headers": {"header1": "value1"}, - } - mail_recipient(**test_email) - time.sleep(0.1) - - msgs = self.get_smtp_messages() - assert len(msgs) == 1 diff --git a/ckan/tests/legacy/mock_mail_server.py b/ckan/tests/legacy/mock_mail_server.py deleted file mode 100644 index 7402b21a4fc..00000000000 --- a/ckan/tests/legacy/mock_mail_server.py +++ /dev/null @@ -1,89 +0,0 @@ -# encoding: utf-8 - -import threading -import asyncore -import socket -from smtpd import SMTPServer - -from ckan.common import config - - -class MockSmtpServer(SMTPServer): - """A mock SMTP server that operates in an asyncore loop""" - - def __init__(self, host, port): - self.msgs = [] - SMTPServer.__init__(self, (host, port), None) - - def process_message(self, peer, mailfrom, rcpttos, data): - self.msgs.append((peer, mailfrom, rcpttos, data)) - - def get_smtp_messages(self): - return self.msgs - - def clear_smtp_messages(self): - self.msgs = [] - - -class MockSmtpServerThread(threading.Thread): - """Runs the mock SMTP server in a thread""" - - def __init__(self, host, port): - self.assert_port_free(host, port) - # init thread - self._stop_event = threading.Event() - self.thread_name = self.__class__ - threading.Thread.__init__(self, name=self.thread_name) - # init smtp server - self.server = MockSmtpServer(host, port) - - def assert_port_free(self, host, port): - test_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - test_socket.setsockopt( - socket.SOL_SOCKET, - socket.SO_REUSEADDR, - test_socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1, - ) - test_socket.bind((host, port)) - test_socket.close() - - def run(self): - while not self._stop_event.isSet(): - asyncore.loop(timeout=0.01, count=1) - - def stop(self, timeout=None): - self._stop_event.set() - threading.Thread.join(self, timeout) - self.server.close() - - def get_smtp_messages(self): - return self.server.get_smtp_messages() - - def clear_smtp_messages(self): - return self.server.clear_smtp_messages() - - -class SmtpServerHarness(object): - """Derive from this class to run MockSMTP - a test harness that - records what email messages are requested to be sent by it.""" - - @classmethod - def setup_class(cls): - smtp_server = config.get("smtp.test_server") or config["smtp_server"] - if ":" in smtp_server: - host, port = smtp_server.split(":") - else: - host, port = smtp_server, 25 - cls.port = port - cls.smtp_thread = MockSmtpServerThread(host, int(port)) - cls.smtp_thread.start() - - @classmethod - def teardown_class(cls): - cls.smtp_thread.stop() - - def get_smtp_messages(self): - return self.smtp_thread.get_smtp_messages() - - def clear_smtp_messages(self): - return self.smtp_thread.clear_smtp_messages() diff --git a/ckan/tests/legacy/test_plugins.py b/ckan/tests/legacy/test_plugins.py index 6c76e7335c0..147aea661ae 100644 --- a/ckan/tests/legacy/test_plugins.py +++ b/ckan/tests/legacy/test_plugins.py @@ -134,7 +134,7 @@ def test_mapper_plugin_fired_on_insert(): @pytest.mark.ckan_config("ckan.plugins", "mapper_plugin") -@pytest.mark.usefixtures("with_plugins", "clean_db") +@pytest.mark.usefixtures("with_plugins", "clean_db", "with_request_context") def test_mapper_plugin_fired_on_delete(): plugin = plugins.get_plugin("mapper_plugin") CreateTestData.create_arbitrary([{"name": u"testpkg"}]) diff --git a/ckan/tests/lib/test_mailer.py b/ckan/tests/lib/test_mailer.py index 9964123878e..a08b3ca5513 100644 --- a/ckan/tests/lib/test_mailer.py +++ b/ckan/tests/lib/test_mailer.py @@ -14,7 +14,6 @@ import ckan.tests.factories as factories import ckan.tests.helpers as helpers from ckan.common import config -from ckan.tests.legacy.mock_mail_server import SmtpServerHarness class MailerBase(object): diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index cd84bc491fc..c9629bdfe20 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -517,9 +517,7 @@ def find_unprefixed_string_literals(filename): u"ckan/tests/legacy/logic/test_tag_vocab.py", u"ckan/tests/legacy/logic/test_validators.py", u"ckan/tests/legacy/misc/test_format_text.py", - u"ckan/tests/legacy/misc/test_mock_mail_server.py", u"ckan/tests/legacy/misc/test_sync.py", - u"ckan/tests/legacy/mock_mail_server.py", u"ckan/tests/legacy/mock_plugin.py", u"ckan/tests/legacy/models/test_activity.py", u"ckan/tests/legacy/models/test_extras.py", diff --git a/ckan/views/__init__.py b/ckan/views/__init__.py index d2652838535..0136306a97b 100644 --- a/ckan/views/__init__.py +++ b/ckan/views/__init__.py @@ -158,10 +158,8 @@ def _identify_user_default(): # with an userid_checker, but that would mean another db access. # See: http://docs.repoze.org/who/1.0/narr.html#module-repoze.who\ # .plugins.sql ) - g.user = request.environ.get(u'REMOTE_USER', u'') + g.user = six.ensure_text(request.environ.get(u'REMOTE_USER', u'')) if g.user: - if six.PY2: - g.user = six.ensure_text(g.user) g.userobj = model.User.by_name(g.user) if g.userobj is None or not g.userobj.is_active(): From c0e540f9fef073b1fbef0c5012c217e56a8ffdd4 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Jan 2020 19:46:14 +0200 Subject: [PATCH 02/10] provide request context --- ckan/logic/action/get.py | 5 +- ckan/logic/validators.py | 5 +- ckan/model/domain_object.py | 3 ++ ckan/tests/legacy/models/test_package.py | 1 + ckan/tests/logic/action/test_create.py | 44 ++++++++-------- ckan/tests/logic/action/test_delete.py | 12 ++--- ckan/tests/logic/action/test_get.py | 66 +++++++++++++----------- ckan/tests/logic/action/test_patch.py | 2 +- ckan/tests/logic/action/test_update.py | 55 ++++++++++---------- ckan/tests/logic/auth/test_create.py | 2 +- ckan/tests/logic/auth/test_delete.py | 4 +- ckan/tests/logic/auth/test_get.py | 2 +- ckan/tests/logic/auth/test_init.py | 2 +- ckan/tests/logic/auth/test_update.py | 10 +++- ckan/tests/logic/test_validators.py | 4 +- 15 files changed, 117 insertions(+), 100 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index b1c4e60da0d..051453a57e4 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1741,9 +1741,8 @@ def package_search(context, data_dict): for package in query.results: if isinstance(package, text_type): package = {result_fl[0]: package} - if package.get('extras'): - package.update(package['extras'] ) - package.pop('extras') + extras = package.pop('extras', {}) + package.update(extras) results.append(package) else: for package in query.results: diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index e18771b7f50..0a0dcb8cac5 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -86,7 +86,7 @@ def int_validator(value, context): except TypeError: try: return int(value) - except ValueError: + except (TypeError, ValueError): pass else: if not part: @@ -503,7 +503,6 @@ def ignore_not_sysadmin(key, data, errors, context): user = context.get('user') ignore_auth = context.get('ignore_auth') - if ignore_auth or (user and authz.is_sysadmin(user)): return @@ -698,7 +697,7 @@ def url_validator(key, data, errors, context): try: pieces = urlparse(url) if all([pieces.scheme, pieces.netloc]) and \ - set(pieces.netloc) <= set(string.letters + string.digits + '-.') and \ + set(pieces.netloc) <= set(string.ascii_letters + string.digits + '-.') and \ pieces.scheme in ['http', 'https']: return except ValueError: diff --git a/ckan/model/domain_object.py b/ckan/model/domain_object.py index f5190a9df8f..b408bb16e9b 100644 --- a/ckan/model/domain_object.py +++ b/ckan/model/domain_object.py @@ -99,6 +99,9 @@ def as_dict(self): _dict[col.name] = val return _dict + def __lt__(self, other): + return self.name < other.name + def __str__(self): return repr(self) diff --git a/ckan/tests/legacy/models/test_package.py b/ckan/tests/legacy/models/test_package.py index 546d7b25da8..f62fe70b972 100644 --- a/ckan/tests/legacy/models/test_package.py +++ b/ckan/tests/legacy/models/test_package.py @@ -27,6 +27,7 @@ def initial_data(self, clean_db): model.Session.commit() model.Session.remove() + @pytest.mark.usefixtures("with_request_context") def test_as_dict(self): pkg = model.Package.by_name(self.name) out = pkg.as_dict() diff --git a/ckan/tests/logic/action/test_create.py b/ckan/tests/logic/action/test_create.py index 8bf5dc3e1d0..acdd39c7e08 100644 --- a/ckan/tests/logic/action/test_create.py +++ b/ckan/tests/logic/action/test_create.py @@ -5,6 +5,7 @@ import cgi import mock import pytest +import six import ckan import ckan.logic as logic @@ -14,7 +15,7 @@ import ckan.tests.helpers as helpers from ckan.common import config -from six import string_types, StringIO +from six import string_types, StringIO, BytesIO from pyfakefs import fake_filesystem @@ -43,7 +44,7 @@ def mock_open_if_open_fails(*args, **kwargs): return fake_open(*args, **kwargs) -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestUserInvite(object): @mock.patch("ckan.lib.mailer.send_invite") def test_invited_user_is_created_as_pending(self, _): @@ -477,8 +478,8 @@ def test_mimetype_by_upload_by_filename(self, monkeypatch): If there's no url or the mimetype can't be guessed by the url, mimetype will be guessed by the extension in the filename """ - test_file = StringIO() - test_file.write( + test_file = BytesIO() + test_file.write(six.ensure_binary( """ "info": { "title": "BC Data Catalogue API", @@ -496,7 +497,7 @@ def test_mimetype_by_upload_by_filename(self, monkeypatch): "version": "3.0.0" } """ - ) + )) test_resource = FakeFileStorage(test_file, "test.json") context = {} @@ -530,15 +531,15 @@ def test_mimetype_by_upload_by_file(self, monkeypatch): If the mimetype can't be guessed by the url or filename, mimetype will be guessed by the contents inside the file """ - test_file = StringIO() - test_file.write( + test_file = BytesIO() + test_file.write(six.ensure_binary( """ Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54 MCGILLIVRAY PASS,1C05,1725,2015/12/31,88,239,,87,27,JAN-01,274 NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41 """ - ) + )) test_resource = FakeFileStorage(test_file, "") context = {} @@ -566,15 +567,15 @@ def test_size_of_resource_by_upload(self, monkeypatch): The size of the resource determined by the uploaded file """ - test_file = StringIO() - test_file.write( + test_file = BytesIO() + test_file.write(six.ensure_binary( """ Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54 MCGILLIVRAY PASS,1C05,1725,2015/12/31,88,239,,87,27,JAN-01,274 NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41 """ - ) + )) test_resource = FakeFileStorage(test_file, "test.csv") context = {} @@ -612,6 +613,7 @@ def test_size_of_resource_by_user(self): size = int(result.pop("size")) assert size == 500 + @pytest.mark.usefixtures("with_request_context") def test_extras(self): user = factories.User() dataset = factories.Dataset(user=user) @@ -636,7 +638,7 @@ def test_extras(self): assert "someotherkey" not in resource -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestMemberCreate(object): def test_group_member_creation(self): user = factories.User() @@ -721,7 +723,7 @@ def test_org_member_creation_raises_validation_error_if_role_missing(self): ) -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestDatasetCreate(object): def test_normal_user_cant_set_id(self): user = factories.User() @@ -927,7 +929,7 @@ def test_return_id_only(self): assert isinstance(dataset, string_types) -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestGroupCreate(object): def test_create_group(self): user = factories.User() @@ -986,7 +988,7 @@ def test_create_matches_show(self): assert created[k] == shown[k], k -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestOrganizationCreate(object): def test_create_organization(self): user = factories.User() @@ -1063,7 +1065,7 @@ def test_create_organization_custom_type(self): assert org["type"] == custom_org_type -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestUserCreate(object): def test_user_create_with_password_hash(self): sysadmin = factories.Sysadmin() @@ -1082,7 +1084,7 @@ def test_user_create_with_password_hash(self): def test_user_create_password_hash_not_for_normal_users(self): normal_user = factories.User() - context = {"user": normal_user["name"]} + context = {"user": normal_user["name"], "ignore_auth": False} user = helpers.call_action( "user_create", @@ -1105,7 +1107,7 @@ def _clear_activities(): model.Session.flush() -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestFollowDataset(object): def test_no_activity(self, app): @@ -1122,7 +1124,7 @@ def test_no_activity(self, app): # https://github.com/ckan/ckan/pull/317 -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestFollowGroup(object): def test_no_activity(self, app): user = factories.User() @@ -1138,7 +1140,7 @@ def test_no_activity(self, app): # https://github.com/ckan/ckan/pull/317 -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestFollowOrganization(object): def test_no_activity(self, app): user = factories.User() @@ -1154,7 +1156,7 @@ def test_no_activity(self, app): # https://github.com/ckan/ckan/pull/317 -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestFollowUser(object): def test_no_activity(self, app): diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index 80fe2af92e4..64b68372bf9 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -14,7 +14,7 @@ import ckan.tests.helpers as helpers -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestDelete: def test_resource_delete(self): user = factories.User() @@ -36,7 +36,7 @@ def test_resource_delete(self): @pytest.mark.ckan_config("ckan.plugins", "image_view") -@pytest.mark.usefixtures("clean_db", "with_plugins") +@pytest.mark.usefixtures("clean_db", "with_plugins", "with_request_context") class TestDeleteResourceViews(object): def test_resource_view_delete(self): resource_view = factories.ResourceView() @@ -122,7 +122,7 @@ def test_tag_delete_with_unicode_returns_unicode_error(self): assert 0, "Should have raised NotFound" -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestGroupPurge(object): def test_a_non_sysadmin_cant_purge_group(self): user = factories.User() @@ -216,7 +216,7 @@ def test_bad_id_returns_404(self): helpers.call_action("group_purge", id="123") -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestOrganizationPurge(object): def test_a_non_sysadmin_cant_purge_org(self): user = factories.User() @@ -313,7 +313,7 @@ def test_bad_id_returns_404(self): helpers.call_action("organization_purge", id="123") -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestDatasetPurge(object): def test_a_non_sysadmin_cant_purge_dataset(self): user = factories.User() @@ -439,7 +439,7 @@ def test_bad_id_returns_404(self): helpers.call_action("dataset_purge", id="123") -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestUserDelete(object): def test_user_delete(self): user = factories.User() diff --git a/ckan/tests/logic/action/test_get.py b/ckan/tests/logic/action/test_get.py index 2e296cb0c3b..eef83a6a682 100644 --- a/ckan/tests/logic/action/test_get.py +++ b/ckan/tests/logic/action/test_get.py @@ -17,7 +17,7 @@ from ckan.lib.search.common import SearchError -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestPackageShow(object): def test_package_show(self): # simple dataset, simple checks @@ -207,7 +207,7 @@ def foo(key, data, errors, context): assert "new_field" not in dataset2 -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestGroupList(object): def test_group_list(self): @@ -304,7 +304,7 @@ def test_group_list_all_fields(self): group_list = helpers.call_action("group_list", all_fields=True) - expected_group = dict(group.items()[:]) + expected_group = dict(group) for field in ("users", "tags", "extras", "groups"): del expected_group[field] @@ -391,7 +391,7 @@ def test_group_list_groups_returned(self): child_group_returned, parent_group_returned = group_list else: child_group_returned, parent_group_returned = group_list[::-1] - expected_parent_group = dict(parent_group.items()[:]) + expected_parent_group = dict(parent_group) assert [g["name"] for g in child_group_returned["groups"]] == [ expected_parent_group["name"] @@ -441,7 +441,7 @@ def test_group_list_wrong_offset(self): helpers.call_action("group_list", offset="-2") -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestGroupShow(object): def test_group_show(self): group = factories.Group(user=factories.User()) @@ -623,7 +623,7 @@ def test_package_limit_configured(self): assert len(results["packages"]) == 5 # i.e. ckan.search.rows_max -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestOrganizationList(object): def test_organization_list(self): @@ -718,7 +718,7 @@ def test_all_fields_limit_configured(self): assert len(results) == 5 # i.e. configured limit -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestOrganizationShow(object): def test_organization_show(self): org = factories.Organization() @@ -802,7 +802,7 @@ def test_package_limit_configured(self): assert len(results["packages"]) == 5 # i.e. ckan.search.rows_max -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestUserList(object): def test_user_list_default_values(self): @@ -810,7 +810,8 @@ def test_user_list_default_values(self): got_users = helpers.call_action("user_list") - assert len(got_users) == 1 + # There is one default user + assert len(got_users) == 2 got_user = got_users[0] assert got_user["id"] == user["id"] assert got_user["name"] == user["name"] @@ -837,7 +838,8 @@ def test_user_list_edits(self): got_users = helpers.call_action("user_list") - assert len(got_users) == 1 + # There is one default user + assert len(got_users) == 2 got_user = got_users[0] assert got_user["number_created_packages"] == 1 @@ -848,7 +850,8 @@ def test_user_list_excludes_deleted_users(self): got_users = helpers.call_action("user_list") - assert len(got_users) == 1 + # There is one default user + assert len(got_users) == 2 assert got_users[0]["name"] == user["name"] def test_user_list_not_all_fields(self): @@ -857,7 +860,8 @@ def test_user_list_not_all_fields(self): got_users = helpers.call_action("user_list", all_fields=False) - assert len(got_users) == 1 + # There is one default user + assert len(got_users) == 2 got_user = got_users[0] assert got_user == user["name"] @@ -875,7 +879,7 @@ def test_user_list_filtered_by_email(self): assert got_user == user_a["name"] -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestUserShow(object): def test_user_show_default_values(self): @@ -1056,7 +1060,7 @@ def test_user_show_include_datasets_includes_draft_sysadmin(self): assert got_user["number_created_packages"] == 3 -@pytest.mark.usefixtures("clean_db", "clean_index") +@pytest.mark.usefixtures("clean_db", "clean_index", "with_request_context") class TestCurrentPackageList(object): def test_current_package_list(self): """ @@ -1131,7 +1135,7 @@ def test_current_package_list_private_datasets_sysadmin_user(self): assert len(current_package_list) == 2 -@pytest.mark.usefixtures("clean_db", "clean_index") +@pytest.mark.usefixtures("clean_db", "clean_index", "with_request_context") class TestPackageAutocomplete(object): def test_package_autocomplete_does_not_return_private_datasets(self): @@ -1185,7 +1189,7 @@ def test_package_autocomplete_works_for_the_middle_part_of_title(self): assert len(package_list) == 2 -@pytest.mark.usefixtures("clean_db", "clean_index") +@pytest.mark.usefixtures("clean_db", "clean_index", "with_request_context") class TestPackageSearch(object): def test_search(self): factories.Dataset(title="Rivers") @@ -1759,7 +1763,7 @@ def test_local_parameters_not_supported(self): @pytest.mark.ckan_config("ckan.plugins", "example_idatasetform") -@pytest.mark.usefixtures("clean_db", "with_plugins") +@pytest.mark.usefixtures("clean_db", "with_plugins", "with_request_context") class TestPackageAutocompleteWithDatasetForm(object): def test_custom_schema_returned(self): dataset1 = factories.Dataset(custom_text="foo") @@ -1786,7 +1790,7 @@ def test_custom_schema_not_returned(self): assert query["results"][0]["extras"][0]["value"] == "foo" -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestBadLimitQueryParameters(object): """test class for #1258 non-int query parameters cause 500 errors @@ -1822,7 +1826,7 @@ def test_package_search_facet_field_is_json(self): helpers.call_action("package_search", **kwargs) -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestOrganizationListForUser(object): """Functional tests for the organization_list_for_user() action function.""" @@ -2353,7 +2357,7 @@ def test_config_option_list(self): """config_option_list returns whitelisted config option keys""" keys = helpers.call_action("config_option_list") - schema_keys = schema.update_configuration_schema().keys() + schema_keys = list(schema.update_configuration_schema().keys()) assert keys == schema_keys @@ -2468,7 +2472,7 @@ def test_tag_list_vocab_not_found(self): helpers.call_action("tag_list", vocabulary_id="does-not-exist") -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestMembersList(object): def test_dataset_delete_marks_membership_of_group_as_deleted(self): sysadmin = factories.Sysadmin() @@ -2591,7 +2595,7 @@ def test_user_delete_marks_membership_of_org_as_deleted(self): assert len(org_members) == 0 -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestFollow(object): def test_followee_list(self): @@ -2639,7 +2643,7 @@ def test_followee_list_with_q(self): class TestStatusShow(object): @pytest.mark.ckan_config("ckan.plugins", "stats") - @pytest.mark.usefixtures("clean_db", "with_plugins") + @pytest.mark.usefixtures("clean_db", "with_plugins", "with_request_context") def test_status_show(self): status = helpers.call_action(u"status_show") @@ -2711,7 +2715,7 @@ def _seconds_since_timestamp(timestamp, format_): return (now - dt).total_seconds() -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestActivityShow(object): def test_simple_without_data(self): dataset = factories.Dataset() @@ -2771,7 +2775,7 @@ def _clear_activities(): model.Session.flush() -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestPackageActivityList(object): def test_create_dataset(self): user = factories.User() @@ -3101,7 +3105,7 @@ def test_sysadmin_user_can_include_hidden_activities(self): ] -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestUserActivityList(object): def test_create_user(self): user = factories.User() @@ -3303,7 +3307,7 @@ def test_limit_hits_max(self): assert len(results) == 7 # i.e. ckan.activity_list_limit_max -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestGroupActivityList(object): def test_create_group(self): user = factories.User() @@ -3542,7 +3546,7 @@ def test_sysadmin_user_can_include_hidden_activities(self): ] -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestOrganizationActivityList(object): def test_create_organization(self): user = factories.User() @@ -3781,7 +3785,7 @@ def test_sysadmin_user_can_include_hidden_activities(self): ] -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestRecentlyChangedPackagesActivityList(object): def test_create_dataset(self): user = factories.User() @@ -3922,7 +3926,7 @@ def test_limit_hits_max(self): assert len(results) == 7 # i.e. ckan.activity_list_limit_max -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestDashboardActivityList(object): def test_create_user(self): user = factories.User() @@ -4031,7 +4035,7 @@ def test_limit_hits_max(self): assert len(results) == 7 # i.e. ckan.activity_list_limit_max -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestDashboardNewActivities(object): def test_users_own_activities(self): # a user's own activities are not shown as "new" diff --git a/ckan/tests/logic/action/test_patch.py b/ckan/tests/logic/action/test_patch.py index 7d958c298f1..255e4ca6d3b 100644 --- a/ckan/tests/logic/action/test_patch.py +++ b/ckan/tests/logic/action/test_patch.py @@ -5,7 +5,7 @@ from ckan.tests import helpers, factories -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestPatch(object): def test_package_patch_updating_single_field(self): user = factories.User() diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py index 8eb082a5d75..38c39dfffac 100644 --- a/ckan/tests/logic/action/test_update.py +++ b/ckan/tests/logic/action/test_update.py @@ -4,7 +4,7 @@ import mock import pytest - +import six import ckan import ckan.lib.app_globals as app_globals import ckan.logic as logic @@ -14,7 +14,7 @@ from ckan import model from ckan.common import config -from six import StringIO +from six import BytesIO from pyfakefs import fake_filesystem try: @@ -44,7 +44,7 @@ def datetime_from_string(s): return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S.%f") -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestUpdate(object): def teardown(self): # Since some of the test methods below use the mock module to patch @@ -509,7 +509,7 @@ def test_update_group_cant_change_type(self): ) -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestDatasetUpdate(object): def test_missing_id(self): user = factories.User() @@ -674,6 +674,7 @@ def test_return_id_only(self): assert updated_dataset == dataset["id"] +@pytest.mark.usefixtures("with_request_context") class TestUpdateSendEmailNotifications(object): @pytest.mark.ckan_config("ckan.activity_streams_email_notifications", True) @mock.patch("ckan.logic.action.update.request") @@ -692,7 +693,7 @@ def test_not_calling_through_paster_validates_auth(self, mock_request): @pytest.mark.ckan_config("ckan.plugins", "image_view") -@pytest.mark.usefixtures("clean_db", "with_plugins") +@pytest.mark.usefixtures("clean_db", "with_plugins", "with_request_context") class TestResourceViewUpdate(object): def test_resource_view_update(self): resource_view = factories.ResourceView() @@ -812,7 +813,7 @@ def test_calling_with_only_id_doesnt_update_anything(self): @pytest.mark.ckan_config("ckan.plugins", "image_view recline_view") -@pytest.mark.usefixtures("clean_db", "with_plugins") +@pytest.mark.usefixtures("clean_db", "with_plugins", "with_request_context") class TestResourceUpdate(object): import cgi @@ -1067,15 +1068,15 @@ def test_mimetype_by_upload_by_file(self, monkeypatch): package=dataset, url="http://localhost/data.csv", name="Test" ) - update_file = StringIO() - update_file.write( + update_file = BytesIO() + update_file.write(six.ensure_binary( """ Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54 MCGILLIVRAY PASS,1C05,1725,2015/12/31,88,239,,87,27,JAN-01,274 NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41 """ - ) + )) update_resource = TestResourceUpdate.FakeFileStorage( update_file, "update_test" ) @@ -1105,8 +1106,8 @@ def test_mimetype_by_upload_by_filename(self, monkeypatch): Real world usage would be using the FileStore API or web UI form to upload a file, with a filename plus extension If there's no url or the mimetype can't be guessed by the url, mimetype will be guessed by the extension in the filename """ - test_file = StringIO() - test_file.write( + test_file = BytesIO() + test_file.write(six.ensure_binary( """ "info": { "title": "BC Data Catalogue API", @@ -1124,7 +1125,7 @@ def test_mimetype_by_upload_by_filename(self, monkeypatch): "version": "3.0.0" } """ - ) + )) test_resource = TestResourceUpdate.FakeFileStorage( test_file, "test.json" ) @@ -1140,15 +1141,15 @@ def test_mimetype_by_upload_by_filename(self, monkeypatch): upload=test_resource, ) - update_file = StringIO() - update_file.write( + update_file = BytesIO() + update_file.write(six.ensure_binary( """ Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54 MCGILLIVRAY PASS,1C05,1725,2015/12/31,88,239,,87,27,JAN-01,274 NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41 """ - ) + )) update_resource = TestResourceUpdate.FakeFileStorage( update_file, "update_test.csv" ) @@ -1199,8 +1200,8 @@ def test_size_of_resource_by_upload(self, monkeypatch): """ The size of the resource determined by the uploaded file """ - test_file = StringIO() - test_file.write( + test_file = BytesIO() + test_file.write(six.ensure_binary( """ "info": { "title": "BC Data Catalogue API", @@ -1218,7 +1219,7 @@ def test_size_of_resource_by_upload(self, monkeypatch): "version": "3.0.0" } """ - ) + )) test_resource = TestResourceUpdate.FakeFileStorage( test_file, "test.json" ) @@ -1234,15 +1235,15 @@ def test_size_of_resource_by_upload(self, monkeypatch): upload=test_resource, ) - update_file = StringIO() - update_file.write( + update_file = BytesIO() + update_file.write(six.ensure_binary( """ Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54 MCGILLIVRAY PASS,1C05,1725,2015/12/31,88,239,,87,27,JAN-01,274 NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41 """ - ) + )) update_resource = TestResourceUpdate.FakeFileStorage( update_file, "update_test.csv" ) @@ -1378,7 +1379,7 @@ def test_resource_format_update(self): assert len(res_views) == 1 -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestConfigOptionUpdate(object): # NOTE: the opposite is tested in @@ -1399,7 +1400,7 @@ def test_app_globals_set_if_defined(self): assert getattr(app_globals.app_globals, globals_key) == value -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestUserUpdate(object): def test_user_update_with_password_hash(self): sysadmin = factories.Sysadmin() @@ -1418,7 +1419,7 @@ def test_user_update_with_password_hash(self): def test_user_create_password_hash_not_for_normal_users(self): normal_user = factories.User() - context = {"user": normal_user["name"]} + context = {"user": normal_user["name"], "ignore_auth": False} user = helpers.call_action( "user_update", @@ -1433,7 +1434,7 @@ def test_user_create_password_hash_not_for_normal_users(self): assert user_obj.password != "pretend-this-is-a-valid-hash" -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestPackageOwnerOrgUpdate(object): def test_package_owner_org_added(self): """A package without an owner_org can have one added.""" @@ -1486,7 +1487,7 @@ def test_package_owner_org_removed(self): assert dataset_obj.owner_org is None -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestBulkOperations(object): def test_bulk_make_private(self): @@ -1581,7 +1582,7 @@ def test_bulk_delete(self): assert dataset.state == "deleted" -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestDashboardMarkActivitiesOld(object): def test_mark_as_old_some_activities_by_a_followed_user(self): # do some activity that will show up on user's dashboard diff --git a/ckan/tests/logic/auth/test_create.py b/ckan/tests/logic/auth/test_create.py index 0541c92f1a1..ab055468cbc 100644 --- a/ckan/tests/logic/auth/test_create.py +++ b/ckan/tests/logic/auth/test_create.py @@ -41,7 +41,7 @@ def test_cud_overrides_acd(): assert not response["success"] -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestRealUsersAuth(object): def test_no_org_user_can_create(self): user = factories.User() diff --git a/ckan/tests/logic/auth/test_delete.py b/ckan/tests/logic/auth/test_delete.py index 77845a251da..e364abade88 100644 --- a/ckan/tests/logic/auth/test_delete.py +++ b/ckan/tests/logic/auth/test_delete.py @@ -13,7 +13,7 @@ logic = helpers.logic -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestDeleteAuth: def test_anon_cant_delete(self): context = {"user": None, "model": model} @@ -110,6 +110,7 @@ def test_anon_cant_clear(): helpers.call_auth("resource_view_clear", context=context, **params) +@pytest.mark.usefixtures("with_request_context") def test_normal_user_cant_clear(): user = factories.User() @@ -119,6 +120,7 @@ def test_normal_user_cant_clear(): helpers.call_auth("resource_view_clear", context=context) +@pytest.mark.usefixtures("with_request_context") def test_sysadmin_user_can_clear(): user = factories.User(sysadmin=True) diff --git a/ckan/tests/logic/auth/test_get.py b/ckan/tests/logic/auth/test_get.py index a5e3fbb33f7..ac27d525c2d 100644 --- a/ckan/tests/logic/auth/test_get.py +++ b/ckan/tests/logic/auth/test_get.py @@ -30,7 +30,7 @@ def test_user_list_email_parameter(): helpers.call_auth("user_list", email="a@example.com", context=context) -@pytest.mark.usefixtures(u"clean_db") +@pytest.mark.usefixtures(u"clean_db", "with_request_context") class TestGetAuth(object): @pytest.mark.ckan_config(u"ckan.auth.public_user_details", u"false") def test_auth_user_show(self): diff --git a/ckan/tests/logic/auth/test_init.py b/ckan/tests/logic/auth/test_init.py index eb37ef2b85e..b4f11933f62 100644 --- a/ckan/tests/logic/auth/test_init.py +++ b/ckan/tests/logic/auth/test_init.py @@ -90,7 +90,7 @@ def test_get_group_object_id_none(): _get_object_id_none("group") -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestInit(object): def test_get_package_object_with_id(self): diff --git a/ckan/tests/logic/auth/test_update.py b/ckan/tests/logic/auth/test_update.py index 1f4cc471870..1140ff4c6ed 100644 --- a/ckan/tests/logic/auth/test_update.py +++ b/ckan/tests/logic/auth/test_update.py @@ -11,6 +11,7 @@ import ckan.tests.helpers as helpers +@pytest.mark.usefixtures("with_request_context") def test_user_update_visitor_cannot_update_user(): """Visitors should not be able to update users' accounts.""" @@ -39,6 +40,7 @@ def test_user_update_visitor_cannot_update_user(): # START-AFTER +@pytest.mark.usefixtures("with_request_context") def test_user_update_user_cannot_update_another_user(): """Users should not be able to update other users' accounts.""" @@ -75,6 +77,7 @@ def test_user_update_user_cannot_update_another_user(): # END-BEFORE +@pytest.mark.usefixtures("with_request_context") def test_user_update_user_can_update_her(): """Users should be authorized to update their own accounts.""" @@ -125,6 +128,7 @@ def test_user_update_with_no_user_in_context(): helpers.call_auth("user_update", context=context, **params) +@pytest.mark.usefixtures("with_request_context") def test_user_generate_own_apikey(): fred = factories.MockUser(name="fred") mock_model = mock.MagicMock() @@ -141,6 +145,7 @@ def test_user_generate_own_apikey(): assert result is True +@pytest.mark.usefixtures("with_request_context") def test_user_generate_apikey_without_logged_in_user(): fred = factories.MockUser(name="fred") mock_model = mock.MagicMock() @@ -153,6 +158,7 @@ def test_user_generate_apikey_without_logged_in_user(): helpers.call_auth("user_generate_apikey", context=context, **params) +@pytest.mark.usefixtures("with_request_context") def test_user_generate_apikey_for_another_user(): fred = factories.MockUser(name="fred") bob = factories.MockUser(name="bob") @@ -169,7 +175,7 @@ def test_user_generate_apikey_for_another_user(): @pytest.mark.ckan_config("ckan.plugins", "image_view") -@pytest.mark.usefixtures("clean_db", "with_plugins") +@pytest.mark.usefixtures("clean_db", "with_plugins", "with_request_context") class TestUpdateWithView(object): def test_anon_can_not_update(self): @@ -245,7 +251,7 @@ def test_not_authorized_if_user_has_no_permissions_on_dataset(self): ) -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") class TestUpdate(object): def test_config_option_update_anon_user(self): """An anon user is not authorized to use config_option_update diff --git a/ckan/tests/logic/test_validators.py b/ckan/tests/logic/test_validators.py index 160e44dc697..ca9b016975d 100644 --- a/ckan/tests/logic/test_validators.py +++ b/ckan/tests/logic/test_validators.py @@ -656,7 +656,7 @@ def test_user_id_or_name_exists_empty(): validators.user_id_or_name_exists("", _make_context()) -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") def test_user_id_or_name_exists(): user = factories.User(name="username") v = validators.user_id_or_name_exists(user["id"], _make_context()) @@ -670,7 +670,7 @@ def test_group_id_or_name_exists_empty(): validators.user_id_or_name_exists("", _make_context()) -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", "with_request_context") def test_group_id_or_name_exists(): group = factories.Group() v = validators.group_id_or_name_exists(group["id"], _make_context()) From 671057a95b37371bf858f38b2bdf04b2000572d3 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Jan 2020 19:56:14 +0200 Subject: [PATCH 03/10] Finish ckan tests --- ckan/model/user.py | 10 +++++----- ckan/tests/model/test_license.py | 4 ++-- ckan/tests/model/test_package.py | 2 +- ckan/tests/model/test_user.py | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ckan/model/user.py b/ckan/model/user.py index 26e90513e44..23884cbf24e 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -109,12 +109,12 @@ def _get_password(self): return self._password def _verify_and_upgrade_from_sha1(self, password): - if isinstance(password, text_type): - password_8bit = password.encode('ascii', 'ignore') - else: - password_8bit = password + # if isinstance(password, text_type): + # password_8bit = password.encode('ascii', 'ignore') + # else: + # password_8bit = password - hashed_pass = sha1(password_8bit + self.password[:40]) + hashed_pass = sha1(six.ensure_binary(password + self.password[:40])) current_hash = passlib.utils.to_native_str(self.password[40:]) if passlib.utils.consteq(hashed_pass.hexdigest(), current_hash): diff --git a/ckan/tests/model/test_license.py b/ckan/tests/model/test_license.py index 24d9add866f..dc8c437446c 100644 --- a/ckan/tests/model/test_license.py +++ b/ckan/tests/model/test_license.py @@ -56,7 +56,7 @@ def test_import_v2_style_register(): assert license.title == "Creative Commons Attribution 4.0" -@pytest.mark.usefixtures("reset") +@pytest.mark.usefixtures("reset", "with_request_context") @pytest.mark.ckan_config( "licenses_group_url", "file:///%s/licenses.v1" % this_dir ) @@ -69,7 +69,7 @@ def test_import_v1_style_register_i18n(app): assert "Altres (Oberta)" in resp.body -@pytest.mark.usefixtures("reset") +@pytest.mark.usefixtures("reset", "with_request_context") @pytest.mark.ckan_config( "licenses_group_url", "file:///%s/licenses.v2" % this_dir ) diff --git a/ckan/tests/model/test_package.py b/ckan/tests/model/test_package.py index 985223ed4c9..2e57c1f83cf 100644 --- a/ckan/tests/model/test_package.py +++ b/ckan/tests/model/test_package.py @@ -6,7 +6,7 @@ from ckan.tests import factories -@pytest.mark.usefixtures(u"clean_db") +@pytest.mark.usefixtures(u"clean_db", u"with_request_context") class TestPackage(object): def test_create(self): # Demonstrate creating a package. diff --git a/ckan/tests/model/test_user.py b/ckan/tests/model/test_user.py index c028b15b5e1..8d398b7e757 100644 --- a/ckan/tests/model/test_user.py +++ b/ckan/tests/model/test_user.py @@ -18,13 +18,13 @@ def _set_password(password): This is needed to create old password hashes in the tests """ - if isinstance(password, text_type): - password_8bit = password.encode("ascii", "ignore") - else: - password_8bit = password + # if isinstance(password, text_type): + # password_8bit = password.encode("ascii", "ignore") + # else: + # password_8bit = password salt = hashlib.sha1(os.urandom(60)) - hash = hashlib.sha1(password_8bit + salt.hexdigest()) + hash = hashlib.sha1(six.ensure_binary(password + salt.hexdigest())) hashed_password = salt.hexdigest() + hash.hexdigest() if not isinstance(hashed_password, text_type): @@ -32,7 +32,7 @@ def _set_password(password): return hashed_password -@pytest.mark.usefixtures("clean_db") +@pytest.mark.usefixtures("clean_db", u"with_request_context") class TestUser: def test_upgrade_from_sha(self): user = factories.User() From edc0fd99dc5f09d011d95b2225ce1dc3b05ff5e3 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Jan 2020 20:35:05 +0200 Subject: [PATCH 04/10] Enable py3 workflow --- .circleci/config.yml | 2 +- .../api/test_email_notifications.py | 10 +-- ckan/tests/model/test_user.py | 4 +- ckanext/datastore/tests/test_plugin.py | 10 +-- ckanext/example_iuploader/test/test_plugin.py | 4 +- ckanext/multilingual/plugin.py | 2 +- .../tests/test_multilingual_plugin.py | 4 +- ckanext/reclineview/tests/test_view.py | 70 +++++++------------ ckanext/resourceproxy/tests/test_proxy.py | 47 +++++-------- ckanext/stats/tests/test_stats_lib.py | 20 ++---- ckanext/textview/tests/test_view.py | 31 ++++---- 11 files changed, 74 insertions(+), 130 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3da3f16b8e..e833e408511 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -143,4 +143,4 @@ workflows: build_and_test: jobs: - test-python-2 - # - test-python-3 + - test-python-3 diff --git a/ckan/tests/legacy/functional/api/test_email_notifications.py b/ckan/tests/legacy/functional/api/test_email_notifications.py index eb9ee963a9b..072a89b8dbb 100644 --- a/ckan/tests/legacy/functional/api/test_email_notifications.py +++ b/ckan/tests/legacy/functional/api/test_email_notifications.py @@ -1,6 +1,7 @@ # encoding: utf-8 import time +import pytest import ckan.model as model import ckan.tests.legacy as tests @@ -419,11 +420,10 @@ def test_00_email_notifications_disabled_by_default(self, mail_server): assert len(mail_server.get_smtp_messages()) == 0 -class TestEmailNotificationsIniSetting(ControllerTestCase): +class TestEmailNotificationsIniSetting(object): """Tests for the ckan.activity_streams_email_notifications config setting. """ - @classmethod def setup_class(cls): @@ -441,7 +441,7 @@ def setup_class(cls): "apikey": testsysadmin.apikey, } - @helpers.change_config("ckan.activity_streams_email_notifications", False) + @pytest.mark.ckan_config("ckan.activity_streams_email_notifications", False) def test_00_send_email_notifications_feature_disabled(self, mail_server): """Send_email_notifications API should error when feature disabled.""" @@ -506,7 +506,7 @@ def test_00_send_email_notifications_feature_disabled(self, mail_server): status=409, ) - @helpers.change_config("ckan.activity_streams_email_notifications", False) + @pytest.mark.ckan_config("ckan.activity_streams_email_notifications", False) def test_01_no_emails_sent_if_turned_off(self, mail_server): """No emails should be sent if the feature is disabled site-wide.""" @@ -534,7 +534,7 @@ def setup_class(cls): # Don't send email notifications for activities older than 1 # microsecond - @helpers.change_config("ckan.email_notifications_since", ".000001") + @pytest.mark.ckan_config("ckan.email_notifications_since", ".000001") def test_00_email_notifications_since(self, mail_server): """No emails should be sent for activities older than email_notifications_since. diff --git a/ckan/tests/model/test_user.py b/ckan/tests/model/test_user.py index 8d398b7e757..fa8d6da81e4 100644 --- a/ckan/tests/model/test_user.py +++ b/ckan/tests/model/test_user.py @@ -19,9 +19,9 @@ def _set_password(password): This is needed to create old password hashes in the tests """ # if isinstance(password, text_type): - # password_8bit = password.encode("ascii", "ignore") + # password_8bit = password.encode("ascii", "ignore") # else: - # password_8bit = password + # password_8bit = password salt = hashlib.sha1(os.urandom(60)) hash = hashlib.sha1(six.ensure_binary(password + salt.hexdigest())) diff --git a/ckanext/datastore/tests/test_plugin.py b/ckanext/datastore/tests/test_plugin.py index 24cf9a5ab8a..ffac6674db4 100644 --- a/ckanext/datastore/tests/test_plugin.py +++ b/ckanext/datastore/tests/test_plugin.py @@ -61,15 +61,9 @@ def test_loading_datastore_last_doesnt_work(self): ) +@pytest.mark.ckan_config("ckan.plugins", "datastore") +@pytest.mark.usefixtures("clean_index", "with_plugins", "with_request_context") class TestPluginDatastoreSearch(object): - @classmethod - def setup_class(cls): - p.load("datastore") - - @classmethod - def teardown_class(cls): - p.unload("datastore") - @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", None) def test_english_is_default_fts_language(self): expected_ts_query = ", plainto_tsquery('english', 'foo') \"query\"" diff --git a/ckanext/example_iuploader/test/test_plugin.py b/ckanext/example_iuploader/test/test_plugin.py index fa5d64dfbda..1f68ed290da 100644 --- a/ckanext/example_iuploader/test/test_plugin.py +++ b/ckanext/example_iuploader/test/test_plugin.py @@ -25,7 +25,7 @@ fs = fake_filesystem.FakeFilesystem() fake_os = fake_filesystem.FakeOsModule(fs) fake_open = fake_filesystem.FakeFileOpen(fs) -CONTENT = b"data" +CONTENT = "data" def mock_open_if_open_fails(*args, **kwargs): @@ -43,7 +43,7 @@ def mock_open_if_open_fails(*args, **kwargs): # open) @pytest.mark.ckan_config("ckan.plugins", "example_iuploader") @pytest.mark.ckan_config("ckan.webassets.path", "/tmp/webassets") -@pytest.mark.usefixtures("with_plugins", "clean_db") +@pytest.mark.usefixtures("with_plugins", "clean_db", "with_request_context") @patch.object(ckan.lib.uploader, "os", fake_os) @patch.object(flask, "send_file", side_effect=[CONTENT]) @patch.object(config["pylons.h"], "uploads_enabled", return_value=True) diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py index 1175a39a228..f72823ae3f4 100644 --- a/ckanext/multilingual/plugin.py +++ b/ckanext/multilingual/plugin.py @@ -226,7 +226,7 @@ def before_index(self, search_data): ## translate rest all_terms = [] - for key, value in six.iteritems(search_data): + for key, value in sorted(six.iteritems(search_data)): if key in KEYS_TO_IGNORE or key.startswith('title'): continue if not isinstance(value, list): diff --git a/ckanext/multilingual/tests/test_multilingual_plugin.py b/ckanext/multilingual/tests/test_multilingual_plugin.py index c9ab5ddc119..c331b1f7cda 100644 --- a/ckanext/multilingual/tests/test_multilingual_plugin.py +++ b/ckanext/multilingual/tests/test_multilingual_plugin.py @@ -190,7 +190,7 @@ def test_translate_terms(self): 'text_de': '', 'text_pt_BR': '', u'title_fr': u'french david', - 'text_fr': u'french note french moon french boon french_moo', + 'text_fr': u'french_moo french note french moon french boon', 'text_ja': '', 'text_sr': '', 'title': u'david', @@ -208,7 +208,7 @@ def test_translate_terms(self): 'tags': [u'moon', 'boon'], 'text_el': '', 'title_en': u'david', - 'text_en': u'an interesting note moon boon moo', + 'text_en': u'moo an interesting note moon boon', 'text_es': '', 'text_sl': '', 'text_pl': '', diff --git a/ckanext/reclineview/tests/test_view.py b/ckanext/reclineview/tests/test_view.py index 48f162645ed..0e71193af10 100644 --- a/ckanext/reclineview/tests/test_view.py +++ b/ckanext/reclineview/tests/test_view.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from ckan.tests.helpers import _get_test_app +import pytest from ckan.common import config import ckan.model as model @@ -12,27 +12,16 @@ from ckan.tests import helpers, factories -class BaseTestReclineViewBase(): - @classmethod - def setup_class(cls): - cls.config_templates = config['ckan.legacy_templates'] - config['ckan.legacy_templates'] = 'false' - - cls.app = _get_test_app() - p.load(cls.view_type) - - cls.p = cls.view_class() +@pytest.mark.ckan_config('ckan.legacy_templates', 'false') +@pytest.mark.usefixtures("with_plugins", "with_request_context") +class BaseTestReclineViewBase(object): + @pytest.fixture(autouse=True) + def initial_data(self, clean_db, with_request_context): create_test_data.CreateTestData.create() - - cls.resource_view, cls.package, cls.resource_id = \ - _create_test_view(cls.view_type) - - @classmethod - def teardown_class(cls): - config['ckan.legacy_templates'] = cls.config_templates - p.unload(cls.view_type) - model.repo.rebuild_db() + self.p = self.view_class() + self.resource_view, self.package, self.resource_id = \ + _create_test_view(self.view_type) def test_can_view(self): data_dict = {'resource': {'datastore_active': True}} @@ -41,15 +30,16 @@ def test_can_view(self): data_dict = {'resource': {'datastore_active': False}} assert not self.p.can_view(data_dict) - def test_title_description_iframe_shown(self): + def test_title_description_iframe_shown(self, app): url = h.url_for('resource.read', id=self.package.name, resource_id=self.resource_id) - result = self.app.get(url) + result = app.get(url) assert self.resource_view['title'] in result assert self.resource_view['description'] in result assert 'data-module="data-viewer"' in result.body +@pytest.mark.ckan_config('ckan.plugins', 'recline_view') class TestReclineView(BaseTestReclineViewBase): view_type = 'recline_view' view_class = plugin.ReclineView @@ -80,28 +70,13 @@ def test_can_view_bad_format_no_datastore(self): assert not self.p.can_view(data_dict) -class TestReclineViewDatastoreOnly(helpers.FunctionalTestBase): - - @classmethod - def setup_class(cls): - cls.app = _get_test_app() - if not p.plugin_loaded('recline_view'): - p.load('recline_view') - if not p.plugin_loaded('datastore'): - p.load('datastore') - app_config = config.copy() - app_config['ckan.legacy_templates'] = 'false' - app_config['ckan.plugins'] = 'recline_view datastore' - app_config['ckan.views.default_views'] = 'recline_view' - - @classmethod - def teardown_class(cls): - if p.plugin_loaded('recline_view'): - p.unload('recline_view') - if p.plugin_loaded('datastore'): - p.unload('datastore') - - def test_create_datastore_only_view(self): +@pytest.mark.ckan_config('ckan.legacy_templates', 'false') +@pytest.mark.ckan_config('ckan.plugins', 'recline_view datastore') +@pytest.mark.ckan_config('ckan.views.default_views', 'recline_view') +@pytest.mark.usefixtures("clean_db", "with_plugins") +class TestReclineViewDatastoreOnly(object): + + def test_create_datastore_only_view(self, app): dataset = factories.Dataset() data = { 'resource': {'package_id': dataset['id']}, @@ -115,11 +90,12 @@ def test_create_datastore_only_view(self): url = h.url_for('resource.read', id=dataset['id'], resource_id=resource_id) - result = self.app.get(url) + result = app.get(url) assert 'data-module="data-viewer"' in result.body +@pytest.mark.ckan_config('ckan.plugins', 'recline_grid_view') class TestReclineGridView(BaseTestReclineViewBase): view_type = 'recline_grid_view' view_class = plugin.ReclineGridView @@ -129,6 +105,7 @@ def test_it_has_no_schema(self): assert schema is None, schema +@pytest.mark.ckan_config('ckan.plugins', 'recline_graph_view') class TestReclineGraphView(BaseTestReclineViewBase): view_type = 'recline_graph_view' view_class = plugin.ReclineGraphView @@ -139,6 +116,7 @@ def test_it_has_the_correct_schema_keys(self): _assert_schema_exists_and_has_keys(schema, expected_keys) +@pytest.mark.ckan_config('ckan.plugins', 'recline_map_view') class TestReclineMapView(BaseTestReclineViewBase): view_type = 'recline_map_view' view_class = plugin.ReclineMapView @@ -170,7 +148,7 @@ def _create_test_view(view_type): def _assert_schema_exists_and_has_keys(schema, expected_keys): assert schema is not None, schema - keys = schema.keys() + keys = list(schema.keys()) keys.sort() expected_keys.sort() diff --git a/ckanext/resourceproxy/tests/test_proxy.py b/ckanext/resourceproxy/tests/test_proxy.py index d2770ef8692..966f120635b 100644 --- a/ckanext/resourceproxy/tests/test_proxy.py +++ b/ckanext/resourceproxy/tests/test_proxy.py @@ -1,7 +1,7 @@ # encoding: utf-8 +import pytest import requests -import unittest import json import responses import six @@ -46,27 +46,15 @@ def set_resource_url(url): return {'resource': resource, 'package': package} -class TestProxyPrettyfied(unittest.TestCase): +@pytest.mark.ckan_config('ckan.plugins', 'resource_proxy') +@pytest.mark.usefixtures("clean_db", "with_plugins", "with_request_context") +class TestProxyPrettyfied(object): serving = False - @classmethod - def setup_class(cls): - cls._original_config = config.copy() - if not p.plugin_loaded('resource_proxy'): - p.load('resource_proxy') - config['ckan.plugins'] = 'resource_proxy' - cls.app = _get_test_app() - + @pytest.fixture(autouse=True) + def initial_data(self, clean_db, with_request_context): create_test_data.CreateTestData.create() - - @classmethod - def teardown_class(cls): - config.clear() - config.update(cls._original_config) - model.repo.rebuild_db() - - def setUp(self): self.url = 'http://www.ckan.org/static/example.json' self.data_dict = set_resource_url(self.url) @@ -87,7 +75,7 @@ def test_resource_proxy_on_200(self): assert "yes, I'm proxied" in result.content, result.content @responses.activate - def test_resource_proxy_on_404(self): + def test_resource_proxy_on_404(self, app): self.mock_out_urls( self.url, body="I'm not here", @@ -99,14 +87,14 @@ def test_resource_proxy_on_404(self): assert result.status_code == 404, result.status_code proxied_url = proxy.get_proxified_resource_url(self.data_dict) - result = self.app.get(proxied_url) + result = app.get(proxied_url) # we expect a 409 because the resourceproxy got an error (404) # from the server assert result.status_code == 409 assert '404' in result.data @responses.activate - def test_large_file(self): + def test_large_file(self, app): cl = blueprint.MAX_FILE_SIZE + 1 self.mock_out_urls( self.url, @@ -114,12 +102,12 @@ def test_large_file(self): body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) - result = self.app.get(proxied_url) + result = app.get(proxied_url) assert result.status_code == 409 assert six.b('too large') in result.data @responses.activate - def test_large_file_streaming(self): + def test_large_file_streaming(self, app): cl = blueprint.MAX_FILE_SIZE + 1 self.mock_out_urls( self.url, @@ -127,31 +115,32 @@ def test_large_file_streaming(self): body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) - result = self.app.get(proxied_url) + result = app.get(proxied_url) assert result.status_code == 409 assert six.b('too large') in result.data @responses.activate - def test_invalid_url(self): + def test_invalid_url(self, app): responses.add_passthru(config['solr_url']) self.data_dict = set_resource_url('http:invalid_url') proxied_url = proxy.get_proxified_resource_url(self.data_dict) - result = self.app.get(proxied_url) + result = app.get(proxied_url) assert result.status_code == 409 assert six.b('Invalid URL') in result.data - def test_non_existent_url(self): + def test_non_existent_url(self, app): self.data_dict = set_resource_url('http://nonexistent.example.com') def f1(): url = self.data_dict['resource']['url'] requests.get(url) - self.assertRaises(requests.ConnectionError, f1) + with pytest.raises(requests.ConnectionError): + f1() proxied_url = proxy.get_proxified_resource_url(self.data_dict) - result = self.app.get(proxied_url) + result = app.get(proxied_url) assert result.status_code == 502 assert six.b('connection error') in result.data diff --git a/ckanext/stats/tests/test_stats_lib.py b/ckanext/stats/tests/test_stats_lib.py index bb862b71029..9e94cb99977 100644 --- a/ckanext/stats/tests/test_stats_lib.py +++ b/ckanext/stats/tests/test_stats_lib.py @@ -10,14 +10,11 @@ from ckanext.stats.tests import StatsFixture -class TestStatsPlugin(StatsFixture): - @classmethod - def setup_class(cls): - - super(TestStatsPlugin, cls).setup_class() - - model.repo.rebuild_db() - +@pytest.mark.ckan_config('ckan.plugins', 'stats') +@pytest.mark.usefixtures("with_plugins", "with_request_context") +class TestStatsPlugin(object): + @pytest.fixture(autouse=True) + def initial_data(self, clean_db, with_request_context): user = factories.User(name="bob") org_users = [{"name": user["name"], "capacity": "editor"}] org1 = factories.Organization(name="org1", users=org_users) @@ -58,13 +55,6 @@ def setup_class(cls): model.Package.by_name(u"test3").notes = "Test 3 notes" model.repo.commit_and_remove() - @classmethod - def teardown_class(cls): - - model.repo.rebuild_db() - - model.Session.remove() - def test_top_rated_packages(self): pkgs = Stats.top_rated_packages() assert pkgs == [] diff --git a/ckanext/textview/tests/test_view.py b/ckanext/textview/tests/test_view.py index dd97b088996..9057ab1d6b8 100644 --- a/ckanext/textview/tests/test_view.py +++ b/ckanext/textview/tests/test_view.py @@ -1,4 +1,6 @@ # encoding: utf-8 + +import pytest from ckan.common import config from six.moves.urllib.parse import urljoin @@ -27,29 +29,19 @@ def _create_test_view(view_type): return resource_view, package, resource_id +@pytest.mark.ckan_config('ckan.plugins', 'text_view') +@pytest.mark.usefixtures("with_plugins") class TestTextView(object): view_type = 'text_view' - @classmethod - def setup_class(cls): - - if not plugins.plugin_loaded('text_view'): - plugins.load('text_view') - - if plugins.plugin_loaded('resource_proxy'): - plugins.unload('resource_proxy') - - cls.p = plugin.TextView() + @pytest.fixture(autouse=True) + def initial_data(self, clean_db, with_request_context): + self.p = plugin.TextView() create_test_data.CreateTestData.create() - cls.resource_view, cls.package, cls.resource_id = \ - _create_test_view(cls.view_type) - - @classmethod - def teardown_class(cls): - plugins.unload('text_view') - model.repo.rebuild_db() + self.resource_view, self.package, self.resource_id = \ + _create_test_view(self.view_type) def test_can_view(self): url_same_domain = urljoin( @@ -75,6 +67,7 @@ def test_can_view(self): data_dict = {'resource': {'format': 'json', 'url': url_different_domain}} + assert not self.p.can_view(data_dict) def test_title_description_iframe_shown(self): @@ -106,8 +99,8 @@ def test_js_included(self): id=self.package.name, resource_id=self.resource_id, view_id=self.resource_view['id']) result = app.get(url) - assert (('text_view.js' in result.data) or # Source file - ('textview.js' in result.data)) # Compiled file + assert (('text_view.js' in result.body) or # Source file + ('textview.js' in result.body)) # Compiled file # Restore the config to its original values config.clear() config.update(original_config) From e87e8ce1c8f8b9a93c5fbb8ff6364003328f67da Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 3 Jan 2020 08:48:21 +0200 Subject: [PATCH 05/10] Fix resourceproxy tests --- ckanext/resourceproxy/tests/test_proxy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ckanext/resourceproxy/tests/test_proxy.py b/ckanext/resourceproxy/tests/test_proxy.py index 966f120635b..99d359257ed 100644 --- a/ckanext/resourceproxy/tests/test_proxy.py +++ b/ckanext/resourceproxy/tests/test_proxy.py @@ -67,18 +67,18 @@ def test_resource_proxy_on_200(self): self.mock_out_urls( self.url, content_type='application/json', - body=JSON_STRING) + body=six.ensure_binary(JSON_STRING)) url = self.data_dict['resource']['url'] result = requests.get(url) assert result.status_code == 200, result.status_code - assert "yes, I'm proxied" in result.content, result.content + assert "yes, I'm proxied" in six.ensure_str(result.content) @responses.activate def test_resource_proxy_on_404(self, app): self.mock_out_urls( self.url, - body="I'm not here", + body=six.ensure_binary("I'm not here"), content_type='application/json', status=404) @@ -91,7 +91,7 @@ def test_resource_proxy_on_404(self, app): # we expect a 409 because the resourceproxy got an error (404) # from the server assert result.status_code == 409 - assert '404' in result.data + assert '404' in result.body @responses.activate def test_large_file(self, app): From fac2a16ba7036a3ae98820573f61ef1f1dddf1bf Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 3 Jan 2020 14:48:44 +0200 Subject: [PATCH 06/10] Update xfails --- ckan/cli/cli.py | 6 ++++++ ckan/cli/datastore.py | 2 +- ckan/cli/seed.py | 16 ++++++++-------- ckan/cli/user.py | 4 ++-- ckan/cli/views.py | 4 ++-- ckan/lib/search/index.py | 2 +- ckan/tests/controllers/test_group.py | 6 ++---- ckan/tests/controllers/test_package.py | 1 - ckan/tests/controllers/test_user.py | 4 ++-- ckan/tests/legacy/lib/test_solr_search_index.py | 1 - ckan/views/__init__.py | 7 +++++-- ckanext/datastore/tests/test_search.py | 14 +------------- 12 files changed, 30 insertions(+), 37 deletions(-) diff --git a/ckan/cli/cli.py b/ckan/cli/cli.py index cf9d97e865e..2bb7c3a1088 100644 --- a/ckan/cli/cli.py +++ b/ckan/cli/cli.py @@ -1,6 +1,7 @@ # encoding: utf-8 import logging +import six from collections import defaultdict import ckan.plugins as p @@ -73,6 +74,11 @@ def __init__(self, conf=None): @click.pass_context def ckan(ctx, config, *args, **kwargs): ctx.obj = CkanCommand(config) + if six.PY2: + ctx.meta["flask_app"] = ctx.obj.app.apps["flask_app"]._wsgi_app + else: + ctx.meta["flask_app"] = ctx.obj.app._wsgi_app + for plugin in p.PluginImplementations(p.IClick): for cmd in plugin.get_commands(): cmd._ckanext = plugin.name diff --git a/ckan/cli/datastore.py b/ckan/cli/datastore.py index 4803777e231..06830992268 100644 --- a/ckan/cli/datastore.py +++ b/ckan/cli/datastore.py @@ -88,7 +88,7 @@ def permissions_sql(maindb, datastoredb, mainuser, writeuser, readuser): def dump(ctx, resource_id, output_file, format, offset, limit, bom): u'''Dump a datastore resource. ''' - flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): dump_to( resource_id, diff --git a/ckan/cli/seed.py b/ckan/cli/seed.py index 1065eb23e8e..764c916957f 100644 --- a/ckan/cli/seed.py +++ b/ckan/cli/seed.py @@ -21,7 +21,7 @@ def seed(): @seed.command(short_help=u'Annakarenina and warandpeace.') @click.pass_context def basic(ctx): - flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): CreateTestData.create_basic_test_data() @@ -29,7 +29,7 @@ def basic(ctx): @seed.command(short_help=u'Realistic data to test search.') @click.pass_context def search(ctx): - flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): CreateTestData.create_search_test_data() @@ -37,7 +37,7 @@ def search(ctx): @seed.command(short_help=u'Government style data.') @click.pass_context def gov(ctx): - flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): CreateTestData.create_gov_test_data() @@ -45,7 +45,7 @@ def gov(ctx): @seed.command(short_help=u'Package relationships data.') @click.pass_context def family(ctx): - flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): CreateTestData.create_family_test_data() @@ -53,7 +53,7 @@ def family(ctx): @seed.command(short_help=u'Create a user "tester" with api key "tester".') @click.pass_context def user(ctx): - flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): CreateTestData.create_test_user() click.echo( @@ -64,7 +64,7 @@ def user(ctx): @seed.command(short_help=u'Test translations of terms.') @click.pass_context def translations(ctx): - flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): CreateTestData.create_translations_test_data() @@ -72,7 +72,7 @@ def translations(ctx): @seed.command(short_help=u'Some test vocabularies.') @click.pass_context def vocabs(ctx): - flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): CreateTestData.create_vocabs_test_data() @@ -80,6 +80,6 @@ def vocabs(ctx): @seed.command(short_help=u'Hierarchy of groups.') @click.pass_context def hierarchy(ctx): - flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): CreateTestData.create_group_hierarchy_test_data() diff --git a/ckan/cli/user.py b/ckan/cli/user.py index 8581a4338ce..e48dba2c778 100644 --- a/ckan/cli/user.py +++ b/ckan/cli/user.py @@ -4,6 +4,7 @@ import sys from pprint import pprint +import six import click from six import text_type @@ -102,10 +103,9 @@ def remove_user(ctx, username): error_shout(u'Please specify the username to be removed') return - flask_app = ctx.obj.app.apps['flask_app']._wsgi_app site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {}) context = {u'user': site_user[u'name']} - with flask_app.test_request_context(): + with ctx.meta['flask_app'].test_request_context(): plugin.toolkit.get_action(u'user_delete')(context, {u'id': username}) click.secho(u'Deleted user: %s' % username, fg=u'green', bold=True) diff --git a/ckan/cli/views.py b/ckan/cli/views.py index a102baff0eb..cf65047efe3 100644 --- a/ckan/cli/views.py +++ b/ckan/cli/views.py @@ -47,7 +47,7 @@ def create(ctx, types, dataset, no_default_filters, search, yes): u"datastore" in p.toolkit.config[u"ckan.plugins"].split() ) - flask_app = ctx.obj.app.apps[u"flask_app"]._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): loaded_view_plugins = _get_view_plugins(types, datastore_enabled) if loaded_view_plugins is None: @@ -148,7 +148,7 @@ def clean(ctx, yes): """ names = [] - flask_app = ctx.obj.app.apps[u"flask_app"]._wsgi_app + flask_app = ctx.meta['flask_app'] with flask_app.test_request_context(): for plugin in p.PluginImplementations(p.IResourceView): names.append(str(plugin.info()[u"name"])) diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py index fcfad79c8b4..cf78f094dfb 100644 --- a/ckan/lib/search/index.py +++ b/ckan/lib/search/index.py @@ -303,7 +303,7 @@ def index_package(self, pkg_dict, defer_commit=False): conn.add(docs=[pkg_dict], commit=commit) except pysolr.SolrError as e: msg = 'Solr returned an error: {0}'.format( - e[:1000] # limit huge responses + e.args[0][:1000] # limit huge responses ) raise SearchIndexError(msg) except socket.error as e: diff --git a/ckan/tests/controllers/test_group.py b/ckan/tests/controllers/test_group.py index 429f4709b50..3b7bf957002 100644 --- a/ckan/tests/controllers/test_group.py +++ b/ckan/tests/controllers/test_group.py @@ -304,7 +304,6 @@ def test_membership_list(self, app): assert user_roles["User One"] == "Admin" assert user_roles["User Two"] == "Member" - @pytest.mark.xfail(reason="DetachedInstance error.") def test_membership_add(self, app): """Member can be added via add member page""" owner = factories.User(fullname="My Owner") @@ -316,10 +315,10 @@ def test_membership_add(self, app): add_response = app.post( url, environ_overrides=env, - data={"save": "", "username": "my-user"}, + data={"save": "", "username": "my-user", "role": "member"}, ) - assert "2 members" in add_response + assert "2 members" in add_response.body add_response_html = BeautifulSoup(add_response.body) user_names = [ @@ -461,7 +460,6 @@ def test_group_follow(self, app): in response ) - @pytest.mark.xfail(reason="DetachedInstance error.") def test_group_follow_not_exist(self, app): """Pass an id for a group that doesn't exist""" user_one = factories.User() diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index 462686a7ffe..172f6312f81 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -1601,7 +1601,6 @@ def test_package_follow(self, app): response = app.post(follow_url, extra_environ=env) assert "You are now following {0}".format(package["title"]) in response - @pytest.mark.xfail(reason="DetachedInstance error.") def test_package_follow_not_exist(self, app): """Pass an id for a package that doesn't exist""" diff --git a/ckan/tests/controllers/test_user.py b/ckan/tests/controllers/test_user.py index e5f752419c1..7fb98bb2923 100644 --- a/ckan/tests/controllers/test_user.py +++ b/ckan/tests/controllers/test_user.py @@ -397,7 +397,6 @@ def test_user_follow(self, app): in response ) - @pytest.mark.xfail(reason="DetachedInstance error.") def test_user_follow_not_exist(self, app): """Pass an id for a user that doesn't exist""" @@ -406,7 +405,8 @@ def test_user_follow_not_exist(self, app): env = {"REMOTE_USER": six.ensure_str(user_one["name"])} follow_url = url_for(controller="user", action="follow", id="not-here") response = app.post(follow_url, extra_environ=env) - assert "user/login" in response.headers["location"] + + assert "You're already logged in" in response.body def test_user_unfollow(self, app): diff --git a/ckan/tests/legacy/lib/test_solr_search_index.py b/ckan/tests/legacy/lib/test_solr_search_index.py index 63633099758..fc00d399019 100644 --- a/ckan/tests/legacy/lib/test_solr_search_index.py +++ b/ckan/tests/legacy/lib/test_solr_search_index.py @@ -60,6 +60,5 @@ def test_1_basic(self): results = self.solr.search(q="sweden", fq=self.fq) result_names = sorted([r["name"] for r in results]) if not result_names: - pytest.xfail("No datasets found") assert [u"se-opengov", u"se-publications"] == result_names diff --git a/ckan/views/__init__.py b/ckan/views/__init__.py index 0136306a97b..4b6242f2427 100644 --- a/ckan/views/__init__.py +++ b/ckan/views/__init__.py @@ -1,5 +1,6 @@ # encoding: utf-8 +from sqlalchemy import inspect from ckan.common import asbool import six from six import text_type @@ -116,6 +117,7 @@ def identify_user(): # Authentication plugins get a chance to run here break as soon as a user # is identified. + authenticators = p.PluginImplementations(p.IAuthenticator) if authenticators: for item in authenticators: @@ -133,8 +135,9 @@ def identify_user(): # If we have a user but not the userobj let's get the userobj. This means # that IAuthenticator extensions do not need to access the user model # directly. - if g.user and not getattr(g, u'userobj', None): - g.userobj = model.User.by_name(g.user) + if g.user: + if not getattr(g, u'userobj', None) or inspect(g.userobj).expired: + g.userobj = model.User.by_name(g.user) # general settings if g.user: diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py index f7e91d7a80a..02d35c5a869 100644 --- a/ckanext/datastore/tests/test_search.py +++ b/ckanext/datastore/tests/test_search.py @@ -587,25 +587,13 @@ def test_search_private_dataset(self, app): "package_id": package["id"], }, ) - - auth = {"Authorization": str(self.sysadmin_user.apikey)} - res = app.post( - "/api/action/datastore_create", - json={"resource_id": resource["id"], "force": True}, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - + helpers.call_action("datastore_create", resource_id=resource["id"], force=True) data = {"resource_id": resource["id"]} auth = {"Authorization": str(self.normal_user.apikey)} res = app.post( "/api/action/datastore_search", json=data, extra_environ=auth, ) - # TODO: on Py3 action called with sysadmin's api-key - if res.status_code != 403: - pytest.xfail("TODO: Fix this test") res_dict = json.loads(res.data) assert res_dict["success"] is False From 9029e43e415ef46e2a4ddfde68aca8a1152087fa Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 8 Jan 2020 15:29:06 +0200 Subject: [PATCH 07/10] Fix sysadmin-add command --- ckan/cli/sysadmin.py | 7 +++---- ckan/cli/user.py | 9 +++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ckan/cli/sysadmin.py b/ckan/cli/sysadmin.py index 7f30f580356..b312b303a99 100644 --- a/ckan/cli/sysadmin.py +++ b/ckan/cli/sysadmin.py @@ -43,16 +43,15 @@ def list_sysadmins(): @sysadmin.command(help=u"Convert user into a sysadmin.") @click.argument(u"username") @click.argument(u"args", nargs=-1) -def add(username, args): - +@click.pass_context +def add(ctx, username, args): user = model.User.by_name(text_type(username)) if not user: click.secho(u'User "%s" not found' % username, fg=u"red") if click.confirm( u"Create new user: %s?" % username, default=True, abort=True ): - # TODO: once, cli.user is merged, invoke `user.add_user` instead - add_user([username] + list(args)) + ctx.forward(add_user) user = model.User.by_name(text_type(username)) user.sysadmin = True diff --git a/ckan/cli/user.py b/ckan/cli/user.py index e48dba2c778..f735a947169 100644 --- a/ckan/cli/user.py +++ b/ckan/cli/user.py @@ -25,7 +25,7 @@ def user(): @click.argument(u'username') @click.argument(u'args', nargs=-1) @click.pass_context -def add_user(username, args): +def add_user(ctx, username, args): u'''Add new user if we use ckan sysadmin add or ckan user add ''' @@ -70,11 +70,16 @@ def add_user(username, args): u'ignore_auth': True, u'user': site_user['name'], } - user_dict = logic.get_action(u'user_create')(context, data_dict) + flask_app = ctx.meta['flask_app'] + # Current user is tested agains sysadmin role during model + # dictization, thus we need request context + with flask_app.test_request_context(): + user_dict = logic.get_action(u'user_create')(context, data_dict) click.secho(u"Successfully created user: %s" % user_dict['name'], fg=u'green', bold=True) except logic.ValidationError as e: error_shout(e) + raise click.Abort() def get_user_str(user): From 0754f499f1e6356143714763ccbeb7ab8a7a9062 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 16 Jan 2020 23:08:52 +0200 Subject: [PATCH 08/10] Universal builtins --- ckan/tests/lib/test_uploader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ckan/tests/lib/test_uploader.py b/ckan/tests/lib/test_uploader.py index b4117202939..90aa68bed2f 100644 --- a/ckan/tests/lib/test_uploader.py +++ b/ckan/tests/lib/test_uploader.py @@ -1,7 +1,10 @@ # encoding: utf-8 import mock import tempfile -import __builtin__ as builtins +try: + import __builtin__ as builtins +except ImportError: + import builtins import flask import paste.fileapp import cStringIO From c1c714d443cfc03b9be3123eaeb0f4b158ee97ac Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 16 Jan 2020 23:35:15 +0200 Subject: [PATCH 09/10] Do not use paste.fileapp --- ckan/tests/lib/test_uploader.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ckan/tests/lib/test_uploader.py b/ckan/tests/lib/test_uploader.py index 90aa68bed2f..13c39d15a00 100644 --- a/ckan/tests/lib/test_uploader.py +++ b/ckan/tests/lib/test_uploader.py @@ -6,8 +6,6 @@ except ImportError: import builtins import flask -import paste.fileapp -import cStringIO from werkzeug.datastructures import FileStorage from pyfakefs import fake_filesystem @@ -27,7 +25,6 @@ class TestInitResourceUpload(object): @mock.patch.object(ckan.lib.uploader, u'os', fake_os) @mock.patch.object(builtins, u'open', side_effect=mock_open_if_open_fails) @mock.patch.object(flask, u'send_file', side_effect=[b'DATA']) - @mock.patch.object(paste.fileapp, u'os', fake_os) @mock.patch.object(config[u'pylons.h'], u'uploads_enabled', return_value=True) @mock.patch.object(ckan.lib.uploader, u'_storage_path', new=u'/doesnt_exist') @@ -50,7 +47,6 @@ def test_resource_without_upload_with_old_werkzeug( @mock.patch.object(ckan.lib.uploader, u'os', fake_os) @mock.patch.object(builtins, u'open', side_effect=mock_open_if_open_fails) @mock.patch.object(flask, u'send_file', side_effect=[b'DATA']) - @mock.patch.object(paste.fileapp, u'os', fake_os) @mock.patch.object(config[u'pylons.h'], u'uploads_enabled', return_value=True) @mock.patch.object(ckan.lib.uploader, u'_storage_path', new=u'/doesnt_exist') @@ -72,7 +68,6 @@ def test_resource_without_upload( @mock.patch.object(ckan.lib.uploader, u'os', fake_os) @mock.patch.object(builtins, u'open', side_effect=mock_open_if_open_fails) @mock.patch.object(flask, u'send_file', side_effect=[b'DATA']) - @mock.patch.object(paste.fileapp, u'os', fake_os) @mock.patch.object(config[u'pylons.h'], u'uploads_enabled', return_value=True) @mock.patch.object(ckan.lib.uploader, u'_storage_path', new=u'/doesnt_exist') From 7d713c98d2f628b141f8b6bcb2410c350d44b6af Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 20 Jan 2020 14:15:40 +0200 Subject: [PATCH 10/10] Fix datastore test --- ckanext/datastore/tests/test_plugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ckanext/datastore/tests/test_plugin.py b/ckanext/datastore/tests/test_plugin.py index ffac6674db4..0964abcf6cb 100644 --- a/ckanext/datastore/tests/test_plugin.py +++ b/ckanext/datastore/tests/test_plugin.py @@ -96,8 +96,7 @@ def test_fts_rank_column_uses_lang_when_casting_to_tsvector(self): u"to_tsvector('french', cast(\"country\" as text))" ) data_dict = {"q": {"country": "Brazil"}, "lang": "french"} - - result = self._datastore_search(data_dict=data_dict) + result = self._datastore_search(data_dict=data_dict, fields_types={}) assert expected_select_content in result["select"][0] def test_adds_fts_on_full_text_field_when_q_is_a_string(self):