From 8500baea2867dcad771617ac84764d0ac990d9b7 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Nov 2019 11:52:34 +0100 Subject: [PATCH 1/7] [#4801] Catch exception outside request context --- ckan/logic/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index 004c11e6810..844efa1f63b 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -232,6 +232,9 @@ def _prepopulate_context(context): except AttributeError: # c.user not set pass + except RuntimeError: + # Outside of request context + pass except TypeError: # c not registered pass From 9dff2edc47facd39808bb897fe9fda300c74418a Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Nov 2019 11:56:33 +0100 Subject: [PATCH 2/7] [#4801] Rename configparser module --- ckan/lib/fanstatic_resources.py | 4 ++-- scripts/4042_fix_resource_extras.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/lib/fanstatic_resources.py b/ckan/lib/fanstatic_resources.py index a0d4ad9e7a6..7700d0708b6 100644 --- a/ckan/lib/fanstatic_resources.py +++ b/ckan/lib/fanstatic_resources.py @@ -3,7 +3,7 @@ import os.path import sys import logging -import ConfigParser +import configparser from ckan.common import config from fanstatic import Library, Resource, Group, get_library_registry @@ -127,7 +127,7 @@ def create_resource(path, lib_name, count, inline=False, supersedes=None): # parse the resource.config file if it exists config_path = os.path.join(resource_path, 'resource.config') if os.path.exists(config_path): - config = ConfigParser.RawConfigParser() + config = configparser.RawConfigParser() config.read(config_path) if config.has_option('main', 'order'): diff --git a/scripts/4042_fix_resource_extras.py b/scripts/4042_fix_resource_extras.py index 7d4e0437ec1..113ac0896bd 100644 --- a/scripts/4042_fix_resource_extras.py +++ b/scripts/4042_fix_resource_extras.py @@ -28,7 +28,7 @@ ''' import json -from ConfigParser import ConfigParser +from configparser import ConfigParser from argparse import ArgumentParser from six.moves import input from sqlalchemy import create_engine From 313f6515b2f704312126d9868afb63b54b9c0890 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Nov 2019 12:16:42 +0100 Subject: [PATCH 3/7] [#4801] Upgrade fanstatic on to a py3 compatible version (only in py3) --- requirements.in | 2 +- requirements.txt | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index ede48112b7f..4e75bc7dfd9 100644 --- a/requirements.in +++ b/requirements.in @@ -5,7 +5,7 @@ Babel==2.3.4 Beaker==1.11.0 bleach==3.0.2 click==6.7 -fanstatic==0.12 +fanstatic==1.1 Flask==1.1.1 Flask-Babel==0.11.2 Jinja2==2.10.1 diff --git a/requirements.txt b/requirements.txt index a2c2365872a..64dba6abc52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,11 +8,11 @@ alembic==1.0.0 babel==2.3.4 beaker==1.11.0 bleach==3.0.2 -certifi==2019.9.11 # via requests +certifi==2019.11.28 # via requests chardet==3.0.4 # via requests click==6.7 decorator==4.4.1 # via sqlalchemy-migrate -fanstatic==0.12 +fanstatic==1.1 flask-babel==0.11.2 flask==1.1.1 funcsigs==1.0.2 # via beaker @@ -27,7 +27,7 @@ passlib==1.6.5 paste==1.7.5.1 pastedeploy==2.0.1 # via pastescript pastescript==2.0.2 -pbr==5.4.3 # via sqlalchemy-migrate +pbr==5.4.4 # via sqlalchemy-migrate polib==1.0.7 psycopg2==2.8.2 pysolr==3.6.0 @@ -43,6 +43,7 @@ repoze.who==2.3 requests==2.22.0 routes==1.13 rq==1.0 +shutilwhich==1.1.0 # via fanstatic simplejson==3.10.0 six==1.13.0 # via bleach, pastescript, python-dateutil, pyutilib, sqlalchemy-migrate sqlalchemy-migrate==0.12.0 From c1bfc70fb371190ea8c591992f92d96c761130f2 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 3 Dec 2019 23:12:08 +0100 Subject: [PATCH 4/7] [#4801] Fix configparser imports --- ckan/lib/fanstatic_resources.py | 6 ++++-- scripts/4042_fix_resource_extras.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ckan/lib/fanstatic_resources.py b/ckan/lib/fanstatic_resources.py index 7700d0708b6..2a8ac26fded 100644 --- a/ckan/lib/fanstatic_resources.py +++ b/ckan/lib/fanstatic_resources.py @@ -3,7 +3,9 @@ import os.path import sys import logging -import configparser + +from six.moves.configparser import RawConfigParser + from ckan.common import config from fanstatic import Library, Resource, Group, get_library_registry @@ -127,7 +129,7 @@ def create_resource(path, lib_name, count, inline=False, supersedes=None): # parse the resource.config file if it exists config_path = os.path.join(resource_path, 'resource.config') if os.path.exists(config_path): - config = configparser.RawConfigParser() + config = RawConfigParser() config.read(config_path) if config.has_option('main', 'order'): diff --git a/scripts/4042_fix_resource_extras.py b/scripts/4042_fix_resource_extras.py index 113ac0896bd..43a816538ef 100644 --- a/scripts/4042_fix_resource_extras.py +++ b/scripts/4042_fix_resource_extras.py @@ -28,7 +28,7 @@ ''' import json -from configparser import ConfigParser +from six.moves.configparser import ConfigParser from argparse import ArgumentParser from six.moves import input from sqlalchemy import create_engine From 6e456b264e95867740fb97452eb6348c9ff8be39 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 4 Dec 2019 10:34:23 +0100 Subject: [PATCH 5/7] Fix bad merge --- ckan/plugins/toolkit.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index a949202ef6c..a9c37b3e031 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -280,8 +280,6 @@ def _initialize(self): t['enqueue_job'] = enqueue_job if six.PY2: - - t['literal'] = webhelpers.html.tags.literal t['response'] = pylons.response self.docstring_overrides['response'] = ''' The Pylons response object. From 733787ca0d63e22501ffe9f0f02e8d1fc37eb015 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 6 Dec 2019 16:37:12 +0100 Subject: [PATCH 6/7] [#4801] Remove test file added by mistake --- ckan/tests/migration/__init__.py | 10 - .../migration/revision_legacy_code_tests.py | 307 ------------------ 2 files changed, 317 deletions(-) delete mode 100644 ckan/tests/migration/__init__.py delete mode 100644 ckan/tests/migration/revision_legacy_code_tests.py diff --git a/ckan/tests/migration/__init__.py b/ckan/tests/migration/__init__.py deleted file mode 100644 index d94dac35161..00000000000 --- a/ckan/tests/migration/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# encoding: utf-8 - -'''**All migration scripts should have tests.** - -.. todo:: - - Write some tests for a migration script, and then use them as an example to - fill out this guidelines section. - -''' diff --git a/ckan/tests/migration/revision_legacy_code_tests.py b/ckan/tests/migration/revision_legacy_code_tests.py deleted file mode 100644 index 780d7c18439..00000000000 --- a/ckan/tests/migration/revision_legacy_code_tests.py +++ /dev/null @@ -1,307 +0,0 @@ -# encoding: utf-8 - -# This file is at the top of the ckan repository, because it needs to be run -# separately from all the other tests, because when it imports -# revision_legacy_code.py it changes the core model, which causes a number of -# test failures which we're not concerned about. - -from difflib import unified_diff -from pprint import pprint, pformat - -from ckan import model - -import ckan.lib.search as search -from ckan.lib.dictization.model_save import package_dict_save -from ckan.lib.create_test_data import CreateTestData -from ckan.tests import helpers - -from ckan.migration.revision_legacy_code import package_dictize_with_revisions as package_dictize -from ckan.migration.revision_legacy_code import RevisionTableMappings, make_package_revision -from ckan.migration.migrate_package_activity import PackageDictizeMonkeyPatch - - -# tests here have been moved from ckan/tests/legacy/lib/test_dictization.py -class TestPackageDictizeWithRevisions(object): - @classmethod - def setup_class(cls): - - cls.package_expected = { - u'author': None, - u'author_email': None, - u'creator_user_id': None, - 'extras': [ - # extra_revision_table is no longer being populated because - # PackageExtra no longer has - # vdm.sqlalchemy.Revisioner(extra_revision_table) (removed in - # #4691) so don't test extras for the moment - # {'key': u'david', 'state': u'active', 'value': u'new_value'}, - # {'key': u'genre', 'state': u'active', 'value': u'new_value'}, - # {'key': u'original media', 'state': u'active', - # 'value': u'book'} - ], - 'groups': [{ - u'name': u'david', - u'capacity': u'public', - u'image_url': u'', - u'image_display_url': u'', - u'description': u'These are books that David likes.', - u'display_name': u"Dave's books", - u'type': u'group', - u'state': u'active', - u'is_organization': False, - u'title': u"Dave's books", - u"approval_status": u"approved"}, - { - u'name': u'roger', - u'capacity': u'public', - u'description': u'Roger likes these books.', - u'image_url': u'', - 'image_display_url': u'', - 'display_name': u"Roger's books", - u'type': u'group', - u'state': u'active', - u'is_organization': False, - u'title': u"Roger's books", - u"approval_status": u"approved"}], - 'isopen': True, - u'license_id': u'other-open', - 'license_title': u'Other (Open)', - 'organization': None, - u'owner_org': None, - u'maintainer': None, - u'maintainer_email': None, - u'name': u'annakarenina', - u'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n\n\n', - 'num_resources': 2, - 'num_tags': 3, - u'private': False, - 'relationships_as_object': [], - 'relationships_as_subject': [], - 'resources': [{u'alt_url': u'alt123', - u'cache_last_updated': None, - u'cache_url': None, - u'description': u'Full text. Needs escaping: " Umlaut: \xfc', - u'format': u'plain text', - u'hash': u'abc123', - u'last_modified': None, - u'mimetype': None, - u'mimetype_inner': None, - u'name': None, - u'position': 0, - u'resource_type': None, - u'size': None, - u'size_extra': u'123', - u'url_type': None, - u'state': u'active', - u'url': u'http://datahub.io/download/x=1&y=2',}, - {u'alt_url': u'alt345', - u'cache_last_updated': None, - u'cache_url': None, - u'description': u'Index of the novel', - u'format': u'JSON', - u'hash': u'def456', - u'last_modified': None, - u'mimetype': None, - u'mimetype_inner': None, - u'name': None, - u'position': 1, - u'resource_type': None, - u'url_type': None, - u'size': None, - u'size_extra': u'345', - u'state': u'active', - u'url': u'http://datahub.io/index.json'}], - u'state': u'active', - 'tags': [{u'name': u'Flexible \u30a1', - 'display_name': u'Flexible \u30a1', - u'state': u'active'}, - {'display_name': u'russian', - u'name': u'russian', - u'state': u'active'}, - {'display_name': u'tolstoy', - u'name': u'tolstoy', - u'state': u'active'}], - u'title': u'A Novel By Tolstoy', - u'type': u'dataset', - u'url': u'http://datahub.io', - u'version': u'0.7a', - } - - def setup(self): - helpers.reset_db() - search.clear_all() - CreateTestData.create() - make_package_revision(model.Package.by_name('annakarenina')) - - def teardown(self): - helpers.reset_db() - search.clear_all() - - def test_09_package_alter(self): - - context = {"model": model, - "session": model.Session, - "user": 'testsysadmin' - } - - anna1 = model.Session.query(model.Package).filter_by(name='annakarenina').one() - - anna_dictized = package_dictize(anna1, context) - - anna_dictized["name"] = u'annakarenina_changed' - anna_dictized["resources"][0]["url"] = u'http://new_url' - - package_dict_save(anna_dictized, context) - model.Session.commit() - model.Session.remove() - make_package_revision(model.Package.by_name('annakarenina_changed')) - - pkg = model.Session.query(model.Package).filter_by(name='annakarenina_changed').one() - - package_dictized = package_dictize(pkg, context) - - resources_revisions = model.Session.query(RevisionTableMappings.instance().ResourceRevision).filter_by(package_id=anna1.id).all() - - sorted_resource_revisions = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1] - for res in sorted_resource_revisions: - print(res.id, res.revision_timestamp, res.state) - assert len(sorted_resource_revisions) == 4 # 2 resources originally, then make_package_revision saves them both again - - # Make sure we remove changeable fields BEFORE we store the pretty-printed version - # for comparison - clean_package_dictized = self.remove_changable_columns(package_dictized) - - anna_original = pformat(anna_dictized) - anna_after_save = pformat(clean_package_dictized) - - assert self.remove_changable_columns(anna_dictized) == clean_package_dictized, \ - "\n".join(unified_diff(anna_original.split("\n"), anna_after_save.split("\n"))) - - # changes to the package, relied upon by later tests - anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed').one() - anna_dictized = package_dictize(anna1, context) - anna_dictized['name'] = u'annakarenina_changed2' - anna_dictized['resources'][0]['url'] = u'http://new_url2' - anna_dictized['tags'][0]['name'] = u'new_tag' - anna_dictized['tags'][0].pop('id') # test if - anna_dictized['extras'][0]['value'] = u'new_value' - - package_dict_save(anna_dictized, context) - model.Session.commit() - model.Session.remove() - make_package_revision(model.Package.by_name('annakarenina_changed2')) - - anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one() - anna_dictized = package_dictize(anna1, context) - anna_dictized['notes'] = 'wee' - anna_dictized['resources'].append({ - 'format': u'plain text', - 'url': u'http://newurl'} - ) - anna_dictized['tags'].append({'name': u'newnew_tag'}) - anna_dictized['extras'].append({'key': 'david', - 'value': u'new_value'}) - - package_dict_save(anna_dictized, context) - model.Session.commit() - model.Session.remove() - make_package_revision(model.Package.by_name('annakarenina_changed2')) - - context = {'model': model, - 'session': model.Session} - - anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one() - - pkgrevisions = model.Session.query(RevisionTableMappings.instance().PackageRevision).filter_by(id=anna1.id).all() - sorted_packages = sorted(pkgrevisions, key=lambda x: x.revision_timestamp) - - context['revision_id'] = sorted_packages[0].revision_id # original state - - with PackageDictizeMonkeyPatch(): - first_dictized = self.remove_changable_columns(package_dictize(anna1, context)) - assert self.remove_changable_columns(self.package_expected) == first_dictized - - context['revision_id'] = sorted_packages[1].revision_id - - second_dictized = self.remove_changable_columns(package_dictize(anna1, context)) - - first_dictized["name"] = u'annakarenina_changed' - first_dictized["resources"][0]["url"] = u'http://new_url' - - assert second_dictized == first_dictized - - context['revision_id'] = sorted_packages[2].revision_id - third_dictized = self.remove_changable_columns(package_dictize(anna1, context)) - - second_dictized['name'] = u'annakarenina_changed2' - second_dictized['resources'][0]['url'] = u'http://new_url2' - second_dictized['tags'][0]['name'] = u'new_tag' - second_dictized['tags'][0]['display_name'] = u'new_tag' - second_dictized['state'] = 'active' - - print('\n'.join(unified_diff(pformat(second_dictized).split('\n'), pformat(third_dictized).split('\n')))) - assert second_dictized == third_dictized - - context['revision_id'] = sorted_packages[3].revision_id # original state - forth_dictized = self.remove_changable_columns(package_dictize(anna1, context)) - - third_dictized['notes'] = 'wee' - third_dictized['resources'].insert(2, { - u'cache_last_updated': None, - u'cache_url': None, - u'description': u'', - u'format': u'plain text', - u'hash': u'', - u'last_modified': None, - u'mimetype': None, - u'mimetype_inner': None, - u'name': None, - u'position': 2, - u'resource_type': None, - u'url_type': None, - u'size': None, - u'state': u'active', - u'url': u'http://newurl'}) - third_dictized['num_resources'] = third_dictized['num_resources'] + 1 - - third_dictized['tags'].insert(1, {'name': u'newnew_tag', 'display_name': u'newnew_tag', 'state': 'active'}) - third_dictized['num_tags'] = third_dictized['num_tags'] + 1 - third_dictized['state'] = 'active' - third_dictized['state'] = 'active' - - pprint(third_dictized) - pprint(forth_dictized) - - assert third_dictized == forth_dictized - - def remove_changable_columns(self, dict, remove_package_id=False): - ids_to_keep = ['license_id', 'creator_user_id'] - if not remove_package_id: - ids_to_keep.append('package_id') - - for key, value in dict.items(): - if key.endswith('id') and key not in ids_to_keep: - dict.pop(key) - if key == 'created': - dict.pop(key) - if 'timestamp' in key: - dict.pop(key) - if key in ['metadata_created','metadata_modified']: - dict.pop(key) - if isinstance(value, list): - for new_dict in value: - self.remove_changable_columns(new_dict, - key in ['resources', 'extras'] or remove_package_id) - - # TEMPORARY HACK - we remove 'extras' so they aren't tested. This - # is due to package_extra_revisions being migrated from ckan/model - # in #4691 but not the rest of the model revisions just yet. Until - # we finish this work (#4664) it is hard to get this working - - # extra_revision_table is no longer being populated because - # PackageExtra no longer has - # vdm.sqlalchemy.Revisioner(extra_revision_table). However #4664 - # will allow use to manually create revisions and test this again. - if key == 'extras': - dict.pop(key) - # END OF HACK - return dict From 8d09297e4f8bfa841ae18af817b037ff979d98d2 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 6 Dec 2019 17:00:45 +0100 Subject: [PATCH 7/7] Fix sphinx warning --- doc/contributing/testing.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/doc/contributing/testing.rst b/doc/contributing/testing.rst index d6918af7339..59b6e171fc5 100644 --- a/doc/contributing/testing.rst +++ b/doc/contributing/testing.rst @@ -454,12 +454,6 @@ Writing :mod:`ckan.plugins` tests .. automodule:: ckan.tests.plugins -Writing :mod:`ckan.migration` tests ------------------------------------ - -.. automodule:: ckan.tests.migration - - Writing :mod:`ckan.ckanext` tests ---------------------------------