From 19e653e75478279b8ae933e0d445956a7949f8f1 Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Tue, 30 Oct 2018 14:07:43 +0100 Subject: [PATCH 01/21] Switching from schema.xml to ManagedSchema --- ckan/lib/search/__init__.py | 51 +++++++++++++++++++----------- contrib/docker/solr/solrconfig.xml | 4 ++- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index dbc8ac12cc5..afb4f9d3ab4 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -55,7 +55,8 @@ def text_traceback(): 'package': PackageSearchQuery } -SOLR_SCHEMA_FILE_OFFSET = '/admin/file/?file=schema.xml' +SOLR_SCHEMA_FILE_OFFSET_MANAGED = '/schema?wt=schema.xml' +SOLR_SCHEMA_FILE_OFFSET_CLASSIC = '/admin/file/?file=schema.xml' def _normalize_type(_type): @@ -246,6 +247,21 @@ def clear_all(): log.debug("Clearing search index...") package_index.clear() +def _get_schema_from_solr(file_offset): + solr_url, solr_user, solr_password = SolrSettings.get() + + http_auth = None + if solr_user is not None and solr_password is not None: + http_auth = solr_user + ':' + solr_password + http_auth = 'Basic ' + http_auth.encode('base64').strip() + + url = solr_url.strip('/') + file_offset + + req = urllib2.Request(url=url) + if http_auth: + req.add_header('Authorization', http_auth) + + return urllib2.urlopen(req) def check_solr_schema_version(schema_file=None): ''' @@ -253,10 +269,15 @@ def check_solr_schema_version(schema_file=None): with this CKAN version. The schema will be retrieved from the SOLR server, using the - offset defined in SOLR_SCHEMA_FILE_OFFSET - ('/admin/file/?file=schema.xml'). The schema_file parameter - allows to override this pointing to different schema file, but - it should only be used for testing purposes. + offset defined in SOLR_SCHEMA_FILE_OFFSET_MANAGED + ('/schema?wt=schema.xml'). If SOLR is set to use the manually + edited `schema.xml`, the schema will be retrieved from the SOLR + server using the offset defined in + SOLR_SCHEMA_FILE_OFFSET_CLASSIC ('/admin/file/?file=schema.xml'). + + The schema_file parameter allows to override this pointing to + different schema file, but it should only be used for testing + purposes. If the CKAN instance is configured to not use SOLR or the SOLR server is not available, the function will return False, as the @@ -275,20 +296,12 @@ def check_solr_schema_version(schema_file=None): # Try to get the schema XML file to extract the version if not schema_file: - solr_url, solr_user, solr_password = SolrSettings.get() - - http_auth = None - if solr_user is not None and solr_password is not None: - http_auth = solr_user + ':' + solr_password - http_auth = 'Basic ' + http_auth.encode('base64').strip() - - url = solr_url.strip('/') + SOLR_SCHEMA_FILE_OFFSET - - req = urllib2.Request(url=url) - if http_auth: - req.add_header('Authorization', http_auth) - - res = urllib2.urlopen(req) + try: + # Try Managed Schema + res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_MANAGED) + except urllib2.HTTPError: + # Fallback to Manually Edited schema.xml + res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_CLASSIC) else: url = 'file://%s' % schema_file res = urllib2.urlopen(url) diff --git a/contrib/docker/solr/solrconfig.xml b/contrib/docker/solr/solrconfig.xml index 9ac620c2a9d..8a3eade3806 100644 --- a/contrib/docker/solr/solrconfig.xml +++ b/contrib/docker/solr/solrconfig.xml @@ -288,7 +288,9 @@ - + + true + From 94df949d5332dac1b2cae423fa71b3bfe0e2748e Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 22 Feb 2019 20:25:21 +0000 Subject: [PATCH 02/21] Add changelog info about migrating while running --- CHANGELOG.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e2ed14febae..79e8a306dbd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,10 +11,11 @@ v.2.9.0 TBA ================== * This version requires script 'migrate_package_activity.py' to be run - *before* CKAN is upgraded to this version (or higher). This is because this - script takes a while to run, adding in the Activity Stream detail, visible - only to admins by default. You will not be able to run ``paster db upgrade`` - until 'migrate_package_activity.py' is done. + *before* CKAN is upgraded to this version (or higher). The idea is you do + this special migration while CKAN is running, because the script takes a + while to run. It adds in the Activity Stream detail, visible only to admins + by default. You will not be able to run ``paster db upgrade`` until + 'migrate_package_activity.py' is done. Download and run migrate_package_activity.py like this: cd /usr/lib/ckan/default/src/ckan/ From 211615f831419e221b1ff7d9a3d5c7e5f4602482 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 22 Feb 2019 21:01:47 +0000 Subject: [PATCH 03/21] Refactor activity_dictize --- ckan/lib/dictization/model_dictize.py | 21 +++----- .../lib/dictization/test_model_dictize.py | 48 +++++++++++++++++++ 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 73c82ce7f6f..b836ff9f861 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -13,7 +13,6 @@ ''' import datetime import urlparse -import copy from ckan.common import config from sqlalchemy.sql import select @@ -637,19 +636,13 @@ def vocabulary_list_dictize(vocabulary_list, context): def activity_dictize(activity, context, include_data=False): activity_dict = d.table_dictize(activity, context) if not include_data: - # take a copy of the activity data, since the original may be used - # elsewhere during the same render and we don't want to affect that - activity_dict['data'] = copy.deepcopy(activity_dict['data']) - # delete all the data apart from the title field on each data object, - # because that is needed to display it in the activity stream - for obj_key in activity_dict['data'].keys(): - obj_data = activity_dict['data'][obj_key] - if isinstance(obj_data, dict): - for key in obj_data.keys(): - if key != 'title': - del obj_data[key] - else: - del activity_dict['data'][obj_key] + # replace the data with just a {'title': title} and not the rest of + # the dataset/group/org/custom obj. we need the title to display it + # in the activity stream. + activity_dict['data'] = { + key: {'title': val['title']} + for (key, val) in activity_dict['data'].items() + if isinstance(val, dict) and 'title' in val} return activity_dict diff --git a/ckan/tests/lib/dictization/test_model_dictize.py b/ckan/tests/lib/dictization/test_model_dictize.py index c26c0590990..35c577020b7 100644 --- a/ckan/tests/lib/dictization/test_model_dictize.py +++ b/ckan/tests/lib/dictization/test_model_dictize.py @@ -608,3 +608,51 @@ def test_vocabulary_dictize_not_including_datasets(self): assert len(vocab_dict["tags"]) == 2 for tag in vocab_dict["tags"]: assert len(tag.get("packages", [])) == 0 + + +class TestActivityDictize(object): + def setup(self): + helpers.reset_db() + + def test_include_data(self): + dataset = factories.Dataset() + user = factories.User() + activity = factories.Activity( + user_id=user['id'], + object_id=dataset['id'], + revision_id=None, + activity_type='new package', + data={ + 'package': copy.deepcopy(dataset), + 'actor': 'Mr Someone', + }) + activity_obj = model.Activity.get(activity['id']) + context = {'model': model, 'session': model.Session} + dictized = model_dictize.activity_dictize(activity_obj, context, + include_data=True) + assert_equal(dictized['user_id'], user['id']) + assert_equal(dictized['activity_type'], 'new package') + assert_equal(dictized['data']['package']['title'], dataset['title']) + assert_equal(dictized['data']['package']['id'], dataset['id']) + assert_equal(dictized['data']['actor'], 'Mr Someone') + + def test_dont_include_data(self): + dataset = factories.Dataset() + user = factories.User() + activity = factories.Activity( + user_id=user['id'], + object_id=dataset['id'], + revision_id=None, + activity_type='new package', + data={ + 'package': copy.deepcopy(dataset), + 'actor': 'Mr Someone', + }) + activity_obj = model.Activity.get(activity['id']) + context = {'model': model, 'session': model.Session} + dictized = model_dictize.activity_dictize(activity_obj, context, + include_data=False) + assert_equal(dictized['user_id'], user['id']) + assert_equal(dictized['activity_type'], 'new package') + assert_equal(dictized['data'], + {'package': {'title': dataset['title']}}) From b95e2c6357a9fb4fa73d3d24e517b293c426ad30 Mon Sep 17 00:00:00 2001 From: Shahar Evron Date: Thu, 28 Feb 2019 16:24:04 +0200 Subject: [PATCH 04/21] Increase size of h1 headings to 1.8em --- ckan/public-bs2/base/less/ckan.less | 2 +- ckan/public/base/less/ckan.less | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/public-bs2/base/less/ckan.less b/ckan/public-bs2/base/less/ckan.less index 9bc3b055023..c78d13dd7f9 100644 --- a/ckan/public-bs2/base/less/ckan.less +++ b/ckan/public-bs2/base/less/ckan.less @@ -76,7 +76,7 @@ iframe { } .embedded-content h1 { - font-size: 1.4em; + font-size: 1.8em; } .embedded-content h2 { diff --git a/ckan/public/base/less/ckan.less b/ckan/public/base/less/ckan.less index f2ed0ef04b9..bbe49f5da91 100644 --- a/ckan/public/base/less/ckan.less +++ b/ckan/public/base/less/ckan.less @@ -78,7 +78,7 @@ iframe { } .embedded-content h1 { - font-size: 1.4em; + font-size: 1.8em; } .embedded-content h2 { From c874ac0163317987bbe0bc6836e6f1cd6da24eb6 Mon Sep 17 00:00:00 2001 From: Alberto Miedes Date: Thu, 28 Feb 2019 20:03:09 +0100 Subject: [PATCH 05/21] Fix hardcoded root paths in user controller --- ckan/controllers/user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index fc965c59c8f..23d143bde60 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -497,7 +497,7 @@ def request_reset(self): mailer.send_reset_link(user_obj) h.flash_success(_('Please check your inbox for ' 'a reset code.')) - h.redirect_to('/') + h.redirect_to(u'home.index') except mailer.MailerException as e: h.flash_error(_('Could not send reset link: %s') % text_type(e)) @@ -542,7 +542,7 @@ def perform_reset(self, id): mailer.create_reset_key(user_obj) h.flash_success(_("Your password has been reset.")) - h.redirect_to('/') + h.redirect_to(u'home.index') except NotAuthorized: h.flash_error(_('Unauthorized to edit user %s') % id) except NotFound as e: From b733cd6f2945f1fbc757515d7543cd4a8394face Mon Sep 17 00:00:00 2001 From: Alberto Miedes Date: Thu, 28 Feb 2019 20:03:21 +0100 Subject: [PATCH 06/21] Fix hardcoded root paths in user view --- ckan/views/user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/views/user.py b/ckan/views/user.py index ab5ba8b7954..c2715e5b402 100644 --- a/ckan/views/user.py +++ b/ckan/views/user.py @@ -563,14 +563,14 @@ def post(self): h.flash_error(_(u'Error sending the email. Try again later ' 'or contact an administrator for help')) log.exception(e) - return h.redirect_to(u'/') + return h.redirect_to(u'home.index') # always tell the user it succeeded, because otherwise we reveal # which accounts exist or not h.flash_success( _(u'A reset link has been emailed to you ' '(unless the account specified does not exist)')) - return h.redirect_to(u'/') + return h.redirect_to(u'home.index') def get(self): self._prepare() @@ -636,7 +636,7 @@ def post(self, id): mailer.create_reset_key(context[u'user_obj']) h.flash_success(_(u'Your password has been reset.')) - return h.redirect_to(u'/') + return h.redirect_to(u'home.index') except logic.NotAuthorized: h.flash_error(_(u'Unauthorized to edit user %s') % id) except logic.NotFound: From 38a16b35e476f49b18a68ebd8382b305fb4dca6a Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 22 Feb 2019 21:15:43 +0000 Subject: [PATCH 07/21] Avoid affecting caller data_dict --- ckan/logic/action/get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index e355b7ff386..874660cb6e6 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -2595,7 +2595,7 @@ def group_activity_list(context, data_dict): ''' # FIXME: Filter out activities whose subject or object the user is not # authorized to read. - data_dict['include_data'] = False + data_dict = dict(data_dict, include_data=False) include_hidden_activity = asbool(context.get('include_hidden_activity')) _check_access('group_activity_list', context, data_dict) From 5523ba092e7c2d76e870ccb67e1595e0b5519552 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 22 Feb 2019 21:57:23 +0000 Subject: [PATCH 08/21] Improve performance of migration query. --- ckan/migration/migrate_package_activity.py | 12 ++--- .../test_migrate_package_activity.py | 44 +++++++++++++++++-- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/ckan/migration/migrate_package_activity.py b/ckan/migration/migrate_package_activity.py index 6418e9732f1..999b35c0399 100644 --- a/ckan/migration/migrate_package_activity.py +++ b/ckan/migration/migrate_package_activity.py @@ -187,10 +187,12 @@ def migrate_dataset(dataset_name, errors): def wipe_activity_detail(delete_activity_detail): from ckan import model - num_activity_detail_rows = \ - model.Session.execute(u'SELECT count(*) FROM "activity_detail";') \ - .fetchall()[0][0] - if num_activity_detail_rows == 0: + activity_detail_has_rows = \ + bool(model.Session.execute( + u'SELECT count(*) ' + 'FROM (SELECT * FROM "activity_detail" LIMIT 1) as t;') \ + .fetchall()[0][0]) + if not activity_detail_has_rows: print(u'\nactivity_detail table is aleady emptied') return print( @@ -204,7 +206,7 @@ def wipe_activity_detail(delete_activity_detail): delete_activity_detail = \ input(u'Delete activity_detail table content? (y/n):') if delete_activity_detail.lower()[:1] != u'y': - sys.exit(0) + return from ckan import model model.Session.execute(u'DELETE FROM "activity_detail";') model.Session.commit() diff --git a/ckan/tests/migration/test_migrate_package_activity.py b/ckan/tests/migration/test_migrate_package_activity.py index eb4c7beb6a7..d7c6f29fff6 100644 --- a/ckan/tests/migration/test_migrate_package_activity.py +++ b/ckan/tests/migration/test_migrate_package_activity.py @@ -5,14 +5,14 @@ import ckan.tests.factories as factories import ckan.tests.helpers as helpers -from ckan.migration.migrate_package_activity import migrate_dataset +from ckan.migration.migrate_package_activity import (migrate_dataset, + wipe_activity_detail) from ckan.model.activity import package_activity_list from ckan import model class TestMigrate(object): - @classmethod - def setup_class(cls): + def setup(self): helpers.reset_db() @classmethod @@ -78,3 +78,41 @@ def test_a_contemporary_activity_needs_no_migration(self): activity_data_after = package_activity_list(dataset['id'], 0, 0)[0].data eq_(activity_data_before, activity_data_after) + + def test_wipe_activity_detail(self): + dataset = factories.Dataset() + user = factories.User() + activity = factories.Activity( + user_id=user['id'], object_id=dataset['id'], revision_id=None, + activity_type='new package', + data={ + 'package': copy.deepcopy(dataset), + 'actor': 'Mr Someone', + }) + ad = model.ActivityDetail( + activity_id=activity['id'], object_id=dataset['id'], + object_type='package', activity_type='new package') + model.Session.add(ad) + model.Session.commit() + eq_(model.Session.query(model.ActivityDetail).count(), 1) + wipe_activity_detail(delete_activity_detail='y') + eq_(model.Session.query(model.ActivityDetail).count(), 0) + + def test_dont_wipe_activity_detail(self): + dataset = factories.Dataset() + user = factories.User() + activity = factories.Activity( + user_id=user['id'], object_id=dataset['id'], revision_id=None, + activity_type='new package', + data={ + 'package': copy.deepcopy(dataset), + 'actor': 'Mr Someone', + }) + ad = model.ActivityDetail( + activity_id=activity['id'], object_id=dataset['id'], + object_type='package', activity_type='new package') + model.Session.add(ad) + model.Session.commit() + eq_(model.Session.query(model.ActivityDetail).count(), 1) + wipe_activity_detail(delete_activity_detail='n') # i.e. don't do it! + eq_(model.Session.query(model.ActivityDetail).count(), 1) From 52f230ed8c95698e351c4133eb3b136bba71fcc1 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 22 Feb 2019 22:08:10 +0000 Subject: [PATCH 09/21] Fix -bs2 typo --- ckan/templates-bs2/package/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates-bs2/package/base.html b/ckan/templates-bs2/package/base.html index 9b382194909..8f90983739e 100644 --- a/ckan/templates-bs2/package/base.html +++ b/ckan/templates-bs2/package/base.html @@ -16,7 +16,7 @@ {% else %}
  • {% link_for _('Datasets'), named_route='dataset.search' %}
    • {% endif %} - {% link_for dataset|truncate(30), named_route='dataset.read', id=pkg.name if is_activity_archive else pkg.name %} + {% link_for dataset|truncate(30), named_route='dataset.read', id=pkg.id if is_activity_archive else pkg.name %} {% else %}
  • {% link_for _('Datasets'), named_route='dataset.search' %}
  • {{ _('Create Dataset') }}
    • From 543c6e35b50766794404382e874b9c3fb95f6215 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 1 Mar 2019 11:57:13 +0000 Subject: [PATCH 10/21] Add migration test for revision being missing Also PEP8 --- ckan/migration/migrate_package_activity.py | 14 ++-- .../test_migrate_package_activity.py | 77 ++++++++++++++++--- .../migration/test_revision_legacy_code.py | 2 +- 3 files changed, 75 insertions(+), 18 deletions(-) diff --git a/ckan/migration/migrate_package_activity.py b/ckan/migration/migrate_package_activity.py index 999b35c0399..cdb5ffcb205 100644 --- a/ckan/migration/migrate_package_activity.py +++ b/ckan/migration/migrate_package_activity.py @@ -137,10 +137,10 @@ def migrate_dataset(dataset_name, errors): ' - no action') continue - # get the dataset as it was at this revision - context[u'revision_id'] = activity[u'revision_id'] + # get the dataset as it was at this revision: # call package_show just as we do in package.py:activity_stream_item(), - # only with a revision_id + # only with a revision_id (to get it as it was then) + context[u'revision_id'] = activity[u'revision_id'] try: dataset = logic.get_action(u'package_show')( context, @@ -157,7 +157,9 @@ def migrate_dataset(dataset_name, errors): try: dataset = {u'title': activity_obj.data['package']['title']} except KeyError: - dataset = None + # unlikely the package is not recorded in the activity, but + # not impossible + dataset = {u'title': u'unknown'} # get rid of revision_timestamp, which wouldn't be there if saved by # during activity_stream_item() - something to do with not specifying @@ -190,8 +192,8 @@ def wipe_activity_detail(delete_activity_detail): activity_detail_has_rows = \ bool(model.Session.execute( u'SELECT count(*) ' - 'FROM (SELECT * FROM "activity_detail" LIMIT 1) as t;') \ - .fetchall()[0][0]) + 'FROM (SELECT * FROM "activity_detail" LIMIT 1) as t;') + .fetchall()[0][0]) if not activity_detail_has_rows: print(u'\nactivity_detail table is aleady emptied') return diff --git a/ckan/tests/migration/test_migrate_package_activity.py b/ckan/tests/migration/test_migrate_package_activity.py index d7c6f29fff6..eefbd6443d6 100644 --- a/ckan/tests/migration/test_migrate_package_activity.py +++ b/ckan/tests/migration/test_migrate_package_activity.py @@ -2,6 +2,7 @@ import copy from nose.tools import eq_ +from collections import defaultdict import ckan.tests.factories as factories import ckan.tests.helpers as helpers @@ -11,7 +12,7 @@ from ckan import model -class TestMigrate(object): +class TestMigrateDataset(object): def setup(self): helpers.reset_db() @@ -79,23 +80,77 @@ def test_a_contemporary_activity_needs_no_migration(self): activity_data_after = package_activity_list(dataset['id'], 0, 0)[0].data eq_(activity_data_before, activity_data_after) + def test_revision_missing(self): + dataset = factories.Dataset(resources=[ + {u'url': u'http://example.com/a.csv', u'format': u'csv'} + ]) + # delete a part of the revision, so package_show for the revision will + # return NotFound + model.Session.query(model.PackageRevision).delete() + model.Session.commit() + # delete 'activity.data.package.resources' so it needs migrating + activity = package_activity_list(dataset['id'], 0, 0)[0] + activity = model.Activity.get(activity.id) + del activity.data['package']['resources'] + model.Session.commit() + + errors = defaultdict(int) + migrate_dataset(dataset['name'], errors) + + eq_(dict(errors), {u'Revision missing': 1}) + activity_data_migrated = \ + package_activity_list(dataset['id'], 0, 0)[0].data + # the title is there so the activity stream can display it + eq_(activity_data_migrated['package']['title'], u'Test Dataset') + # the rest of the dataset is missing - better that than just the + # dictized package without resources, extras etc + assert u'resources' not in activity_data_migrated['package'] + + def test_revision_and_data_missing(self): + dataset = factories.Dataset(resources=[ + {u'url': u'http://example.com/a.csv', u'format': u'csv'} + ]) + # delete a part of the revision, so package_show for the revision will + # return NotFound + model.Session.query(model.PackageRevision).delete() + model.Session.commit() + # delete 'activity.data.package' so it needs migrating AND the package + # title won't be available, so we test how the migration deals with + # that + activity = package_activity_list(dataset['id'], 0, 0)[0] + activity = model.Activity.get(activity.id) + del activity.data['package'] + model.Session.commit() + + errors = defaultdict(int) + migrate_dataset(dataset['name'], errors) + + eq_(dict(errors), {u'Revision missing': 1}) + activity_data_migrated = \ + package_activity_list(dataset['id'], 0, 0)[0].data + # the title is there so the activity stream can display it + eq_(activity_data_migrated['package']['title'], u'unknown') + assert u'resources' not in activity_data_migrated['package'] + + +class TestWipeActivityDetail(object): def test_wipe_activity_detail(self): dataset = factories.Dataset() user = factories.User() activity = factories.Activity( user_id=user['id'], object_id=dataset['id'], revision_id=None, - activity_type='new package', + activity_type=u'new package', data={ - 'package': copy.deepcopy(dataset), - 'actor': 'Mr Someone', + u'package': copy.deepcopy(dataset), + u'actor': u'Mr Someone', }) ad = model.ActivityDetail( activity_id=activity['id'], object_id=dataset['id'], - object_type='package', activity_type='new package') + object_type=u'package', activity_type=u'new package') model.Session.add(ad) model.Session.commit() eq_(model.Session.query(model.ActivityDetail).count(), 1) - wipe_activity_detail(delete_activity_detail='y') + wipe_activity_detail(delete_activity_detail=u'y') eq_(model.Session.query(model.ActivityDetail).count(), 0) def test_dont_wipe_activity_detail(self): @@ -103,16 +158,16 @@ def test_dont_wipe_activity_detail(self): user = factories.User() activity = factories.Activity( user_id=user['id'], object_id=dataset['id'], revision_id=None, - activity_type='new package', + activity_type=u'new package', data={ - 'package': copy.deepcopy(dataset), - 'actor': 'Mr Someone', + u'package': copy.deepcopy(dataset), + u'actor': u'Mr Someone', }) ad = model.ActivityDetail( activity_id=activity['id'], object_id=dataset['id'], - object_type='package', activity_type='new package') + object_type=u'package', activity_type=u'new package') model.Session.add(ad) model.Session.commit() eq_(model.Session.query(model.ActivityDetail).count(), 1) - wipe_activity_detail(delete_activity_detail='n') # i.e. don't do it! + wipe_activity_detail(delete_activity_detail=u'n') # i.e. don't do it! eq_(model.Session.query(model.ActivityDetail).count(), 1) diff --git a/ckan/tests/migration/test_revision_legacy_code.py b/ckan/tests/migration/test_revision_legacy_code.py index c278ed6450d..e6dbb4f51f8 100644 --- a/ckan/tests/migration/test_revision_legacy_code.py +++ b/ckan/tests/migration/test_revision_legacy_code.py @@ -13,7 +13,7 @@ # tests here have been moved from ckan/tests/legacy/lib/test_dictization.py -class TestPackageDictizeWithRevisions: +class TestPackageDictizeWithRevisions(object): @classmethod def setup_class(cls): # clean the db so we can run these tests on their own From 1a238968aa764873f0cbadf32bc380e87e3ecc46 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 1 Mar 2019 14:42:16 +0000 Subject: [PATCH 11/21] Migration cleaner with context, not that there has been any trouble. --- ckan/migration/migrate_package_activity.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ckan/migration/migrate_package_activity.py b/ckan/migration/migrate_package_activity.py index cdb5ffcb205..1d0dfd9948f 100644 --- a/ckan/migration/migrate_package_activity.py +++ b/ckan/migration/migrate_package_activity.py @@ -102,7 +102,6 @@ def migrate_dataset(dataset_name, errors): import ckan.logic as logic from ckan import model - context = get_context() # 'hidden' activity is that by site_user, such as harvests, which are # not shown in the activity stream because they can be too numerous. # However these do have Activity objects, and if a hidden Activity is @@ -110,14 +109,13 @@ def migrate_dataset(dataset_name, errors): # non-hidden Activity, then it does a diff with the hidden one (rather than # the most recent non-hidden one), so it is important to store the # package_dict in hidden Activity objects. - context[u'include_hidden_activity'] = True + context = dict(get_context(), include_hidden_activity=True) package_activity_stream = logic.get_action(u'package_activity_list')( context, {u'id': dataset_name}) num_activities = len(package_activity_stream) if not num_activities: print(u' No activities') - context[u'for_view'] = False # Iterate over this package's existing activity stream objects for i, activity in enumerate(package_activity_stream): # e.g. activity = @@ -140,7 +138,13 @@ def migrate_dataset(dataset_name, errors): # get the dataset as it was at this revision: # call package_show just as we do in package.py:activity_stream_item(), # only with a revision_id (to get it as it was then) - context[u'revision_id'] = activity[u'revision_id'] + context = dict( + get_context(), + for_view=False, + revision_id=activity[u'revision_id'], + use_cache=False, # avoid the cache (which would give us the + # latest revision) + ) try: dataset = logic.get_action(u'package_show')( context, From 1836e812ce02dae82cf85312d9f7493db601e393 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 1 Mar 2019 14:47:50 +0000 Subject: [PATCH 12/21] Do migration of revisions in chronological order - makes more sense --- ckan/migration/migrate_package_activity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/migration/migrate_package_activity.py b/ckan/migration/migrate_package_activity.py index 1d0dfd9948f..63463440e27 100644 --- a/ckan/migration/migrate_package_activity.py +++ b/ckan/migration/migrate_package_activity.py @@ -117,7 +117,7 @@ def migrate_dataset(dataset_name, errors): print(u' No activities') # Iterate over this package's existing activity stream objects - for i, activity in enumerate(package_activity_stream): + for i, activity in enumerate(reversed(package_activity_stream)): # e.g. activity = # {'activity_type': u'changed package', # 'id': u'62107f87-7de0-4d17-9c30-90cbffc1b296', From 254aa84ca3474f4e045bc374190ce05d39fce75c Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 1 Mar 2019 14:50:54 +0000 Subject: [PATCH 13/21] Fix package_show of old revision Fix in package_dictize_with_revisions of 'metadata_modified' and 'organization' - it was showing the current package's version, not the older revision. Test fix - del activity.data['package'] was unreliably committed. --- ckan/migration/revision_legacy_code.py | 11 +++--- .../test_migrate_package_activity.py | 36 ++++++++++++++----- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/ckan/migration/revision_legacy_code.py b/ckan/migration/revision_legacy_code.py index 5eb5da0c830..375bf3dfe95 100644 --- a/ckan/migration/revision_legacy_code.py +++ b/ckan/migration/revision_legacy_code.py @@ -107,7 +107,7 @@ def package_dictize_with_revisions(pkg, context): else: group = model.group_revision_table q = select([group] - ).where(group.c.id == pkg.owner_org) \ + ).where(group.c.id == result_dict['owner_org']) \ .where(group.c.state == u'active') result = execute(q, group, context) organizations = d.obj_list_dictize(result, context) @@ -153,9 +153,12 @@ def package_dictize_with_revisions(pkg, context): result_dict['license_title'] = pkg.license_id # creation and modification date - result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() - result_dict['metadata_created'] = pkg.metadata_created.isoformat() \ - if pkg.metadata_created else None + if is_latest_revision: + result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() + # (If not is_latest_revision, don't use pkg which is the latest version. + # Instead, use the dates already in result_dict that came from the dictized + # PackageRevision) + result_dict['metadata_created'] = pkg.metadata_created.isoformat() return result_dict diff --git a/ckan/tests/migration/test_migrate_package_activity.py b/ckan/tests/migration/test_migrate_package_activity.py index eefbd6443d6..6efd486e5cb 100644 --- a/ckan/tests/migration/test_migrate_package_activity.py +++ b/ckan/tests/migration/test_migrate_package_activity.py @@ -1,7 +1,7 @@ # encoding: utf-8 import copy -from nose.tools import eq_ +from nose.tools import assert_equal as eq_ from collections import defaultdict import ckan.tests.factories as factories @@ -52,19 +52,25 @@ def test_migration_with_multiple_revisions(self): helpers.call_action(u'package_update', **dataset) activity = package_activity_list(dataset['id'], 0, 0)[1] - activity_data_as_it_should_be = copy.deepcopy(activity.data) + activity_data_as_it_should_be = copy.deepcopy(activity.data['package']) - # Remove 'activity.data.package' to provoke the migration to regenerate - # it from package_revision (etc) + # Remove 'activity.data.package.resources' to provoke the migration to + # regenerate it from package_revision (etc) activity = model.Activity.get(activity.id) - del activity.data['package'] - model.repo.commit_and_remove() + activity.data = {u'actor': None, u'package': {u'title': 'Title 2'}} + model.Session.commit() + model.Session.remove() + # double check that worked... + assert not \ + model.Activity.get(activity.id).data['package'].get('resources') + migrate_dataset(dataset['name'], {}) + eq_.__self__.maxDiff = None activity_data_migrated = \ - package_activity_list(dataset['id'], 0, 0)[1].data + package_activity_list(dataset['id'], 0, 0)[1].data['package'] eq_(activity_data_as_it_should_be, activity_data_migrated) - eq_(activity_data_migrated['package']['title'], u'Title 2') + eq_(activity_data_migrated['title'], u'Title 2') def test_a_contemporary_activity_needs_no_migration(self): # An Activity created by a change under the current CKAN should not @@ -91,8 +97,13 @@ def test_revision_missing(self): # delete 'activity.data.package.resources' so it needs migrating activity = package_activity_list(dataset['id'], 0, 0)[0] activity = model.Activity.get(activity.id) - del activity.data['package']['resources'] + activity.data = {u'actor': None, + u'package': {u'title': 'Test Dataset'}} model.Session.commit() + model.Session.remove() + # double check that worked... + assert not \ + model.Activity.get(activity.id).data['package'].get('resources') errors = defaultdict(int) migrate_dataset(dataset['name'], errors) @@ -134,6 +145,13 @@ def test_revision_and_data_missing(self): class TestWipeActivityDetail(object): + def setup(self): + helpers.reset_db() + + @classmethod + def teardown_class(cls): + helpers.reset_db() + def test_wipe_activity_detail(self): dataset = factories.Dataset() user = factories.User() From 11113ec66a8ffeea3dc4558e2853a5c188d29e44 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 1 Mar 2019 16:30:12 +0000 Subject: [PATCH 14/21] Cope with all errors migrating revisions --- ckan/migration/migrate_package_activity.py | 14 +++++--- .../test_migrate_package_activity.py | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/ckan/migration/migrate_package_activity.py b/ckan/migration/migrate_package_activity.py index 63463440e27..bc02d47353b 100644 --- a/ckan/migration/migrate_package_activity.py +++ b/ckan/migration/migrate_package_activity.py @@ -32,6 +32,7 @@ import sys from collections import defaultdict from six.moves import input +from six import text_type # not importing anything from ckan until after the arg parsing, to fail on bad # args quickly. @@ -149,10 +150,15 @@ def migrate_dataset(dataset_name, errors): dataset = logic.get_action(u'package_show')( context, {u'id': activity[u'object_id'], u'include_tracking': False}) - except logic.NotFound as exc: - print(u' Revision missing! Skipping this version ' - '(revision_id={})'.format(activity[u'revision_id'])) - errors['Revision missing'] += 1 + except Exception as exc: + if isinstance(exc, logic.NotFound): + error_msg = u'Revision missing' + else: + error_msg = text_type(exc) + print(u' Error: {}! Skipping this version ' + '(revision_id={})' + .format(error_msg, activity[u'revision_id'])) + errors[error_msg] += 1 # We shouldn't leave the activity.data['package'] with missing # resources, extras & tags, which could cause the package_read # template to raise an exception, when user clicks "View this diff --git a/ckan/tests/migration/test_migrate_package_activity.py b/ckan/tests/migration/test_migrate_package_activity.py index 6efd486e5cb..cca85f0572c 100644 --- a/ckan/tests/migration/test_migrate_package_activity.py +++ b/ckan/tests/migration/test_migrate_package_activity.py @@ -4,12 +4,15 @@ from nose.tools import assert_equal as eq_ from collections import defaultdict +import mock + import ckan.tests.factories as factories import ckan.tests.helpers as helpers from ckan.migration.migrate_package_activity import (migrate_dataset, wipe_activity_detail) from ckan.model.activity import package_activity_list from ckan import model +import ckan.logic class TestMigrateDataset(object): @@ -143,6 +146,36 @@ def test_revision_and_data_missing(self): eq_(activity_data_migrated['package']['title'], u'unknown') assert u'resources' not in activity_data_migrated['package'] + def test_package_show_error(self): + dataset = factories.Dataset(resources=[ + {u'url': u'http://example.com/a.csv', u'format': u'csv'} + ]) + # delete 'activity.data.package.resources' so it needs migrating + activity = package_activity_list(dataset['id'], 0, 0)[0] + activity = model.Activity.get(activity.id) + activity.data = {u'actor': None, + u'package': {u'title': 'Test Dataset'}} + model.Session.commit() + model.Session.remove() + # double check that worked... + assert not \ + model.Activity.get(activity.id).data['package'].get('resources') + + errors = defaultdict(int) + # package_show raises an exception - could be because data doesn't + # conform to the latest dataset schema or is incompatible with + # currently installed plugins. Those errors shouldn't prevent the + # migration from going ahead. + ckan.logic._actions['package_show'] = \ + mock.MagicMock(side_effect=Exception('Schema error')) + try: + migrate_dataset(dataset['name'], errors) + finally: + # restore package_show + ckan.logic.clear_actions_cache() + + eq_(dict(errors), {u'Schema error': 1}) + class TestWipeActivityDetail(object): def setup(self): From bdf6c0e9409064c985f6da83e89cabebf4dadddf Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 1 Mar 2019 16:38:51 +0000 Subject: [PATCH 15/21] Pep8 --- ckan/migration/migrate_package_activity.py | 4 ++-- .../migration/test_migrate_package_activity.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ckan/migration/migrate_package_activity.py b/ckan/migration/migrate_package_activity.py index bc02d47353b..ef30387dcd1 100644 --- a/ckan/migration/migrate_package_activity.py +++ b/ckan/migration/migrate_package_activity.py @@ -143,8 +143,8 @@ def migrate_dataset(dataset_name, errors): get_context(), for_view=False, revision_id=activity[u'revision_id'], - use_cache=False, # avoid the cache (which would give us the - # latest revision) + use_cache=False, # avoid the cache (which would give us the + # latest revision) ) try: dataset = logic.get_action(u'package_show')( diff --git a/ckan/tests/migration/test_migrate_package_activity.py b/ckan/tests/migration/test_migrate_package_activity.py index cca85f0572c..c064817d881 100644 --- a/ckan/tests/migration/test_migrate_package_activity.py +++ b/ckan/tests/migration/test_migrate_package_activity.py @@ -60,12 +60,12 @@ def test_migration_with_multiple_revisions(self): # Remove 'activity.data.package.resources' to provoke the migration to # regenerate it from package_revision (etc) activity = model.Activity.get(activity.id) - activity.data = {u'actor': None, u'package': {u'title': 'Title 2'}} + activity.data = {u'actor': None, u'package': {u'title': u'Title 2'}} model.Session.commit() model.Session.remove() # double check that worked... assert not \ - model.Activity.get(activity.id).data['package'].get('resources') + model.Activity.get(activity.id).data['package'].get(u'resources') migrate_dataset(dataset['name'], {}) @@ -101,12 +101,12 @@ def test_revision_missing(self): activity = package_activity_list(dataset['id'], 0, 0)[0] activity = model.Activity.get(activity.id) activity.data = {u'actor': None, - u'package': {u'title': 'Test Dataset'}} + u'package': {u'title': u'Test Dataset'}} model.Session.commit() model.Session.remove() # double check that worked... assert not \ - model.Activity.get(activity.id).data['package'].get('resources') + model.Activity.get(activity.id).data['package'].get(u'resources') errors = defaultdict(int) migrate_dataset(dataset['name'], errors) @@ -154,12 +154,12 @@ def test_package_show_error(self): activity = package_activity_list(dataset['id'], 0, 0)[0] activity = model.Activity.get(activity.id) activity.data = {u'actor': None, - u'package': {u'title': 'Test Dataset'}} + u'package': {u'title': u'Test Dataset'}} model.Session.commit() model.Session.remove() # double check that worked... assert not \ - model.Activity.get(activity.id).data['package'].get('resources') + model.Activity.get(activity.id).data['package'].get(u'resources') errors = defaultdict(int) # package_show raises an exception - could be because data doesn't @@ -167,7 +167,7 @@ def test_package_show_error(self): # currently installed plugins. Those errors shouldn't prevent the # migration from going ahead. ckan.logic._actions['package_show'] = \ - mock.MagicMock(side_effect=Exception('Schema error')) + mock.MagicMock(side_effect=Exception(u'Schema error')) try: migrate_dataset(dataset['name'], errors) finally: From 2e49024e7ca59a689300db1c4dc996d11ef4cb01 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 1 Mar 2019 17:14:14 +0000 Subject: [PATCH 16/21] Add migration so that the check in "db upgrade" never has to be run after that (it can be expensive for large sites) --- .../088_package_activity_migration_check.py | 20 +++++++++++++++++++ ckan/model/__init__.py | 6 ++++-- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 ckan/migration/versions/088_package_activity_migration_check.py diff --git a/ckan/migration/versions/088_package_activity_migration_check.py b/ckan/migration/versions/088_package_activity_migration_check.py new file mode 100644 index 00000000000..2cd02c8b517 --- /dev/null +++ b/ckan/migration/versions/088_package_activity_migration_check.py @@ -0,0 +1,20 @@ +# encoding: utf-8 + +import sys + +from ckan.migration.migrate_package_activity import num_unmigrated + + +def upgrade(migrate_engine): + num_unmigrated_dataset_activities = num_unmigrated(migrate_engine) + if num_unmigrated_dataset_activities: + print(''' + !!! ERROR !!! + You have {num_unmigrated} unmigrated package activities. + + You cannot do this db upgrade until you completed the package activity + migration first. Full instructions for this situation are here: + + https://github.com/ckan/ckan/wiki/Migrate-package-activity#if-you-tried-to-upgrade-from-ckan-28-or-earlier-to-ckan-29-and-it-stopped-at-paster-db-upgrade + '''.format(num_unmigrated=num_unmigrated_dataset_activities)) + sys.exit(1) diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py index 57cb60917b5..36d881b268b 100644 --- a/ckan/model/__init__.py +++ b/ckan/model/__init__.py @@ -275,8 +275,10 @@ def upgrade_db(self, version=None): self.setup_migration_version_control() version_before = mig.db_version(self.metadata.bind, self.migrate_repository) from ckan.migration.migrate_package_activity import num_unmigrated - # if still at version 0 there can't be any activities needing migrating - if version_before > 0: + # If still at version 0 there can't be any activities needing + # migrating. This check is also done in migration 88, so if you've + # upgraded that far then we don't need to do it again. + if 0 < version_before < 88: num_unmigrated_dataset_activities = num_unmigrated(meta.engine) if num_unmigrated_dataset_activities: print(''' From f1ad20662a429c749b6dc7a88b6c587b0468adfa Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Mon, 4 Mar 2019 15:02:17 +0200 Subject: [PATCH 17/21] Remove calls to deprecated _resource_preview --- ckan/controllers/package.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 29ac0512bff..a385ecb901c 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -393,14 +393,13 @@ def read(self, id): # can the resources be previewed? for resource in c.pkg_dict['resources']: - # Backwards compatibility with preview interface - resource['can_be_previewed'] = self._resource_preview( - {'resource': resource, 'package': c.pkg_dict}) - resource_views = get_action('resource_view_list')( context, {'id': resource['id']}) resource['has_views'] = len(resource_views) > 0 + # Backwards compatibility with preview interface + resource['can_be_previewed'] = bool(len(resource_views)) + package_type = c.pkg_dict['type'] or 'dataset' self._setup_template_variables(context, {'id': id}, package_type=package_type) @@ -1116,13 +1115,12 @@ def resource_read(self, id, resource_id): c.datastore_api = '%s/api/action' % \ config.get('ckan.site_url', '').rstrip('/') - c.resource['can_be_previewed'] = self._resource_preview( - {'resource': c.resource, 'package': c.package}) - resource_views = get_action('resource_view_list')( context, {'id': resource_id}) c.resource['has_views'] = len(resource_views) > 0 + c.resource['can_be_previewed'] = bool(len(resource_views)) + current_resource_view = None view_id = request.GET.get('view_id') if c.resource['can_be_previewed'] and not view_id: From ad8883810a9238c25c39614a63a82863f278d40b Mon Sep 17 00:00:00 2001 From: Shahar Evron Date: Wed, 6 Mar 2019 10:21:33 +0200 Subject: [PATCH 18/21] Fix block / div nesting in the user/read_base template --- ckan/templates/user/read_base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/user/read_base.html b/ckan/templates/user/read_base.html index cf652b78def..e5e635abd54 100644 --- a/ckan/templates/user/read_base.html +++ b/ckan/templates/user/read_base.html @@ -97,8 +97,8 @@

    {{ user.display_name }}

    {{ user.apikey }}
    {% endif %} - {% endblock %} + {% endblock %} {% endblock %} From 1cccbdb42025417fb350b0bd27f025d3b94a2228 Mon Sep 17 00:00:00 2001 From: cclauss Date: Sat, 9 Mar 2019 09:05:25 +0100 Subject: [PATCH 19/21] Travis CI: The sudo tag is deprecated [Travis are now recommending removing the __sudo__ tag](https://blog.travis-ci.com/2018-11-19-required-linux-infrastructure-migration). "_If you currently specify __sudo: false__ in your __.travis.yml__, we recommend removing that configuration_" --- .travis.yml | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6f1ea86126a..0531a077fef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,22 @@ group: travis_latest language: python +flake8-steps: &flake8-steps + env: FLAKE8=true + cache: pip + install: pip install flake8 + before_script: + - flake8 --version + # stop the build if there are Python syntax errors or undefined names + - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ./ckan/include/rjsmin.py + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + script: + - true + matrix: include: - python: "2.7" - sudo: required services: - docker @@ -29,31 +41,8 @@ matrix: - docker ps -a - python: "2.7" - env: FLAKE8=true - cache: pip - install: - - pip install flake8 - before_script: - - flake8 --version - # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ./ckan/include/rjsmin.py - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - script: - - true + <<: *flake8-steps - python: "3.7" - env: FLAKE8=true dist: xenial # required for Python 3.7 - sudo: required # required for Python 3.7 - cache: pip - install: - - pip install flake8 - before_script: - - flake8 --version - # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - script: - - true + <<: *flake8-steps From 28daa10bbefb4eca8d41f00d467ef2b239e2b3fc Mon Sep 17 00:00:00 2001 From: Bruno Lavoie Date: Mon, 11 Mar 2019 15:46:00 -0400 Subject: [PATCH 20/21] Do not prefix with schema public Fix the migration to be schema agnostic, and let PostgreSQL Search Path work as expected. Also, quote user table name because it is a reserved word. --- .../versions/065_add_email_notifications_preference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/migration/versions/065_add_email_notifications_preference.py b/ckan/migration/versions/065_add_email_notifications_preference.py index ced562eb589..c6b7e3786e0 100644 --- a/ckan/migration/versions/065_add_email_notifications_preference.py +++ b/ckan/migration/versions/065_add_email_notifications_preference.py @@ -7,6 +7,6 @@ def upgrade(migrate_engine): metadata = MetaData() metadata.bind = migrate_engine migrate_engine.execute(''' -ALTER TABLE public.user +ALTER TABLE "user" ADD COLUMN activity_streams_email_notifications BOOLEAN DEFAULT FALSE; ''') From f54b4ee9b54d00c41c50d7318c0f2430c44f9d4c Mon Sep 17 00:00:00 2001 From: Bruno Lavoie Date: Mon, 11 Mar 2019 15:48:53 -0400 Subject: [PATCH 21/21] [#4678] Do not prefix with schema public Fix the migration to be schema agnostic, and let PostgreSQL Search Path work as expected. Also, quote user table name because it is a reserved word. --- .../versions/065_add_email_notifications_preference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/migration/versions/065_add_email_notifications_preference.py b/ckan/migration/versions/065_add_email_notifications_preference.py index ced562eb589..c6b7e3786e0 100644 --- a/ckan/migration/versions/065_add_email_notifications_preference.py +++ b/ckan/migration/versions/065_add_email_notifications_preference.py @@ -7,6 +7,6 @@ def upgrade(migrate_engine): metadata = MetaData() metadata.bind = migrate_engine migrate_engine.execute(''' -ALTER TABLE public.user +ALTER TABLE "user" ADD COLUMN activity_streams_email_notifications BOOLEAN DEFAULT FALSE; ''')