From 3a53ab291f338c911a1ce4bcf6a8ed7ce92aeb53 Mon Sep 17 00:00:00 2001 From: Mitchell Stanton-Cook Date: Mon, 18 Apr 2016 16:15:58 +1000 Subject: [PATCH 01/54] [2939] & [1651] How to solr-jetty JSP support There are still issues with solr-jetty. The documentation additions here provide a recipe to handle/fix the 'HTTP ERROR 500... JSP support not configured' error. --- doc/maintaining/installing/install-from-source.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index a2e2b23b959..5f2a2b3aaa3 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -295,6 +295,20 @@ installed, we need to install and configure Solr. JAVA_HOME=/usr/lib/jvm/java-6-openjdk-i386/ + .. note:: + + If you get the message ``HTTP ERROR 500... JSP support not configured.`` + then you will need to install a solr-jetty bug fix. You do this by:: + + cd /tmp + wget https://launchpad.net/~vshn/+archive/ubuntu/solr/+files/solr-jetty-jsp-fix_1.0.2_all.deb + sudo dpkg -i solr-jetty-jsp-fix_1.0.2_all.deb + sudo service solr restart + + You should now see a welcome page from Solr if you open + http://localhost:8983/solr/ in your web browser (replace localhost with + your server address if needed). + #. Replace the default ``schema.xml`` file with a symlink to the CKAN schema file included in the sources. From fb5cef3c10f6001fb0b684789f5c75b5537136f6 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 6 Feb 2017 17:36:51 -0500 Subject: [PATCH 02/54] [#3428] replace db._get_engine with db.get_(read|write)_engine --- ckanext/datastore/db.py | 35 +++++++++++------------ ckanext/datastore/logic/action.py | 37 +++++++------------------ ckanext/datastore/plugin.py | 13 +++------ ckanext/datastore/tests/test_create.py | 6 ++-- ckanext/datastore/tests/test_delete.py | 3 +- ckanext/datastore/tests/test_dump.py | 3 +- ckanext/datastore/tests/test_helpers.py | 4 +-- ckanext/datastore/tests/test_search.py | 10 ++----- ckanext/datastore/tests/test_unit.py | 3 +- ckanext/datastore/tests/test_upsert.py | 9 ++---- 10 files changed, 44 insertions(+), 79 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 140b09556fe..cf4c96e91f4 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -98,9 +98,16 @@ def _is_valid_table_name(name): return _is_valid_field_name(name) -def _get_engine(data_dict): +def get_read_engine(): + return _get_engine_from_url(config['ckan.datastore.read_url']) + + +def get_write_engine(): + return _get_engine_from_url(config['ckan.datastore.write_url']) + + +def _get_engine_from_url(connection_url): '''Get either read or write engine.''' - connection_url = data_dict['connection_url'] engine = _engines.get(connection_url) if not engine: @@ -127,9 +134,7 @@ def _cache_types(context): log.info("Create nested type. Native JSON: {0!r}".format( native_json)) - data_dict = { - 'connection_url': config['ckan.datastore.write_url']} - engine = _get_engine(data_dict) + engine = get_write_engine() with engine.begin() as connection: connection.execute( 'CREATE TYPE "nested" AS (json {0}, extra text)'.format( @@ -921,7 +926,6 @@ def validate(context, data_dict): fields_types) # Remove default elements in data_dict - del data_dict_copy['connection_url'] del data_dict_copy['resource_id'] data_dict_copy.pop('id', None) @@ -1056,7 +1060,7 @@ def create(context, data_dict): :raises InvalidDataError: if there is an invalid value in the given data ''' - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) _cache_types(context) @@ -1121,7 +1125,7 @@ def upsert(context, data_dict): Any error results in total failure! For now pass back the actual error. Should be transactional. ''' - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) @@ -1164,7 +1168,7 @@ def upsert(context, data_dict): def delete(context, data_dict): - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() _cache_types(context) @@ -1188,7 +1192,7 @@ def delete(context, data_dict): def search(context, data_dict): - engine = _get_engine(data_dict) + engine = get_read_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) _cache_types(context) @@ -1215,7 +1219,7 @@ def search(context, data_dict): def search_sql(context, data_dict): - engine = _get_engine(data_dict) + engine = get_read_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) _cache_types(context) @@ -1307,7 +1311,7 @@ def _change_privilege(context, data_dict, what): def make_private(context, data_dict): log.info('Making resource {resource_id!r} private'.format(**data_dict)) - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() trans = context['connection'].begin() try: @@ -1319,7 +1323,7 @@ def make_private(context, data_dict): def make_public(context, data_dict): log.info('Making resource {resource_id!r} public'.format(**data_dict)) - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() trans = context['connection'].begin() try: @@ -1332,10 +1336,7 @@ def make_public(context, data_dict): def get_all_resources_ids_in_datastore(): read_url = config.get('ckan.datastore.read_url') write_url = config.get('ckan.datastore.write_url') - data_dict = { - 'connection_url': read_url or write_url - } resources_sql = sqlalchemy.text(u'''SELECT name FROM "_table_metadata" WHERE alias_of IS NULL''') - query = _get_engine(data_dict).execute(resources_sql) + query = get_read_engine().execute(resources_sql) return [q[0] for q in query.fetchall()] diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index e519670bf19..1236f379a8a 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -122,8 +122,6 @@ def datastore_create(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = config['ckan.datastore.write_url'] - # validate aliases aliases = datastore_helpers.get_list(data_dict.get('aliases', [])) for alias in aliases: @@ -193,7 +191,6 @@ def datastore_create(context, data_dict): result.pop('id', None) result.pop('private', None) - result.pop('connection_url') return result @@ -247,12 +244,10 @@ def datastore_upsert(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = config['ckan.datastore.write_url'] - res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + results = db.get_write_engine().execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: @@ -262,7 +257,6 @@ def datastore_upsert(context, data_dict): result = db.upsert(context, data_dict) result.pop('id', None) - result.pop('connection_url') return result @@ -289,11 +283,9 @@ def _type_lookup(t): resource_id = _get_or_bust(data_dict, 'id') resource = p.toolkit.get_action('resource_show')(context, {'id':resource_id}) - data_dict['connection_url'] = config['ckan.datastore.read_url'] - resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=resource_id) + results = db.get_read_engine().execute(resources_sql, id=resource_id) res_exists = results.rowcount > 0 if not res_exists: raise p.toolkit.ObjectNotFound(p.toolkit._( @@ -309,7 +301,7 @@ def _type_lookup(t): SELECT column_name, data_type FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = :resource_id; ''') - schema_results = db._get_engine(data_dict).execute(schema_sql, resource_id=resource_id) + schema_results = db.get_read_engine().execute(schema_sql, resource_id=resource_id) for row in schema_results.fetchall(): k = row[0] v = row[1] @@ -322,7 +314,7 @@ def _type_lookup(t): meta_sql = sqlalchemy.text(u''' SELECT count(_id) FROM "{0}"; '''.format(resource_id)) - meta_results = db._get_engine(data_dict).execute(meta_sql, resource_id=resource_id) + meta_results = db.get_read_engine().execute(meta_sql, resource_id=resource_id) info['meta']['count'] = meta_results.fetchone()[0] finally: if schema_results: @@ -374,12 +366,10 @@ def datastore_delete(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = config['ckan.datastore.write_url'] - res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + results = db.get_read_engine().execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: @@ -403,7 +393,6 @@ def datastore_delete(context, data_dict): 'datastore_active': False}) result.pop('id', None) - result.pop('connection_url') return result @@ -473,11 +462,13 @@ def datastore_search(context, data_dict): raise p.toolkit.ValidationError(errors) res_id = data_dict['resource_id'] - data_dict['connection_url'] = config['ckan.datastore.write_url'] resources_sql = sqlalchemy.text(u'''SELECT alias_of FROM "_table_metadata" WHERE name = :id''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + # XXX: write connection because of private tables, we + # should be able to make this read once we stop using pg + # permissions enforcement + results = db.get_write_engine().execute(resources_sql, id=res_id) # Resource only has to exist in the datastore (because it could be an alias) if not results.rowcount > 0: @@ -495,7 +486,6 @@ def datastore_search(context, data_dict): result = db.search(context, data_dict) result.pop('id', None) - result.pop('connection_url') return result @@ -537,11 +527,8 @@ def datastore_search_sql(context, data_dict): p.toolkit.check_access('datastore_search_sql', context, data_dict) - data_dict['connection_url'] = config['ckan.datastore.read_url'] - result = db.search_sql(context, data_dict) result.pop('id', None) - result.pop('connection_url') return result @@ -560,8 +547,6 @@ def datastore_make_private(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - data_dict['connection_url'] = config['ckan.datastore.write_url'] - if not _resource_exists(context, data_dict): raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(res_id) @@ -586,8 +571,6 @@ def datastore_make_public(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - data_dict['connection_url'] = config['ckan.datastore.write_url'] - if not _resource_exists(context, data_dict): raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(res_id) @@ -607,7 +590,7 @@ def _resource_exists(context, data_dict): resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + results = db.get_read_engine().execute(resources_sql, id=res_id) return results.rowcount > 0 diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 073ffcee214..36ce2c552fe 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -33,9 +33,7 @@ def _is_legacy_mode(config): Returns True if `ckan.datastore.read_url` is not set in the provided config object or CKAN is running on Postgres < 9.x ''' - write_url = config.get('ckan.datastore.write_url') - - engine = db._get_engine({'connection_url': write_url}) + engine = db.get_write_engine() connection = engine.connect() return (not config.get('ckan.datastore.read_url') or @@ -110,8 +108,7 @@ def configure(self, config): else: self.read_url = self.config['ckan.datastore.read_url'] - self.read_engine = db._get_engine( - {'connection_url': self.read_url}) + self.read_engine = db.get_read_engine() if not model.engine_is_pg(self.read_engine): log.warn('We detected that you do not use a PostgreSQL ' 'database. The DataStore will NOT work and DataStore ' @@ -138,7 +135,6 @@ def notify(self, entity, operation=None): for resource in entity.resources: try: func(context, { - 'connection_url': self.write_url, 'resource_id': resource.id}) except p.toolkit.ObjectNotFound: pass @@ -172,7 +168,7 @@ def _is_read_only_database(self): ''' Returns True if no connection has CREATE privileges on the public schema. This is the case if replication is enabled.''' for url in [self.ckan_url, self.write_url, self.read_url]: - connection = db._get_engine({'connection_url': url}).connect() + connection = db._get_engine_from_url(url).connect() try: sql = u"SELECT has_schema_privilege('public', 'CREATE')" is_writable = connection.execute(sql).first()[0] @@ -198,8 +194,7 @@ def _read_connection_has_correct_privileges(self): only user. A table is created by the write user to test the read only user. ''' - write_connection = db._get_engine( - {'connection_url': self.write_url}).connect() + write_connection = db.get_write_engine().connect() read_connection_user = sa_url.make_url(self.read_url).username drop_foo_sql = u'DROP TABLE IF EXISTS _foo' diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index a95bcf68ef8..22d0e5c71e1 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -193,8 +193,7 @@ def _get_index_names(self, resource_id): return [result[0] for result in results] def _execute_sql(self, sql, *args): - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() session = orm.scoped_session(orm.sessionmaker(bind=engine)) return session.connection().execute(sql, *args) @@ -255,8 +254,7 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py index 53f9d487f06..ace6d4f5c38 100644 --- a/ckanext/datastore/tests/test_delete.py +++ b/ckanext/datastore/tests/test_delete.py @@ -42,8 +42,7 @@ def setup_class(cls): 'rating with %': '42%'}] } - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 8b3abbd58ba..cd7c8ac980a 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -80,8 +80,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine({ - 'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod diff --git a/ckanext/datastore/tests/test_helpers.py b/ckanext/datastore/tests/test_helpers.py index 9cc05c3dded..eed67000c20 100644 --- a/ckanext/datastore/tests/test_helpers.py +++ b/ckanext/datastore/tests/test_helpers.py @@ -70,9 +70,7 @@ def setup_class(cls): if not config.get('ckan.datastore.read_url'): raise nose.SkipTest('Datastore runs on legacy mode, skipping...') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']} - ) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) datastore_test_helpers.clear_db(cls.Session) diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py index c472b52bdc3..f1b0fe64c48 100644 --- a/ckanext/datastore/tests/test_search.py +++ b/ckanext/datastore/tests/test_search.py @@ -165,9 +165,7 @@ def setup_class(cls): u'characters': None, u'rating with %': u'99%'}] - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']} - ) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -838,8 +836,7 @@ def setup_class(cls): u'published': None}] cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}] - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -941,8 +938,7 @@ def test_read_private(self): 'user': self.sysadmin_user.name, 'model': model} data_dict = { - 'resource_id': self.data['resource_id'], - 'connection_url': config['ckan.datastore.write_url']} + 'resource_id': self.data['resource_id']} p.toolkit.get_action('datastore_make_private')(context, data_dict) query = 'SELECT * FROM "{0}"'.format(self.data['resource_id']) data = {'sql': query} diff --git a/ckanext/datastore/tests/test_unit.py b/ckanext/datastore/tests/test_unit.py index c02d30c60d6..2a74e3386cd 100644 --- a/ckanext/datastore/tests/test_unit.py +++ b/ckanext/datastore/tests/test_unit.py @@ -35,8 +35,7 @@ def test_is_valid_table_name(self): def test_pg_version_check(self): if not tests.is_datastore_supported(): raise nose.SkipTest("Datastore not supported") - engine = db._get_engine( - {'connection_url': config['sqlalchemy.url']}) + engine = db._get_engine_from_url(config['sqlalchemy.url']) connection = engine.connect() assert db._pg_version_is_at_least(connection, '8.0') assert not db._pg_version_is_at_least(connection, '10.0') diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py index b7a09bf7dae..e798bec38dc 100644 --- a/ckanext/datastore/tests/test_upsert.py +++ b/ckanext/datastore/tests/test_upsert.py @@ -113,8 +113,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -365,8 +364,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -472,8 +470,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod From 361acc14ba743c04c134cbc36369ef4cfbabe0e5 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 7 Feb 2017 11:47:24 -0500 Subject: [PATCH 03/54] [#3428] one more db._get_engine replacement --- ckanext/datapusher/tests/test_interfaces.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ckanext/datapusher/tests/test_interfaces.py b/ckanext/datapusher/tests/test_interfaces.py index 49b8efd7fad..8439dbff45c 100644 --- a/ckanext/datapusher/tests/test_interfaces.py +++ b/ckanext/datapusher/tests/test_interfaces.py @@ -61,8 +61,7 @@ def setup_class(cls): cls.sysadmin_user = factories.User(name='testsysadmin', sysadmin=True) cls.normal_user = factories.User(name='annafan') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod From 7cc6c377353c518b673483316d1e1bbbfa270575 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 7 Feb 2017 12:36:27 -0500 Subject: [PATCH 04/54] [#3428] one more db._get_engine replacement --- ckanext/datapusher/tests/test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index d603ca386ab..cdf4024374d 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -68,8 +68,7 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) From c58d70f5690721a9b49e853f2f0636797d879cf5 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 13 Feb 2017 15:47:33 -0500 Subject: [PATCH 05/54] [#3428] datastore_function actions --- ckan/lib/navl/validators.py | 6 +++++ ckanext/datastore/db.py | 37 +++++++++++++++++++++++++++++-- ckanext/datastore/logic/action.py | 37 +++++++++++++++++++++++++++++++ ckanext/datastore/logic/auth.py | 9 ++++++++ ckanext/datastore/logic/schema.py | 17 ++++++++++++++ 5 files changed, 104 insertions(+), 2 deletions(-) diff --git a/ckan/lib/navl/validators.py b/ckan/lib/navl/validators.py index 6e508c846a8..cfcb1a2d5d2 100644 --- a/ckan/lib/navl/validators.py +++ b/ckan/lib/navl/validators.py @@ -117,3 +117,9 @@ def convert_int(value, context): except ValueError: raise Invalid(_('Please enter an integer value')) +def unicode_only(value): + '''Accept only unicode values''' + + if not isinstance(value, unicode): + raise Invalid(_('Must be a Unicode string value')) + return value diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 063c220cee5..62b41ce9ef9 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1384,9 +1384,42 @@ def make_public(context, data_dict): def get_all_resources_ids_in_datastore(): - read_url = config.get('ckan.datastore.read_url') - write_url = config.get('ckan.datastore.write_url') resources_sql = sqlalchemy.text(u'''SELECT name FROM "_table_metadata" WHERE alias_of IS NULL''') query = get_read_engine().execute(resources_sql) return [q[0] for q in query.fetchall()] + + +def create_trigger_function(name, definition, or_replace): + sql = u''' + CREATE {or_replace} FUNCTION + {name}() RETURNS trigger AS {definition} + LANGUAGE plpgsql;'''.format( + or_replace=u'OR REPLACE' if or_replace else u'', + name=datastore_helpers.identifier(name), + definition=datastore_helpers.literal(definition)) + + _write_engine_execute(sql) + + +def drop_function(name, if_exists): + sql = u''' + DROP FUNCTION {if_exists} {name}; + '''.format( + if_exists=u'IF EXISTS' if if_exists else u'', + name=datastore_helpers.identifier(name)) + + _write_engine_execute(sql) + + +def _write_engine_execute(sql): + connection = get_write_engine().connect() + trans = connection.begin() + try: + connection.execute(sql) + trans.commit() + except Exception: + trans.rollback() + raise + finally: + connection.close() diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 1236f379a8a..397647687b8 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -605,3 +605,40 @@ def _check_read_only(context, resource_id): 'read-only': ['Cannot edit read-only resource. Either pass' '"force=True" or change url-type to "datastore"'] }) + + +@logic.validate(dsschema.datastore_function_create_schema) +def datastore_function_create(context, data_dict): + u''' + Create a trigger function for use with datastore_create + + :param name: function name + :type name: string + :param or_replace: True to replace if function already exists + (default: False) + :type or_replace: bool + :param rettype: set to 'trigger' + (only trigger functions may be created at this time) + :type rettype: string + :param definition: PL/pgSQL function body for trigger function + :type definition: string + ''' + p.toolkit.check_access('datastore_function_create', context, data_dict) + + db.create_trigger_function( + name=data_dict['name'], + definition=data_dict['definition'], + or_replace=data_dict['or_replace']) + + +@logic.validate(dsschema.datastore_function_delete_schema) +def datastore_function_delete(context, data_dict): + u''' + Delete a trigger function + + :param name: function name + :type name: string + ''' + p.toolkit.check_access('datastore_function_delete', context, data_dict) + + db.drop_function(data_dict['name']) diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py index fc54f82bd5e..824a73ee46b 100644 --- a/ckanext/datastore/logic/auth.py +++ b/ckanext/datastore/logic/auth.py @@ -58,3 +58,12 @@ def datastore_search_sql(context, data_dict): def datastore_change_permissions(context, data_dict): return datastore_auth(context, data_dict) + + +def datastore_function_create(context, data_dict): + '''sysadmin-only: functions can be used to skip access checks''' + return {'success': False} + + +def datastore_function_delete(context, data_dict): + return {'success': False} diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 2d7d7f69ae4..88238858cf4 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -16,6 +16,7 @@ boolean_validator = get_validator('boolean_validator') int_validator = get_validator('int_validator') OneOf = get_validator('OneOf') +unicode_only = get_validator('unicode_only') def rename(old, new): @@ -153,3 +154,19 @@ def datastore_search_schema(): '__before': [rename('id', 'resource_id')] } return schema + + +def datastore_function_create_schema(): + return { + 'name': [unicode_only, not_empty], + 'or_replace': [default(False), boolean_validator], + # we're only exposing functions for triggers at the moment + 'rettype': [unicode_only, OneOf(['trigger'])], + 'definition': [unicode_only], + } + + +def datastore_function_delete_schema(): + return { + 'name': [unicode_only, not_empty], + } From 24e8169cae103e383eefdb1d4e2905d10d4aa330 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 16 Feb 2017 18:33:49 -0500 Subject: [PATCH 06/54] [#3428] working triggers (exceptions not handled yet) --- ckanext/datastore/db.py | 35 ++++++++++++++++++++++++++++++- ckanext/datastore/logic/action.py | 4 ++++ ckanext/datastore/logic/schema.py | 12 +++++++++++ ckanext/datastore/plugin.py | 33 +++++++++++++++++------------ 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 62b41ce9ef9..22d1386761a 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1130,6 +1130,11 @@ def create(context, data_dict): create_table(context, data_dict) else: alter_table(context, data_dict) + if 'triggers' in data_dict: + _create_triggers( + context['connection'], + data_dict['resource_id'], + data_dict['triggers']) insert_data(context, data_dict) create_indexes(context, data_dict) create_alias(context, data_dict) @@ -1167,6 +1172,34 @@ def create(context, data_dict): context['connection'].close() +def _create_triggers(connection, resource_id, triggers): + u''' + Delete existing triggers on table then create triggers + + Currently our schema requires "before insert or update" + triggers run on each row, so we're not reading "when" + or "for_each" parameters from triggers list. + ''' + existing = connection.execute( + u'SELECT tgname FROM pg_trigger WHERE tgrelid = %s::regclass', + resource_id) + sql_list = ( + [u'DROP TRIGGER {name} ON {table}'.format( + name=datastore_helpers.identifier(r[0]), + table=datastore_helpers.identifier(resource_id)) + for r in existing] + + [u'''CREATE TRIGGER {name} + BEFORE INSERT OR UPDATE ON {table} + FOR EACH ROW EXECUTE PROCEDURE {function}()'''.format( + # 1000 triggers per table should be plenty + name=datastore_helpers.identifier(u't%03d' % i), + table=datastore_helpers.identifier(resource_id), + function=datastore_helpers.identifier(t['function'])) + for i, t in enumerate(triggers)]) + if sql_list: + connection.execute(u';\n'.join(sql_list)) + + def upsert(context, data_dict): ''' This method combines upsert insert and update on the datastore. The method @@ -1397,7 +1430,7 @@ def create_trigger_function(name, definition, or_replace): LANGUAGE plpgsql;'''.format( or_replace=u'OR REPLACE' if or_replace else u'', name=datastore_helpers.identifier(name), - definition=datastore_helpers.literal(definition)) + definition=datastore_helpers.literal_string(definition)) _write_engine_execute(sql) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 397647687b8..6b0c9c26272 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -55,6 +55,10 @@ def datastore_create(context, data_dict): :type primary_key: list or comma separated string :param indexes: indexes on table (optional) :type indexes: list or comma separated string + :param triggers: triggers to apply to this table, eg: [ + {"function": "trigger_clean_reference"}, + {"function": "trigger_check_codes"}] + :type triggers: list of dictionaries Please note that setting the ``aliases``, ``indexes`` or ``primary_key`` replaces the exising aliases or constraints. Setting ``records`` appends the provided records to the resource. diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 88238858cf4..9ee317fcb0b 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -17,6 +17,7 @@ int_validator = get_validator('int_validator') OneOf = get_validator('OneOf') unicode_only = get_validator('unicode_only') +default = get_validator('default') def rename(old, new): @@ -107,6 +108,17 @@ def datastore_create_schema(): }, 'primary_key': [ignore_missing, list_of_strings_or_string], 'indexes': [ignore_missing, list_of_strings_or_string], + 'triggers': { + 'when': [ + default(u'before insert or update'), + unicode_only, + OneOf([u'before insert or update'])], + 'for_each': [ + default(u'row'), + unicode_only, + OneOf([u'row'])], + 'function': [not_empty, unicode_only], + }, '__junk': [empty], '__before': [rename('id', 'resource_id')] } diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index f738a3ebaa8..80f23fa9677 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -216,12 +216,15 @@ def _read_connection_has_correct_privileges(self): return True def get_actions(self): - actions = {'datastore_create': action.datastore_create, - 'datastore_upsert': action.datastore_upsert, - 'datastore_delete': action.datastore_delete, - 'datastore_search': action.datastore_search, - 'datastore_info': action.datastore_info, - } + actions = { + 'datastore_create': action.datastore_create, + 'datastore_upsert': action.datastore_upsert, + 'datastore_delete': action.datastore_delete, + 'datastore_search': action.datastore_search, + 'datastore_info': action.datastore_info, + 'datastore_function_create': action.datastore_function_create, + 'datastore_function_delete': action.datastore_function_delete, + } if not self.legacy_mode: if self.enable_sql_search: # Only enable search_sql if the config does not disable it @@ -233,13 +236,17 @@ def get_actions(self): return actions def get_auth_functions(self): - return {'datastore_create': auth.datastore_create, - 'datastore_upsert': auth.datastore_upsert, - 'datastore_delete': auth.datastore_delete, - 'datastore_info': auth.datastore_info, - 'datastore_search': auth.datastore_search, - 'datastore_search_sql': auth.datastore_search_sql, - 'datastore_change_permissions': auth.datastore_change_permissions} + return { + 'datastore_create': auth.datastore_create, + 'datastore_upsert': auth.datastore_upsert, + 'datastore_delete': auth.datastore_delete, + 'datastore_info': auth.datastore_info, + 'datastore_search': auth.datastore_search, + 'datastore_search_sql': auth.datastore_search_sql, + 'datastore_change_permissions': auth.datastore_change_permissions, + 'datastore_function_create': auth.datastore_function_create, + 'datastore_function_delete': auth.datastore_function_delete, + } def before_map(self, m): m.connect( From 18f0d58f5df4adc355640c1b2b8e736320065212 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 23 Feb 2017 15:04:52 -0500 Subject: [PATCH 07/54] [#3428] start of function create tests --- ckan/tests/helpers.py | 9 ++++++++- ckanext/datastore/db.py | 6 +++++- ckanext/datastore/tests/test_create.py | 25 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index 0d0e80a64db..49d099f9fc1 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -179,7 +179,8 @@ class FunctionalTestBase(object): Allows configuration changes by overriding _apply_config_changes and resetting the CKAN config after your test class has run. It creates a webtest.TestApp at self.app for your class to use to make HTTP requests - to the CKAN web UI or API. + to the CKAN web UI or API. Also loads plugins defined by + _load_plugins in the class definition. If you're overriding methods that this class provides, like setup_class() and teardown_class(), make sure to use super() to call this class's methods @@ -196,10 +197,13 @@ def _get_test_app(cls): # leading _ because nose is terrible @classmethod def setup_class(cls): + import ckan.plugins as p # Make a copy of the Pylons config, so we can restore it in teardown. cls._original_config = dict(config) cls._apply_config_changes(config) cls._get_test_app() + for plugin in getattr(cls, '_load_plugins', []): + p.load(plugin) @classmethod def _apply_config_changes(cls, cfg): @@ -214,6 +218,9 @@ def setup(self): @classmethod def teardown_class(cls): + import ckan.plugins as p + for plugin in reversed(getattr(cls, '_load_plugins', [])): + p.unload(plugin) # Restore the Pylons config to its original values, in case any tests # changed any config settings. config.clear() diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 22d1386761a..5cb8813da7c 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1432,7 +1432,11 @@ def create_trigger_function(name, definition, or_replace): name=datastore_helpers.identifier(name), definition=datastore_helpers.literal_string(definition)) - _write_engine_execute(sql) + try: + _write_engine_execute(sql) + except ProgrammingError as pe: + raise ValidationError({ + u'definition': [pe.args[0].split('\n')[0].decode('utf8')]}) def drop_function(name, if_exists): diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 22d0e5c71e1..493a433e802 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -19,6 +19,7 @@ import ckanext.datastore.db as db from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type +from ckan.plugins.toolkit import ValidationError class TestDatastoreCreateNewTests(object): @@ -905,3 +906,27 @@ def test_datastore_create_with_invalid_data_value(self): assert res_dict['success'] is False assert res_dict['error']['__type'] == 'Validation Error' assert res_dict['error']['message'].startswith('The data was invalid') + + +class TestDatastoreFunctionCreateTests(helpers.FunctionalTestBase): + _load_plugins = [u'datastore'] + + def test_nop_trigger(self): + helpers.call_action( + u'datastore_function_create', + name=u'test_nop', + or_replace=True, + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + + def test_invalid_definition(self): + try: + helpers.call_action( + u'datastore_function_create', + name=u'test_invalid_def', + rettype=u'trigger', + definition=u'HELLO WORLD') + except ValidationError as ve: + assert_equal( + ve.error_dict['definition'], + [u'(ProgrammingError) syntax error at or near "HELLO"']) From 2324fe9ec8daff9dbf4a4cdb54b30b102755b223 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 27 Feb 2017 15:43:31 -0500 Subject: [PATCH 08/54] [#3428] properly clear functions in datastore tests --- ckanext/datastore/tests/helpers.py | 24 ++++++++++++++++++++++++ ckanext/datastore/tests/test_create.py | 8 +++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/ckanext/datastore/tests/helpers.py b/ckanext/datastore/tests/helpers.py index 7e707a0e75e..b88499beaf3 100644 --- a/ckanext/datastore/tests/helpers.py +++ b/ckanext/datastore/tests/helpers.py @@ -1,9 +1,13 @@ # encoding: utf-8 +from sqlalchemy import orm + import ckan.model as model import ckan.lib.cli as cli import ckan.plugins as p +from ckan.tests.helpers import FunctionalTestBase +from ckanext.datastore import db def extract(d, keys): @@ -17,6 +21,16 @@ def clear_db(Session): results = c.execute(drop_tables) for result in results: c.execute(result[0]) + + drop_functions_sql = u''' + SELECT 'drop function ' || quote_ident(proname) || '();' + FROM pg_proc INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid) + WHERE ns.nspname = 'public' + ''' + drop_functions = u''.join(r[0] for r in c.execute(drop_functions_sql)) + if drop_functions: + c.execute(drop_functions) + Session.commit() Session.remove() @@ -42,3 +56,13 @@ def set_url_type(resources, user): context, {'id': resource.id}) resource['url_type'] = 'datastore' p.toolkit.get_action('resource_update')(context, resource) + + +class DatastoreFunctionalTestBase(FunctionalTestBase): + _load_plugins = (u'datastore', ) + + @classmethod + def setup_class(cls): + engine = db.get_write_engine() + rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine))) + super(DatastoreFunctionalTestBase, cls).setup_class() diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 493a433e802..f9b24c25cd4 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -18,7 +18,8 @@ import ckan.tests.factories as factories import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type +from ckanext.datastore.tests.helpers import ( + rebuild_all_dbs, set_url_type, DatastoreFunctionalTestBase) from ckan.plugins.toolkit import ValidationError @@ -908,14 +909,11 @@ def test_datastore_create_with_invalid_data_value(self): assert res_dict['error']['message'].startswith('The data was invalid') -class TestDatastoreFunctionCreateTests(helpers.FunctionalTestBase): - _load_plugins = [u'datastore'] - +class TestDatastoreFunctionCreateTests(DatastoreFunctionalTestBase): def test_nop_trigger(self): helpers.call_action( u'datastore_function_create', name=u'test_nop', - or_replace=True, rettype=u'trigger', definition=u'BEGIN RETURN NEW; END;') From d644cc8133c97f088c721f556c7519eb471b0aa6 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 27 Feb 2017 16:38:16 -0500 Subject: [PATCH 09/54] [#3428] test redefinition and or_replace --- ckanext/datastore/db.py | 6 +++-- ckanext/datastore/tests/helpers.py | 3 ++- ckanext/datastore/tests/test_create.py | 37 ++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 5cb8813da7c..a9a33bccaba 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1435,8 +1435,10 @@ def create_trigger_function(name, definition, or_replace): try: _write_engine_execute(sql) except ProgrammingError as pe: - raise ValidationError({ - u'definition': [pe.args[0].split('\n')[0].decode('utf8')]}) + message = pe.args[0].split('\n')[0].decode('utf8') + key = u'name' if message.startswith( + u'(ProgrammingError) function') else u'definition' + raise ValidationError({key: [message.split(u') ', 1)[-1]]}) def drop_function(name, if_exists): diff --git a/ckanext/datastore/tests/helpers.py b/ckanext/datastore/tests/helpers.py index b88499beaf3..277253b82a6 100644 --- a/ckanext/datastore/tests/helpers.py +++ b/ckanext/datastore/tests/helpers.py @@ -24,7 +24,8 @@ def clear_db(Session): drop_functions_sql = u''' SELECT 'drop function ' || quote_ident(proname) || '();' - FROM pg_proc INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid) + FROM pg_proc + INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid) WHERE ns.nspname = 'public' ''' drop_functions = u''.join(r[0] for r in c.execute(drop_functions_sql)) diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index f9b24c25cd4..63b78d2a665 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -926,5 +926,38 @@ def test_invalid_definition(self): definition=u'HELLO WORLD') except ValidationError as ve: assert_equal( - ve.error_dict['definition'], - [u'(ProgrammingError) syntax error at or near "HELLO"']) + ve.error_dict, + {u'definition': + [u'syntax error at or near "HELLO"']}) + + def test_redefined_trigger(self): + helpers.call_action( + u'datastore_function_create', + name=u'test_redefined', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + try: + helpers.call_action( + u'datastore_function_create', + name=u'test_redefined', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'name':[ + u'function "test_redefined" already exists ' + u'with same argument types']}) + + def test_redefined_with_or_replace_trigger(self): + helpers.call_action( + u'datastore_function_create', + name=u'test_replaceme', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + helpers.call_action( + u'datastore_function_create', + name=u'test_replaceme', + or_replace=True, + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') From a4423c05502039c8f46e9532d2f597ac9df8713c Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 28 Feb 2017 10:27:42 -0500 Subject: [PATCH 10/54] [#3428] test datastore_create with missing function --- ckanext/datastore/db.py | 8 ++++++-- ckanext/datastore/tests/test_create.py | 26 +++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index a9a33bccaba..da7150b6267 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1196,8 +1196,12 @@ def _create_triggers(connection, resource_id, triggers): table=datastore_helpers.identifier(resource_id), function=datastore_helpers.identifier(t['function'])) for i, t in enumerate(triggers)]) - if sql_list: - connection.execute(u';\n'.join(sql_list)) + try: + if sql_list: + connection.execute(u';\n'.join(sql_list)) + except ProgrammingError as pe: + message = pe.args[0].split('\n')[0].decode('utf8') + raise ValidationError({u'triggers': [message.split(u') ', 1)[-1]]}) def upsert(context, data_dict): diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 63b78d2a665..c3a14ec25d9 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -909,7 +909,7 @@ def test_datastore_create_with_invalid_data_value(self): assert res_dict['error']['message'].startswith('The data was invalid') -class TestDatastoreFunctionCreateTests(DatastoreFunctionalTestBase): +class TestDatastoreFunctionCreate(DatastoreFunctionalTestBase): def test_nop_trigger(self): helpers.call_action( u'datastore_function_create', @@ -929,6 +929,8 @@ def test_invalid_definition(self): ve.error_dict, {u'definition': [u'syntax error at or near "HELLO"']}) + else: + assert 0, u'no validation error' def test_redefined_trigger(self): helpers.call_action( @@ -948,6 +950,8 @@ def test_redefined_trigger(self): {u'name':[ u'function "test_redefined" already exists ' u'with same argument types']}) + else: + assert 0, u'no validation error' def test_redefined_with_or_replace_trigger(self): helpers.call_action( @@ -961,3 +965,23 @@ def test_redefined_with_or_replace_trigger(self): or_replace=True, rettype=u'trigger', definition=u'BEGIN RETURN NEW; END;') + + +class TestDatastoreCreateTriggers(DatastoreFunctionalTestBase): + def test_create_with_missing_trigger(self): + ds = factories.Dataset() + + try: + helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + records=[{u'spam': u'SPAM'}, {u'spam': u'EGGS'}], + triggers=[{u'function': u'no_such_trigger_function'}]) + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'triggers':[ + u'function no_such_trigger_function() does not exist']}) + else: + assert 0, u'no validation error' From f1b1a93b7b553eb25aa24f408d4f177bc6c2d1b3 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 28 Feb 2017 10:48:20 -0500 Subject: [PATCH 11/54] [#3428] test triggers apply to _create and _upsert --- ckanext/datastore/tests/test_create.py | 54 ++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index c3a14ec25d9..fc2a08c75ee 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -985,3 +985,57 @@ def test_create_with_missing_trigger(self): u'function no_such_trigger_function() does not exist']}) else: assert 0, u'no validation error' + + def test_create_trigger_applies_to_records(self): + ds = factories.Dataset() + + helpers.call_action( + u'datastore_function_create', + name=u'spamify_trigger', + rettype=u'trigger', + definition=u''' + BEGIN + NEW.spam := 'spam spam ' || NEW.spam || ' spam'; + RETURN NEW; + END;''') + res = helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + records=[{u'spam': u'SPAM'}, {u'spam': u'EGGS'}], + triggers=[{u'function': u'spamify_trigger'}]) + assert_equal( + helpers.call_action( + u'datastore_search', + fields=[u'spam'], + resource_id=res['resource_id'])['records'], + [{u'spam': u'spam spam SPAM spam'}, {u'spam': u'spam spam EGGS spam'}]) + + def test_upsert_trigger_applies_to_records(self): + ds = factories.Dataset() + + helpers.call_action( + u'datastore_function_create', + name=u'more_spam_trigger', + rettype=u'trigger', + definition=u''' + BEGIN + NEW.spam := 'spam spam ' || NEW.spam || ' spam'; + RETURN NEW; + END;''') + res = helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + triggers=[{u'function': u'more_spam_trigger'}]) + helpers.call_action( + u'datastore_upsert', + method=u'insert', + resource_id=res['resource_id'], + records=[{u'spam': u'BEANS'}, {u'spam': u'SPAM'}]) + assert_equal( + helpers.call_action( + u'datastore_search', + fields=[u'spam'], + resource_id=res['resource_id'])['records'], + [{u'spam': u'spam spam BEANS spam'}, {u'spam': u'spam spam SPAM spam'}]) From 39832485604399cadaaf8033e32eb714c90be055 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 28 Feb 2017 11:22:24 -0500 Subject: [PATCH 12/54] [#3428] trigger exceptions raise ValidationErrors --- ckanext/datastore/db.py | 5 ++ ckanext/datastore/tests/test_create.py | 70 +++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index da7150b6267..2b3b05a49d5 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -719,6 +719,9 @@ def upsert_data(context, data_dict): toolkit._("The data was invalid (for example: a numeric value " "is out of range or was inserted into a text field)." )) + except sqlalchemy.exc.InternalError as err: + message = err.args[0].split('\n')[0].decode('utf8') + raise ValidationError({u'records': [message.split(u') ', 1)[-1]]}) elif method in [_UPDATE, _UPSERT]: unique_keys = _get_unique_key(context, data_dict) @@ -1457,6 +1460,8 @@ def drop_function(name, if_exists): def _write_engine_execute(sql): connection = get_write_engine().connect() + # No special meaning for '%' in sql parameter: + connection = connection.execution_options(no_parameters=True) trans = connection.begin() try: connection.execute(sql) diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index fc2a08c75ee..ba8c94adf08 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -1009,7 +1009,9 @@ def test_create_trigger_applies_to_records(self): u'datastore_search', fields=[u'spam'], resource_id=res['resource_id'])['records'], - [{u'spam': u'spam spam SPAM spam'}, {u'spam': u'spam spam EGGS spam'}]) + [ + {u'spam': u'spam spam SPAM spam'}, + {u'spam': u'spam spam EGGS spam'}]) def test_upsert_trigger_applies_to_records(self): ds = factories.Dataset() @@ -1038,4 +1040,68 @@ def test_upsert_trigger_applies_to_records(self): u'datastore_search', fields=[u'spam'], resource_id=res['resource_id'])['records'], - [{u'spam': u'spam spam BEANS spam'}, {u'spam': u'spam spam SPAM spam'}]) + [ + {u'spam': u'spam spam BEANS spam'}, + {u'spam': u'spam spam SPAM spam'}]) + + def test_create_trigger_exception(self): + ds = factories.Dataset() + + helpers.call_action( + u'datastore_function_create', + name=u'spamexception_trigger', + rettype=u'trigger', + definition=u''' + BEGIN + IF NEW.spam != 'spam' THEN + RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam; + END IF; + RETURN NEW; + END;''') + try: + res = helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + records=[{u'spam': u'spam'}, {u'spam': u'EGGS'}], + triggers=[{u'function': u'spamexception_trigger'}]) + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'records':[ + u'"EGGS"? Yeeeeccch!']}) + else: + assert 0, u'no validation error' + + def test_upsert_trigger_exception(self): + ds = factories.Dataset() + + helpers.call_action( + u'datastore_function_create', + name=u'spamonly_trigger', + rettype=u'trigger', + definition=u''' + BEGIN + IF NEW.spam != 'spam' THEN + RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam; + END IF; + RETURN NEW; + END;''') + res = helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + triggers=[{u'function': u'spamonly_trigger'}]) + try: + helpers.call_action( + u'datastore_upsert', + method=u'insert', + resource_id=res['resource_id'], + records=[{u'spam': u'spam'}, {u'spam': u'BEANS'}]) + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'records':[ + u'"BEANS"? Yeeeeccch!']}) + else: + assert 0, u'no validation error' From e0d3f056457730343e3d628885ba764bdd109913 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 28 Feb 2017 11:40:06 -0500 Subject: [PATCH 13/54] [#3428] fix datastore_function_delete --- ckanext/datastore/db.py | 8 ++++-- ckanext/datastore/logic/action.py | 2 +- ckanext/datastore/logic/schema.py | 1 + ckanext/datastore/tests/test_delete.py | 36 +++++++++++++++++++++++++- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 2b3b05a49d5..31e893ebf37 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1450,12 +1450,16 @@ def create_trigger_function(name, definition, or_replace): def drop_function(name, if_exists): sql = u''' - DROP FUNCTION {if_exists} {name}; + DROP FUNCTION {if_exists} {name}(); '''.format( if_exists=u'IF EXISTS' if if_exists else u'', name=datastore_helpers.identifier(name)) - _write_engine_execute(sql) + try: + _write_engine_execute(sql) + except ProgrammingError as pe: + message = pe.args[0].split('\n')[0].decode('utf8') + raise ValidationError({u'name': [message.split(u') ', 1)[-1]]}) def _write_engine_execute(sql): diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 6b0c9c26272..cd2ef3d3dd9 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -645,4 +645,4 @@ def datastore_function_delete(context, data_dict): ''' p.toolkit.check_access('datastore_function_delete', context, data_dict) - db.drop_function(data_dict['name']) + db.drop_function(data_dict['name'], data_dict['if_exists']) diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 9ee317fcb0b..d747d3bd293 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -181,4 +181,5 @@ def datastore_function_create_schema(): def datastore_function_delete_schema(): return { 'name': [unicode_only, not_empty], + 'if_exists': [default(False), boolean_validator], } diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py index ace6d4f5c38..0fc9dd6ada2 100644 --- a/ckanext/datastore/tests/test_delete.py +++ b/ckanext/datastore/tests/test_delete.py @@ -2,6 +2,7 @@ import json import nose +from nose.tools import assert_equal import sqlalchemy import sqlalchemy.orm as orm @@ -10,10 +11,13 @@ import ckan.lib.create_test_data as ctd import ckan.model as model import ckan.tests.legacy as tests +from ckan.tests import helpers from ckan.common import config +from ckan.plugins.toolkit import ValidationError import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type +from ckanext.datastore.tests.helpers import ( + rebuild_all_dbs, set_url_type, DatastoreFunctionalTestBase) class TestDatastoreDelete(tests.WsgiAppCase): @@ -236,3 +240,33 @@ def test_delete_with_blank_filters(self): assert(len(results['result']['records']) == 0) self._delete() + + +class TestDatastoreFunctionDelete(DatastoreFunctionalTestBase): + def test_create_delete(self): + helpers.call_action( + u'datastore_function_create', + name=u'test_nop', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + helpers.call_action( + u'datastore_function_delete', + name=u'test_nop') + + def test_delete_nonexistant(self): + try: + helpers.call_action( + u'datastore_function_delete', + name=u'test_not_there') + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'name': [u'function test_not_there() does not exist']}) + else: + assert 0, u'no validation error' + + def test_delete_if_exitst(self): + helpers.call_action( + u'datastore_function_delete', + name=u'test_not_there_either', + if_exists=True) From 99acbff9ccecfeedc07e0d0e14e89914c2cbcb79 Mon Sep 17 00:00:00 2001 From: Yan Date: Tue, 7 Mar 2017 16:56:34 +0200 Subject: [PATCH 14/54] [#3471] Fix for small issues and Create dataset page and Dashboard tabs --- ckan/public/base/less/forms.less | 2 +- ckan/public/base/less/media.less | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 2800dde3e98..a024a6f51fb 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -170,7 +170,7 @@ textarea { @media (min-width: 980px) { .form-horizontal .info-block { - padding: 6px 0 6px 25px; + padding: 0 0 6px 25px; } .form-horizontal .info-inline { float: right; diff --git a/ckan/public/base/less/media.less b/ckan/public/base/less/media.less index 900af8dd028..c4b9abbefb0 100644 --- a/ckan/public/base/less/media.less +++ b/ckan/public/base/less/media.less @@ -28,6 +28,11 @@ .media-grid { margin-left:-27px; } + .module-content { + .wide .media-grid { + margin-left:-25px; + } + } } .media-item { From 8b015ad921d3a24ea60f31256360ff1c0bef1fe6 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 28 Mar 2017 11:28:34 +0000 Subject: [PATCH 15/54] [3513] setuptools version checked in setup.py --- doc/maintaining/installing/install-from-source.rst | 8 +++++++- setup.py | 9 +++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index c022946a9e7..101bbb95f35 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -126,7 +126,13 @@ c. Install the Python modules that CKAN requires into your virtualenv: pip install -r |virtualenv|/src/ckan/requirements.txt -d. Deactivate and reactivate your virtualenv, to make sure you're using the +d. Upgrade 'setuptools': + + .. parsed-literal:: + + pip install -U setuptools + +e. Deactivate and reactivate your virtualenv, to make sure you're using the virtualenv's copies of commands like ``paster`` rather than any system-wide installed copies: diff --git a/setup.py b/setup.py index 90cce8bfd82..122b8a9dd0a 100644 --- a/setup.py +++ b/setup.py @@ -6,12 +6,17 @@ del os.link try: - from setuptools import setup, find_packages + from setuptools import setup, find_packages, __version__ except ImportError: from ez_setup import use_setuptools use_setuptools() - from setuptools import setup, find_packages + from setuptools import setup, find_packages, __version__ +assert __version__ >= '18.5', 'setuptools version error'\ + '\nYou need a newer version of setuptools. Do this:\n' \ + ' pip install -U setuptools\n' \ + 'before installing ckan into your python environment again.' + from ckan import (__version__, __description__, __long_description__, __license__) From ae2588fd1e0ab33717334833d54f6ab147385206 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 28 Mar 2017 14:11:05 +0000 Subject: [PATCH 16/54] [3513] Fix version expression to cope with e.g. 2.2. Clearer text. --- setup.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 122b8a9dd0a..a48cec6a3c2 100644 --- a/setup.py +++ b/setup.py @@ -6,16 +6,23 @@ del os.link try: - from setuptools import setup, find_packages, __version__ + from setuptools import (setup, find_packages, + __version__ as setuptools_version) except ImportError: from ez_setup import use_setuptools use_setuptools() - from setuptools import setup, find_packages, __version__ + from setuptools import (setup, find_packages, + __version__ as setuptools_version) -assert __version__ >= '18.5', 'setuptools version error'\ - '\nYou need a newer version of setuptools. Do this:\n' \ - ' pip install -U setuptools\n' \ - 'before installing ckan into your python environment again.' +MIN_SETUPTOOLS_VERSION = 18.5 +assert setuptools_version >= str(MIN_SETUPTOOLS_VERSION) and \ + int(setuptools_version.split('.')[0]) >= int(MIN_SETUPTOOLS_VERSION),\ + ('setuptools version error' + '\nYou need a newer version of setuptools.\n' + 'You have %s and you need at least %s.\nDo this:\n' + ' pip install -U setuptools\n' \ + 'and then try again to install ckan into your python environment.' % + (setuptools_version, MIN_SETUPTOOLS_VERSION)) from ckan import (__version__, __description__, __long_description__, __license__) From a83c314dcf831c530a9c98f40b4ff46a62838491 Mon Sep 17 00:00:00 2001 From: David Read Date: Wed, 29 Mar 2017 12:21:04 +0000 Subject: [PATCH 17/54] [3513] Specify particular version of setuptools, rather than just the latest. Moved import upwards. --- setup.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index a48cec6a3c2..50421d81097 100644 --- a/setup.py +++ b/setup.py @@ -14,18 +14,22 @@ from setuptools import (setup, find_packages, __version__ as setuptools_version) +from ckan import (__version__, __description__, __long_description__, + __license__) + MIN_SETUPTOOLS_VERSION = 18.5 +SUGGESTED_SETUPTOOLS_VERSION = 18.5 assert setuptools_version >= str(MIN_SETUPTOOLS_VERSION) and \ int(setuptools_version.split('.')[0]) >= int(MIN_SETUPTOOLS_VERSION),\ ('setuptools version error' '\nYou need a newer version of setuptools.\n' - 'You have %s and you need at least %s.\nDo this:\n' - ' pip install -U setuptools\n' \ - 'and then try again to install ckan into your python environment.' % - (setuptools_version, MIN_SETUPTOOLS_VERSION)) - -from ckan import (__version__, __description__, __long_description__, - __license__) + 'You have {current}, you need at least {minimum} and the suggested ' + 'version is {suggested}.\nDo this:\n' + ' pip install setuptools=={suggested}\n' + 'and then try again to install ckan into your python environment.'.format( + current=setuptools_version, minimum=MIN_SETUPTOOLS_VERSION, + suggested=SUGGESTED_SETUPTOOLS_VERSION)) + entry_points = { 'nose.plugins.0.10': [ From b5071c0bb854cdfa6af06c3ba12c8c0ab964f165 Mon Sep 17 00:00:00 2001 From: David Read Date: Wed, 29 Mar 2017 13:36:11 +0000 Subject: [PATCH 18/54] [3513] Recommended version is now 20.4 and in its own requirements file. --- doc/maintaining/installing/install-from-source.rst | 4 ++-- requirement-setuptools.txt | 1 + setup.py | 14 +++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 requirement-setuptools.txt diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index 101bbb95f35..282cade2acf 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -126,11 +126,11 @@ c. Install the Python modules that CKAN requires into your virtualenv: pip install -r |virtualenv|/src/ckan/requirements.txt -d. Upgrade 'setuptools': +d. Install the recommended version of 'setuptools': .. parsed-literal:: - pip install -U setuptools + pip install -r |virtualenv|/src/ckan/requirement-setuptools.txt e. Deactivate and reactivate your virtualenv, to make sure you're using the virtualenv's copies of commands like ``paster`` rather than any system-wide diff --git a/requirement-setuptools.txt b/requirement-setuptools.txt new file mode 100644 index 00000000000..44e41169c85 --- /dev/null +++ b/requirement-setuptools.txt @@ -0,0 +1 @@ +setuptools==20.4 \ No newline at end of file diff --git a/setup.py b/setup.py index 50421d81097..c1d44b1991f 100644 --- a/setup.py +++ b/setup.py @@ -17,18 +17,18 @@ from ckan import (__version__, __description__, __long_description__, __license__) -MIN_SETUPTOOLS_VERSION = 18.5 -SUGGESTED_SETUPTOOLS_VERSION = 18.5 +MIN_SETUPTOOLS_VERSION = 20.4 assert setuptools_version >= str(MIN_SETUPTOOLS_VERSION) and \ int(setuptools_version.split('.')[0]) >= int(MIN_SETUPTOOLS_VERSION),\ ('setuptools version error' '\nYou need a newer version of setuptools.\n' - 'You have {current}, you need at least {minimum} and the suggested ' - 'version is {suggested}.\nDo this:\n' - ' pip install setuptools=={suggested}\n' + 'You have {current}, you need at least {minimum}' + '\nInstall the recommended version:\n' + ' pip install -r requirement-setuptools.txt\n' 'and then try again to install ckan into your python environment.'.format( - current=setuptools_version, minimum=MIN_SETUPTOOLS_VERSION, - suggested=SUGGESTED_SETUPTOOLS_VERSION)) + current=setuptools_version, + minimum=MIN_SETUPTOOLS_VERSION + )) entry_points = { From f46a01e356e93a395c0d020e1624c2b3edbabdf2 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 30 Mar 2017 10:46:06 +0100 Subject: [PATCH 19/54] [3513] Move setuptools upgrade forward, else ckan will not install. --- .../installing/install-from-source.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index 282cade2acf..0db47166280 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -94,7 +94,13 @@ a. Create a Python `virtual environment `_ |activate| -b. Install the CKAN source code into your virtualenv. +b. Install the recommended version of 'setuptools': + + .. parsed-literal:: + + pip install -r |virtualenv|/src/ckan/requirement-setuptools.txt + +c. Install the CKAN source code into your virtualenv. To install the latest stable release of CKAN (CKAN |latest_release_version|), run: @@ -116,7 +122,7 @@ b. Install the CKAN source code into your virtualenv. production websites! Only install this version if you're doing CKAN development. -c. Install the Python modules that CKAN requires into your virtualenv: +d. Install the Python modules that CKAN requires into your virtualenv: .. versionchanged:: 2.1 In CKAN 2.0 and earlier the requirement file was called @@ -126,12 +132,6 @@ c. Install the Python modules that CKAN requires into your virtualenv: pip install -r |virtualenv|/src/ckan/requirements.txt -d. Install the recommended version of 'setuptools': - - .. parsed-literal:: - - pip install -r |virtualenv|/src/ckan/requirement-setuptools.txt - e. Deactivate and reactivate your virtualenv, to make sure you're using the virtualenv's copies of commands like ``paster`` rather than any system-wide installed copies: From d163c22c3871539f491281be8abb1e44b134939c Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 31 Mar 2017 16:06:13 +0100 Subject: [PATCH 20/54] [3513] Sort autobuild hopefully. --- bin/travis-install-dependencies | 1 + circle.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies index 8b9b1430d8f..77514786c2e 100755 --- a/bin/travis-install-dependencies +++ b/bin/travis-install-dependencies @@ -29,6 +29,7 @@ sudo -E -u postgres ./bin/postgres_init/1_create_ckan_db.sh sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh export PIP_USE_MIRRORS=true +pip install -r requirement-setuptools.txt --allow-all-external pip install -r requirements.txt --allow-all-external pip install -r dev-requirements.txt --allow-all-external diff --git a/circle.yml b/circle.yml index d9fff638dea..e560a8057a0 100644 --- a/circle.yml +++ b/circle.yml @@ -23,6 +23,7 @@ dependencies: && chmod +x ~/.local/bin/circleci-matrix" override: + - pip install -r requirement-setuptools.txt - pip install -r requirements.txt - pip install -r dev-requirements.txt - python setup.py develop From 3348a32c3f7c6f95e3da92a71bb3750f87d81a8c Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 6 Feb 2017 17:36:51 -0500 Subject: [PATCH 21/54] [#3428] replace db._get_engine with db.get_(read|write)_engine --- ckanext/datastore/db.py | 35 +++++++++++------------ ckanext/datastore/logic/action.py | 37 +++++++------------------ ckanext/datastore/plugin.py | 13 +++------ ckanext/datastore/tests/test_create.py | 6 ++-- ckanext/datastore/tests/test_delete.py | 3 +- ckanext/datastore/tests/test_dump.py | 3 +- ckanext/datastore/tests/test_helpers.py | 4 +-- ckanext/datastore/tests/test_search.py | 10 ++----- ckanext/datastore/tests/test_unit.py | 3 +- ckanext/datastore/tests/test_upsert.py | 9 ++---- 10 files changed, 44 insertions(+), 79 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index ed01ce95bfd..5c19d9713b1 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -98,9 +98,16 @@ def _is_valid_table_name(name): return _is_valid_field_name(name) -def _get_engine(data_dict): +def get_read_engine(): + return _get_engine_from_url(config['ckan.datastore.read_url']) + + +def get_write_engine(): + return _get_engine_from_url(config['ckan.datastore.write_url']) + + +def _get_engine_from_url(connection_url): '''Get either read or write engine.''' - connection_url = data_dict['connection_url'] engine = _engines.get(connection_url) if not engine: @@ -127,9 +134,7 @@ def _cache_types(context): log.info("Create nested type. Native JSON: {0!r}".format( native_json)) - data_dict = { - 'connection_url': config['ckan.datastore.write_url']} - engine = _get_engine(data_dict) + engine = get_write_engine() with engine.begin() as connection: connection.execute( 'CREATE TYPE "nested" AS (json {0}, extra text)'.format( @@ -967,7 +972,6 @@ def validate(context, data_dict): fields_types) # Remove default elements in data_dict - del data_dict_copy['connection_url'] del data_dict_copy['resource_id'] data_dict_copy.pop('id', None) data_dict_copy.pop('include_total', None) @@ -1116,7 +1120,7 @@ def create(context, data_dict): :raises InvalidDataError: if there is an invalid value in the given data ''' - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) _cache_types(context) @@ -1181,7 +1185,7 @@ def upsert(context, data_dict): Any error results in total failure! For now pass back the actual error. Should be transactional. ''' - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) @@ -1224,7 +1228,7 @@ def upsert(context, data_dict): def delete(context, data_dict): - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() _cache_types(context) @@ -1248,7 +1252,7 @@ def delete(context, data_dict): def search(context, data_dict): - engine = _get_engine(data_dict) + engine = get_read_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) _cache_types(context) @@ -1275,7 +1279,7 @@ def search(context, data_dict): def search_sql(context, data_dict): - engine = _get_engine(data_dict) + engine = get_read_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) _cache_types(context) @@ -1367,7 +1371,7 @@ def _change_privilege(context, data_dict, what): def make_private(context, data_dict): log.info('Making resource {resource_id!r} private'.format(**data_dict)) - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() trans = context['connection'].begin() try: @@ -1379,7 +1383,7 @@ def make_private(context, data_dict): def make_public(context, data_dict): log.info('Making resource {resource_id!r} public'.format(**data_dict)) - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() trans = context['connection'].begin() try: @@ -1392,10 +1396,7 @@ def make_public(context, data_dict): def get_all_resources_ids_in_datastore(): read_url = config.get('ckan.datastore.read_url') write_url = config.get('ckan.datastore.write_url') - data_dict = { - 'connection_url': read_url or write_url - } resources_sql = sqlalchemy.text(u'''SELECT name FROM "_table_metadata" WHERE alias_of IS NULL''') - query = _get_engine(data_dict).execute(resources_sql) + query = get_read_engine().execute(resources_sql) return [q[0] for q in query.fetchall()] diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 27e74599668..ff8b07eb47f 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -122,8 +122,6 @@ def datastore_create(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = config['ckan.datastore.write_url'] - # validate aliases aliases = datastore_helpers.get_list(data_dict.get('aliases', [])) for alias in aliases: @@ -153,7 +151,6 @@ def datastore_create(context, data_dict): result.pop('id', None) result.pop('private', None) - result.pop('connection_url') return result @@ -207,12 +204,10 @@ def datastore_upsert(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = config['ckan.datastore.write_url'] - res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + results = db.get_write_engine().execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: @@ -222,7 +217,6 @@ def datastore_upsert(context, data_dict): result = db.upsert(context, data_dict) result.pop('id', None) - result.pop('connection_url') return result @@ -249,11 +243,9 @@ def _type_lookup(t): resource_id = _get_or_bust(data_dict, 'id') resource = p.toolkit.get_action('resource_show')(context, {'id':resource_id}) - data_dict['connection_url'] = config['ckan.datastore.read_url'] - resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=resource_id) + results = db.get_read_engine().execute(resources_sql, id=resource_id) res_exists = results.rowcount > 0 if not res_exists: raise p.toolkit.ObjectNotFound(p.toolkit._( @@ -269,7 +261,7 @@ def _type_lookup(t): SELECT column_name, data_type FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = :resource_id; ''') - schema_results = db._get_engine(data_dict).execute(schema_sql, resource_id=resource_id) + schema_results = db.get_read_engine().execute(schema_sql, resource_id=resource_id) for row in schema_results.fetchall(): k = row[0] v = row[1] @@ -282,7 +274,7 @@ def _type_lookup(t): meta_sql = sqlalchemy.text(u''' SELECT count(_id) FROM "{0}"; '''.format(resource_id)) - meta_results = db._get_engine(data_dict).execute(meta_sql, resource_id=resource_id) + meta_results = db.get_read_engine().execute(meta_sql, resource_id=resource_id) info['meta']['count'] = meta_results.fetchone()[0] finally: if schema_results: @@ -334,12 +326,10 @@ def datastore_delete(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = config['ckan.datastore.write_url'] - res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + results = db.get_read_engine().execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: @@ -361,7 +351,6 @@ def datastore_delete(context, data_dict): set_datastore_active_flag(model, data_dict, False) result.pop('id', None) - result.pop('connection_url') return result @@ -431,11 +420,13 @@ def datastore_search(context, data_dict): raise p.toolkit.ValidationError(errors) res_id = data_dict['resource_id'] - data_dict['connection_url'] = config['ckan.datastore.write_url'] resources_sql = sqlalchemy.text(u'''SELECT alias_of FROM "_table_metadata" WHERE name = :id''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + # XXX: write connection because of private tables, we + # should be able to make this read once we stop using pg + # permissions enforcement + results = db.get_write_engine().execute(resources_sql, id=res_id) # Resource only has to exist in the datastore (because it could be an alias) if not results.rowcount > 0: @@ -453,7 +444,6 @@ def datastore_search(context, data_dict): result = db.search(context, data_dict) result.pop('id', None) - result.pop('connection_url') return result @@ -495,11 +485,8 @@ def datastore_search_sql(context, data_dict): p.toolkit.check_access('datastore_search_sql', context, data_dict) - data_dict['connection_url'] = config['ckan.datastore.read_url'] - result = db.search_sql(context, data_dict) result.pop('id', None) - result.pop('connection_url') return result @@ -518,8 +505,6 @@ def datastore_make_private(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - data_dict['connection_url'] = config['ckan.datastore.write_url'] - if not _resource_exists(context, data_dict): raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(res_id) @@ -544,8 +529,6 @@ def datastore_make_public(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - data_dict['connection_url'] = config['ckan.datastore.write_url'] - if not _resource_exists(context, data_dict): raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(res_id) @@ -615,7 +598,7 @@ def _resource_exists(context, data_dict): resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + results = db.get_read_engine().execute(resources_sql, id=res_id) return results.rowcount > 0 diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index dc35bd96f28..f0ba5853aa5 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -34,9 +34,7 @@ def _is_legacy_mode(config): Returns True if `ckan.datastore.read_url` is not set in the provided config object or CKAN is running on Postgres < 9.x ''' - write_url = config.get('ckan.datastore.write_url') - - engine = db._get_engine({'connection_url': write_url}) + engine = db.get_write_engine() connection = engine.connect() return (not config.get('ckan.datastore.read_url') or @@ -112,8 +110,7 @@ def configure(self, config): else: self.read_url = self.config['ckan.datastore.read_url'] - self.read_engine = db._get_engine( - {'connection_url': self.read_url}) + self.read_engine = db.get_read_engine() if not model.engine_is_pg(self.read_engine): log.warn('We detected that you do not use a PostgreSQL ' 'database. The DataStore will NOT work and DataStore ' @@ -140,7 +137,6 @@ def notify(self, entity, operation=None): for resource in entity.resources: try: func(context, { - 'connection_url': self.write_url, 'resource_id': resource.id}) except p.toolkit.ObjectNotFound: pass @@ -174,7 +170,7 @@ def _is_read_only_database(self): ''' Returns True if no connection has CREATE privileges on the public schema. This is the case if replication is enabled.''' for url in [self.ckan_url, self.write_url, self.read_url]: - connection = db._get_engine({'connection_url': url}).connect() + connection = db._get_engine_from_url(url).connect() try: sql = u"SELECT has_schema_privilege('public', 'CREATE')" is_writable = connection.execute(sql).first()[0] @@ -200,8 +196,7 @@ def _read_connection_has_correct_privileges(self): only user. A table is created by the write user to test the read only user. ''' - write_connection = db._get_engine( - {'connection_url': self.write_url}).connect() + write_connection = db.get_write_engine().connect() read_connection_user = sa_url.make_url(self.read_url).username drop_foo_sql = u'DROP TABLE IF EXISTS _foo' diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index a95bcf68ef8..22d0e5c71e1 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -193,8 +193,7 @@ def _get_index_names(self, resource_id): return [result[0] for result in results] def _execute_sql(self, sql, *args): - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() session = orm.scoped_session(orm.sessionmaker(bind=engine)) return session.connection().execute(sql, *args) @@ -255,8 +254,7 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py index f5c14fd4944..56fa93e5730 100644 --- a/ckanext/datastore/tests/test_delete.py +++ b/ckanext/datastore/tests/test_delete.py @@ -47,8 +47,7 @@ def setup_class(cls): 'rating with %': '42%'}] } - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index c460db873de..5db9ac33fd5 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -90,8 +90,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine({ - 'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod diff --git a/ckanext/datastore/tests/test_helpers.py b/ckanext/datastore/tests/test_helpers.py index 9cc05c3dded..eed67000c20 100644 --- a/ckanext/datastore/tests/test_helpers.py +++ b/ckanext/datastore/tests/test_helpers.py @@ -70,9 +70,7 @@ def setup_class(cls): if not config.get('ckan.datastore.read_url'): raise nose.SkipTest('Datastore runs on legacy mode, skipping...') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']} - ) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) datastore_test_helpers.clear_db(cls.Session) diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py index c472b52bdc3..f1b0fe64c48 100644 --- a/ckanext/datastore/tests/test_search.py +++ b/ckanext/datastore/tests/test_search.py @@ -165,9 +165,7 @@ def setup_class(cls): u'characters': None, u'rating with %': u'99%'}] - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']} - ) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -838,8 +836,7 @@ def setup_class(cls): u'published': None}] cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}] - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -941,8 +938,7 @@ def test_read_private(self): 'user': self.sysadmin_user.name, 'model': model} data_dict = { - 'resource_id': self.data['resource_id'], - 'connection_url': config['ckan.datastore.write_url']} + 'resource_id': self.data['resource_id']} p.toolkit.get_action('datastore_make_private')(context, data_dict) query = 'SELECT * FROM "{0}"'.format(self.data['resource_id']) data = {'sql': query} diff --git a/ckanext/datastore/tests/test_unit.py b/ckanext/datastore/tests/test_unit.py index c02d30c60d6..2a74e3386cd 100644 --- a/ckanext/datastore/tests/test_unit.py +++ b/ckanext/datastore/tests/test_unit.py @@ -35,8 +35,7 @@ def test_is_valid_table_name(self): def test_pg_version_check(self): if not tests.is_datastore_supported(): raise nose.SkipTest("Datastore not supported") - engine = db._get_engine( - {'connection_url': config['sqlalchemy.url']}) + engine = db._get_engine_from_url(config['sqlalchemy.url']) connection = engine.connect() assert db._pg_version_is_at_least(connection, '8.0') assert not db._pg_version_is_at_least(connection, '10.0') diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py index b7a09bf7dae..e798bec38dc 100644 --- a/ckanext/datastore/tests/test_upsert.py +++ b/ckanext/datastore/tests/test_upsert.py @@ -113,8 +113,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -365,8 +364,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -472,8 +470,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod From ce33d8ef481ae30e0950807246f1930eb13eaa6b Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 7 Feb 2017 11:47:24 -0500 Subject: [PATCH 22/54] [#3428] one more db._get_engine replacement --- ckanext/datapusher/tests/test_interfaces.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ckanext/datapusher/tests/test_interfaces.py b/ckanext/datapusher/tests/test_interfaces.py index 49b8efd7fad..8439dbff45c 100644 --- a/ckanext/datapusher/tests/test_interfaces.py +++ b/ckanext/datapusher/tests/test_interfaces.py @@ -61,8 +61,7 @@ def setup_class(cls): cls.sysadmin_user = factories.User(name='testsysadmin', sysadmin=True) cls.normal_user = factories.User(name='annafan') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod From 5fa93799e74c866263907a9efda33d83140c0b77 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 7 Feb 2017 12:36:27 -0500 Subject: [PATCH 23/54] [#3428] one more db._get_engine replacement --- ckanext/datapusher/tests/test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index d603ca386ab..cdf4024374d 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -68,8 +68,7 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) From 856ae968ae870129a43d679bab88c4a0d3878095 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Fri, 7 Apr 2017 14:14:52 -0400 Subject: [PATCH 24/54] [#3523] use postgresql row_to_json + LazyJSONObject --- ckan/lib/lazyjson.py | 55 ++++++++++++++++++++++++++ ckanext/datastore/db.py | 40 +++++++++++++------ ckanext/datastore/plugin.py | 11 +++++- ckanext/datastore/tests/test_dump.py | 2 +- ckanext/datastore/tests/test_search.py | 8 ++-- requirements.in | 1 + requirements.txt | 2 +- 7 files changed, 98 insertions(+), 21 deletions(-) create mode 100644 ckan/lib/lazyjson.py diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py new file mode 100644 index 00000000000..d3c0934cc78 --- /dev/null +++ b/ckan/lib/lazyjson.py @@ -0,0 +1,55 @@ +# encoding: utf-8 + + +from simplejson import loads, RawJSON, dumps + + +class LazyJSONObject(RawJSON): + u''' + An object that behaves like a dict returned from json.loads + but when passed to simplejson.dumps will render original + string passed when possible. Accepts and produces only + unicode strings containing a single JSON object. + ''' + def __init__(self, json_string): + assert isinstance(json_string, unicode) + self._json_string = json_string + self._json_dict = None + + def _loads(self): + if not self._json_dict: + self._json_dict = loads(self._json_string) + self._json_string = None + return self._json_dict + + def __nonzero__(self): + return True + + def __repr__(self): + if self._json_string: + return u'' % self._json_string + return u'' % self._json_dict + + @property + def encoded_json(self): + if self._json_string: + return self._json_string + return dumps( + self._json_dict, + ensure_ascii=False, + separators=(u',', u':')) + + +def _loads_method(name): + def method(self, *args, **kwargs): + return getattr(self._loads(), name)(*args, **kwargs) + return method + + +for fn in [u'__contains__', u'__delitem__', u'__eq__', u'__ge__', + u'__getitem__', u'__gt__', u'__iter__', u'__le__', u'__len__', + u'__lt__', u'__ne__', u'__setitem__', u'clear', u'copy', + u'fromkeys', u'get', u'has_key', u'items', u'iteritems', + u'iterkeys', u'itervalues', u'keys', u'pop', u'popitem', + u'setdefault', u'update', u'values']: + setattr(LazyJSONObject, fn, _loads_method(fn)) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 5c19d9713b1..1cd165f619e 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -15,6 +15,8 @@ import ckan.lib.cli as cli import ckan.plugins as p import ckan.plugins.toolkit as toolkit +from ckan.lib.lazyjson import LazyJSONObject + import ckanext.datastore.helpers as datastore_helpers import ckanext.datastore.interfaces as interfaces import psycopg2.extras @@ -1029,9 +1031,12 @@ def search_data(context, data_dict): else: sort_clause = '' - sql_string = u'''SELECT {distinct} {select} - FROM "{resource}" {ts_query} - {where} {sort} LIMIT {limit} OFFSET {offset}'''.format( + sql_string = u''' + SELECT row_to_json(j)::text from ( + SELECT {distinct} {select} + FROM "{resource}" {ts_query} + {where} {sort} LIMIT {limit} OFFSET {offset} + ) AS j'''.format( distinct=distinct, select=select_columns, resource=resource_id, @@ -1041,11 +1046,23 @@ def search_data(context, data_dict): limit=limit, offset=offset) - results = _execute_single_statement(context, sql_string, where_values) + records = [ + LazyJSONObject(r[0]) for r in + _execute_single_statement(context, sql_string, where_values)] + data_dict['records'] = records + + field_info = _get_field_info( + context['connection'], data_dict['resource_id']) + result_fields = [] + for field_id, field_type in fields_types.iteritems(): + f = {u'id': field_id, u'type': field_type} + if field_id in field_info: + f['info'] = field_info[f['id']] + result_fields.append(f) + data_dict['fields'] = result_fields + _unrename_json_field(data_dict) _insert_links(data_dict, limit, offset) - r = format_results(context, results, data_dict, _get_field_info( - context['connection'], data_dict['resource_id'])) if data_dict.get('include_total', True): count_sql_string = u'''SELECT {distinct} count(*) @@ -1058,7 +1075,7 @@ def search_data(context, data_dict): context, count_sql_string, where_values) data_dict['total'] = count_result.fetchall()[0][0] - return r + return data_dict def _execute_single_statement(context, sql_string, where_values): @@ -1072,16 +1089,13 @@ def _execute_single_statement(context, sql_string, where_values): return results -def format_results(context, results, data_dict, field_info=None): +def format_results(context, results, data_dict): result_fields = [] for field in results.cursor.description: - f = { + result_fields.append({ 'id': field[0].decode('utf-8'), 'type': _get_type(context, field[1]) - } - if field_info and f['id'] in field_info: - f['info'] = field_info[f['id']] - result_fields.append(f) + }) records = [] for row in results: diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index f0ba5853aa5..3f1f7dd2c0d 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -383,8 +383,15 @@ def datastore_search(self, context, data_dict, fields_types, query_dict): sort = self._sort(data_dict, fields_types) where = self._where(data_dict, fields_types) - select_cols = [ - datastore_helpers.identifier(field_id) for field_id in field_ids] + select_cols = [] + for field_id in field_ids: + fmt = u'{0}' + if fields_types.get(field_id) == u'nested': + fmt = u'({0}).json as {0}' + elif fields_types.get(field_id) == u'timestamp': + fmt = u"to_char({0}, 'YYYY-MM-DD\"T\"HH24:MI:SS') as {0}" + select_cols.append(fmt.format( + datastore_helpers.identifier(field_id))) if rank_column: select_cols.append(rank_column) diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 5db9ac33fd5..ec49fbdb068 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -153,7 +153,7 @@ def test_dump_json(self): str(self.data['resource_id'])), extra_environ=auth) content = res.body.decode('utf-8') expected_content = ( - u'{\n "fields": [{"type":"int4","id":"_id"},{"type":"text",' + u'{\n "fields": [{"type":"int","id":"_id"},{"type":"text",' u'"id":"b\xfck"},{"type":"text","id":"author"},{"type":"timestamp"' u',"id":"published"},{"type":"_text","id":"characters"},' u'{"type":"_text","id":"random_letters"},{"type":"json",' diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py index f1b0fe64c48..46eac61a288 100644 --- a/ckanext/datastore/tests/test_search.py +++ b/ckanext/datastore/tests/test_search.py @@ -20,6 +20,7 @@ assert_equals = nose.tools.assert_equals assert_raises = nose.tools.assert_raises +assert_in = nose.tools.assert_in class TestDatastoreSearchNewTest(object): @@ -510,14 +511,13 @@ def test_search_full_text(self): ) for record in result['records']] assert results == self.expected_records, result['records'] - expected_fields = [{u'type': u'int4', u'id': u'_id'}, + expected_fields = [{u'type': u'int', u'id': u'_id'}, {u'type': u'text', u'id': u'b\xfck'}, {u'type': u'text', u'id': u'author'}, {u'type': u'timestamp', u'id': u'published'}, - {u'type': u'json', u'id': u'nested'}, - {u'type': u'float4', u'id': u'rank'}] + {u'type': u'json', u'id': u'nested'}] for field in expected_fields: - assert field in result['fields'], field + assert_in(field, result['fields']) # test multiple word queries (connected with and) data = {'resource_id': self.data['resource_id'], diff --git a/requirements.in b/requirements.in index 547b7156473..9acc369a4fa 100644 --- a/requirements.in +++ b/requirements.in @@ -26,6 +26,7 @@ repoze.who==2.3 requests==2.11.1 Routes==1.13 rq==0.6.0 +simplejson==3.10.0 sqlalchemy-migrate==0.10.0 SQLAlchemy==0.9.6 sqlparse==0.2.2 diff --git a/requirements.txt b/requirements.txt index 1995936865b..8851fa81c4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ repoze.who==2.3 requests==2.11.1 Routes==1.13 # via pylons rq==0.6.0 -simplejson==3.8.2 # via pylons +simplejson==3.10.0 six==1.10.0 # via bleach, html5lib, pastescript, pyutilib.component.core, sqlalchemy-migrate sqlalchemy-migrate==0.10.0 SQLAlchemy==0.9.6 # via sqlalchemy-migrate From a553c1d380ea08419600c7feff4ed12d889d6b8c Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Fri, 7 Apr 2017 20:58:20 -0400 Subject: [PATCH 25/54] [#3523] datastore_search records_format param, update dump --- ckanext/datastore/controller.py | 36 ++++++--- ckanext/datastore/db.py | 55 +++++++++++-- ckanext/datastore/logic/action.py | 12 ++- ckanext/datastore/logic/schema.py | 3 + ckanext/datastore/plugin.py | 19 +++-- ckanext/datastore/tests/test_dump.py | 18 ++--- ckanext/datastore/writer.py | 112 ++++++++++----------------- 7 files changed, 155 insertions(+), 100 deletions(-) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 09f91d160ab..d6c49b14efd 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -100,16 +100,22 @@ def dictionary(self, id, resource_id): def dump_to(resource_id, output, fmt, offset, limit, options): + if fmt == 'csv': + writer_factory = csv_writer + records_format = 'csv' + elif fmt == 'tsv': + writer_factory = tsv_writer + records_format = 'tsv' + elif fmt == 'json': + writer_factory = json_writer + records_format = 'lists' + elif fmt == 'xml': + writer_factory = xml_writer + records_format = 'objects' + def start_writer(fields): bom = options.get(u'bom', False) - if fmt == 'csv': - return csv_writer(output, fields, resource_id, bom) - if fmt == 'tsv': - return tsv_writer(output, fields, resource_id, bom) - if fmt == 'json': - return json_writer(output, fields, resource_id, bom) - if fmt == 'xml': - return xml_writer(output, fields, resource_id, bom) + return writer_factory(output, fields, resource_id, bom) def result_page(offs, lim): return get_action('datastore_search')(None, { @@ -118,6 +124,7 @@ def result_page(offs, lim): PAGINATE_BY if limit is None else min(PAGINATE_BY, lim), 'offset': offs, + 'records_format': records_format, }) result = result_page(offset, limit) @@ -128,13 +135,20 @@ def result_page(offs, lim): if limit is not None and limit <= 0: break - for record in result['records']: - wr.writerow([record[column] for column in columns]) + records = result['records'] + + wr.write_records(records) - if len(result['records']) < PAGINATE_BY: + if records_format == 'objects' or records_format == 'lists': + if len(records) < PAGINATE_BY: + break + elif not records: break + offset += PAGINATE_BY if limit is not None: limit -= PAGINATE_BY + if limit <= 0: + break result = result_page(offset, limit) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 1cd165f619e..5f19932a6a9 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -11,6 +11,7 @@ import urllib import urllib2 import urlparse +from cStringIO import StringIO import ckan.lib.cli as cli import ckan.plugins as p @@ -977,6 +978,7 @@ def validate(context, data_dict): del data_dict_copy['resource_id'] data_dict_copy.pop('id', None) data_dict_copy.pop('include_total', None) + data_dict_copy.pop('records_format', None) for key, values in data_dict_copy.iteritems(): if not values: @@ -1031,12 +1033,38 @@ def search_data(context, data_dict): else: sort_clause = '' - sql_string = u''' + records_format = data_dict['records_format'] + if records_format == u'objects': + sql_fmt = u''' SELECT row_to_json(j)::text from ( SELECT {distinct} {select} FROM "{resource}" {ts_query} {where} {sort} LIMIT {limit} OFFSET {offset} - ) AS j'''.format( + ) AS j''' + elif records_format == u'lists': + select_columns = u" || ',' || ".join( + s for s in query_dict['select'] + ).replace('%', '%%') + sql_fmt = u''' + SELECT {distinct} '[' || {select} || ']' + FROM "{resource}" {ts_query} + {where} {sort} LIMIT {limit} OFFSET {offset}''' + elif records_format == u'csv': + sql_fmt = u''' + COPY ( + SELECT {distinct} {select} + FROM "{resource}" {ts_query} + {where} {sort} LIMIT {limit} OFFSET {offset} + ) TO STDOUT csv DELIMITER ',' ''' + elif records_format == u'tsv': + sql_fmt = u''' + COPY ( + SELECT {distinct} {select} + FROM "{resource}" {ts_query} + {where} {sort} LIMIT {limit} OFFSET {offset} + ) TO STDOUT csv DELIMITER '\t' ''' + + sql_string = sql_fmt.format( distinct=distinct, select=select_columns, resource=resource_id, @@ -1046,9 +1074,15 @@ def search_data(context, data_dict): limit=limit, offset=offset) - records = [ - LazyJSONObject(r[0]) for r in - _execute_single_statement(context, sql_string, where_values)] + if records_format == u'csv' or records_format == u'tsv': + buf = StringIO() + _execute_single_statement_copy_to( + context, sql_string, where_values, buf) + records = buf.getvalue() + else: + records = [ + LazyJSONObject(r[0]) for r in + _execute_single_statement(context, sql_string, where_values)] data_dict['records'] = records field_info = _get_field_info( @@ -1089,6 +1123,17 @@ def _execute_single_statement(context, sql_string, where_values): return results +def _execute_single_statement_copy_to(context, sql_string, where_values, buf): + if not datastore_helpers.is_single_statement(sql_string): + raise ValidationError({ + 'query': ['Query is not a single statement.'] + }) + + cursor = context['connection'].connection.cursor() + cursor.copy_expert(sql_string, buf) + cursor.close() + + def format_results(context, results, data_dict): result_fields = [] for field in results.cursor.description: diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index ff8b07eb47f..34321954481 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -386,6 +386,16 @@ def datastore_search(context, data_dict): :param sort: comma separated field names with ordering e.g.: "fieldname1, fieldname2 desc" :type sort: string + :param include_total: True to return total matching record count + (optional, default: true) + :type include_total: bool + :param records_format: the format for the records return value: + 'objects' (default) list of {fieldname1: value1, ...} dicts, + 'lists' list of [value1, value2, ...] lists, + 'csv' string containing comma-separated values with no header, + 'tsv' string containing tab-separated values with no header + :type records_format: controlled list + Setting the ``plain`` flag to false enables the entire PostgreSQL `full text search query language`_. @@ -411,7 +421,7 @@ def datastore_search(context, data_dict): :param total: number of total matching records :type total: int :param records: list of matching results - :type records: list of dictionaries + :type records: depends on records_format value passed ''' schema = context.get('schema', dsschema.datastore_search_schema()) diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index d2eecc4caf9..3c0d1f07ff4 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -151,6 +151,9 @@ def datastore_search_schema(): 'sort': [ignore_missing, list_of_strings_or_string], 'distinct': [ignore_missing, boolean_validator], 'include_total': [default(True), boolean_validator], + 'records_format': [ + default(u'objects'), + OneOf([u'objects', u'lists', u'csv', u'tsv'])], '__junk': [empty], '__before': [rename('id', 'resource_id')] } diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 3f1f7dd2c0d..bd8eda23ba8 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -384,12 +384,21 @@ def datastore_search(self, context, data_dict, fields_types, query_dict): where = self._where(data_dict, fields_types) select_cols = [] + records_format = data_dict.get(u'records_format') + json_values = records_format in (u'objects', u'lists') for field_id in field_ids: - fmt = u'{0}' - if fields_types.get(field_id) == u'nested': - fmt = u'({0}).json as {0}' - elif fields_types.get(field_id) == u'timestamp': - fmt = u"to_char({0}, 'YYYY-MM-DD\"T\"HH24:MI:SS') as {0}" + fmt = u'to_json({0})' if records_format == u'lists' else u'{0}' + typ = fields_types.get(field_id) + if typ == u'nested': + fmt = u'({0}).json' + elif typ == u'timestamp': + fmt = u"to_char({0}, 'YYYY-MM-DD\"T\"HH24:MI:SS')" + if json_values: + fmt = u"to_json({0})".format(fmt) + elif typ.startswith(u'_') or typ.endswith(u'[]'): + fmt = u'array_to_json({0})' + if records_format == u'objects': + fmt += u' as {0}' select_cols.append(fmt.format( datastore_helpers.identifier(field_id))) if rank_column: diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index ec49fbdb068..2a354463b0b 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -108,7 +108,7 @@ def test_dump_basic(self): u',characters,random_letters,nested') assert_equals(content[:len(expected)], expected) assert_in('warandpeace', content) - assert_in('"[""Princess Anna"", ""Sergius""]"', content) + assert_in('"[""Princess Anna"",""Sergius""]"', content) # get with alias instead of id res = self.app.get('/datastore/dump/{0}'.format(str( @@ -128,9 +128,9 @@ def test_dump_limit(self): expected_content = ( u'_id,b\xfck,author,published,characters,random_letters,' u'nested\r\n1,annakarenina,tolstoy,2005-03-01T00:00:00,' - u'"[""Princess Anna"", ""Sergius""]",' - u'"[""a"", ""e"", ""x""]","[""b"", ' - u'{""moo"": ""moo""}]"\r\n') + u'"[""Princess Anna"",""Sergius""]",' + u'"[""a"",""e"",""x""]","[""b"", ' + u'{""moo"": ""moo""}]"\n') assert_equals(content, expected_content) def test_dump_tsv(self): @@ -142,9 +142,9 @@ def test_dump_tsv(self): expected_content = ( u'_id\tb\xfck\tauthor\tpublished\tcharacters\trandom_letters\t' u'nested\r\n1\tannakarenina\ttolstoy\t2005-03-01T00:00:00\t' - u'"[""Princess Anna"", ""Sergius""]"\t' - u'"[""a"", ""e"", ""x""]"\t"[""b"", ' - u'{""moo"": ""moo""}]"\r\n') + u'"[""Princess Anna"",""Sergius""]"\t' + u'"[""a"",""e"",""x""]"\t"[""b"", ' + u'{""moo"": ""moo""}]"\n') assert_equals(content, expected_content) def test_dump_json(self): @@ -159,8 +159,8 @@ def test_dump_json(self): u'{"type":"_text","id":"random_letters"},{"type":"json",' u'"id":"nested"}],\n "records": [\n ' u'[1,"annakarenina","tolstoy","2005-03-01T00:00:00",' - u'["Princess Anna","Sergius"],["a","e","x"],["b",' - u'{"moo":"moo"}]]\n]}\n') + u'["Princess Anna","Sergius"],["a","e","x"],["b", ' + u'{"moo": "moo"}]]\n]}\n') assert_equals(content, expected_content) def test_dump_xml(self): diff --git a/ckanext/datastore/writer.py b/ckanext/datastore/writer.py index 5cf75e4efeb..47907978d0a 100644 --- a/ckanext/datastore/writer.py +++ b/ckanext/datastore/writer.py @@ -2,20 +2,12 @@ from contextlib import contextmanager from email.utils import encode_rfc2231 -import json +from simplejson import dumps from xml.etree.cElementTree import Element, SubElement, ElementTree import unicodecsv -UTF8_BOM = u'\uFEFF'.encode(u'utf-8') - - -def _json_dump_nested(value): - is_nested = isinstance(value, (list, dict)) - - if is_nested: - return json.dumps(value) - return value +from codecs import BOM_UTF8 @contextmanager @@ -27,10 +19,6 @@ def csv_writer(response, fields, name=None, bom=False): :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file - - >>> with csv_writer(response, fields) as d: - >>> d.writerow(row1) - >>> d.writerow(row2) ''' if hasattr(response, u'headers'): @@ -39,11 +27,12 @@ def csv_writer(response, fields, name=None, bom=False): response.headers['Content-disposition'] = ( b'attachment; filename="{name}.csv"'.format( name=encode_rfc2231(name))) - wr = CSVWriter(response, fields, encoding=u'utf-8') if bom: - response.write(UTF8_BOM) - wr.writerow(f['id'] for f in fields) - yield wr + response.write(BOM_UTF8) + + unicodecsv.writer(response, encoding=u'utf-8').writerow( + f['id'] for f in fields) + yield TextWriter(response) @contextmanager @@ -55,10 +44,6 @@ def tsv_writer(response, fields, name=None, bom=False): :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file - - >>> with tsv_writer(response, fields) as d: - >>> d.writerow(row1) - >>> d.writerow(row2) ''' if hasattr(response, u'headers'): @@ -68,23 +53,22 @@ def tsv_writer(response, fields, name=None, bom=False): response.headers['Content-disposition'] = ( b'attachment; filename="{name}.tsv"'.format( name=encode_rfc2231(name))) - wr = CSVWriter( - response, fields, encoding=u'utf-8', dialect=unicodecsv.excel_tab, - ) if bom: - response.write(UTF8_BOM) - wr.writerow(f['id'] for f in fields) - yield wr + response.write(BOM_UTF8) + unicodecsv.writer( + response, encoding=u'utf-8', dialect=unicodecsv.excel_tab).writerow( + f['id'] for f in fields) + yield TextWriter(response) -class CSVWriter(object): - def __init__(self, response, columns, *args, **kwargs): - self._wr = unicodecsv.writer(response, *args, **kwargs) - self.columns = columns - def writerow(self, row): - return self._wr.writerow([ - _json_dump_nested(val) for val in row]) +class TextWriter(object): + u'text in, text out' + def __init__(self, response): + self.response = response + + def write_records(self, records): + self.response.write(records) @contextmanager @@ -96,10 +80,6 @@ def json_writer(response, fields, name=None, bom=False): :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file - - >>> with json_writer(response, fields) as d: - >>> d.writerow(row1) - >>> d.writerow(row2) ''' if hasattr(response, u'headers'): @@ -110,31 +90,29 @@ def json_writer(response, fields, name=None, bom=False): b'attachment; filename="{name}.json"'.format( name=encode_rfc2231(name))) if bom: - response.write(UTF8_BOM) + response.write(BOM_UTF8) response.write( - b'{\n "fields": %s,\n "records": [' % json.dumps( + b'{\n "fields": %s,\n "records": [' % dumps( fields, ensure_ascii=False, separators=(u',', u':'))) - yield JSONWriter(response, [f['id'] for f in fields]) + yield JSONWriter(response) response.write(b'\n]}\n') class JSONWriter(object): - def __init__(self, response, columns): + def __init__(self, response): self.response = response - self.columns = columns self.first = True - def writerow(self, row): - if self.first: - self.first = False - self.response.write(b'\n ') - else: - self.response.write(b',\n ') - self.response.write(json.dumps( - row, - ensure_ascii=False, - separators=(u',', u':'), - sort_keys=True).encode(u'utf-8')) + def write_records(self, records): + for r in records: + if self.first: + self.first = False + self.response.write(b'\n ') + else: + self.response.write(b',\n ') + + self.response.write(dumps( + r, ensure_ascii=False, separators=(u',', u':'))) @contextmanager @@ -146,10 +124,6 @@ def xml_writer(response, fields, name=None, bom=False): :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file - - >>> with xml_writer(response, fields) as d: - >>> d.writerow(row1) - >>> d.writerow(row2) ''' if hasattr(response, u'headers'): @@ -160,7 +134,7 @@ def xml_writer(response, fields, name=None, bom=False): b'attachment; filename="{name}.xml"'.format( name=encode_rfc2231(name))) if bom: - response.write(UTF8_BOM) + response.write(BOM_UTF8) response.write(b'\n') yield XMLWriter(response, [f['id'] for f in fields]) response.write(b'\n') @@ -194,12 +168,12 @@ def _insert_node(self, root, k, v, key_attr=None): if key_attr is not None: element.attrib[self._key_attr] = unicode(key_attr) - def writerow(self, row): - root = Element(u'row') - if self.id_col: - root.attrib[u'_id'] = unicode(row[0]) - row = row[1:] - for k, v in zip(self.columns, row): - self._insert_node(root, k, v) - ElementTree(root).write(self.response, encoding=u'utf-8') - self.response.write(b'\n') + def write_records(self, records): + for r in records: + root = Element(u'row') + if self.id_col: + root.attrib[u'_id'] = unicode(r[u'_id']) + for c in self.columns: + self._insert_node(root, c, r[c]) + ElementTree(root).write(self.response, encoding=u'utf-8') + self.response.write(b'\n') From d7ab29e6d3a6633f88d5a64d087b4ab48ac222c6 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sat, 8 Apr 2017 18:02:41 -0400 Subject: [PATCH 26/54] [#3523] dump doesnt need totals --- ckanext/datastore/controller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index d6c49b14efd..27ba7752607 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -125,6 +125,7 @@ def result_page(offs, lim): else min(PAGINATE_BY, lim), 'offset': offs, 'records_format': records_format, + 'include_total': 'false', # XXX: default() is broken }) result = result_page(offset, limit) From 90756afcb8b8710792c306344452f83b3cf1c7e3 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Wed, 12 Apr 2017 15:16:21 -0400 Subject: [PATCH 27/54] [#3428] create_function: allow arguments+return types --- ckanext/datastore/db.py | 10 ++++++++-- ckanext/datastore/logic/action.py | 4 +++- ckanext/datastore/logic/schema.py | 6 +++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 31e893ebf37..2c7b2a75b86 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1430,13 +1430,19 @@ def get_all_resources_ids_in_datastore(): return [q[0] for q in query.fetchall()] -def create_trigger_function(name, definition, or_replace): +def create_function(name, arguments, rettype, definition, or_replace): sql = u''' CREATE {or_replace} FUNCTION - {name}() RETURNS trigger AS {definition} + {name}({args}) RETURNS {rettype} AS {definition} LANGUAGE plpgsql;'''.format( or_replace=u'OR REPLACE' if or_replace else u'', name=datastore_helpers.identifier(name), + args=u', '.join( + u'{argname} {argtype}'.format( + argname=datastore_helpers.identifier(a['argname']), + argtype=datastore_helpers.identifier(a['argtype'])) + for a in arguments), + rettype=datastore_helpers.identifier(rettype), definition=datastore_helpers.literal_string(definition)) try: diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index cd2ef3d3dd9..359b8130e17 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -629,8 +629,10 @@ def datastore_function_create(context, data_dict): ''' p.toolkit.check_access('datastore_function_create', context, data_dict) - db.create_trigger_function( + db.create_function( name=data_dict['name'], + arguments=data_dict.get('arguments', []), + rettype=data_dict['rettype'], definition=data_dict['definition'], or_replace=data_dict['or_replace']) diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index d747d3bd293..aa18e553f7d 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -173,7 +173,11 @@ def datastore_function_create_schema(): 'name': [unicode_only, not_empty], 'or_replace': [default(False), boolean_validator], # we're only exposing functions for triggers at the moment - 'rettype': [unicode_only, OneOf(['trigger'])], + 'arguments': { + 'argname': [unicode_only, not_empty], + 'argtype': [unicode_only, not_empty], + }, + 'rettype': [default(u'void'), unicode_only], 'definition': [unicode_only], } From 63e7c138c944f035ecdbbe1bd7c6a75dff55a692 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 14 Apr 2017 22:56:05 +0100 Subject: [PATCH 28/54] [#3543] Install from source now works with 16.04 --- doc/conf.py | 3 +- .../installing/install-from-source.rst | 30 ++++++++++++------- doc/maintaining/installing/solr.rst | 29 +++++++++++++----- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 3ed57700fbb..c7ddf563e99 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -52,7 +52,8 @@ .. |storage_path| replace:: |storage_parent_dir|/default .. |reload_apache| replace:: sudo service apache2 reload .. |restart_apache| replace:: sudo service apache2 restart -.. |restart_solr| replace:: sudo service jetty restart +# restart_solr is deprecated +.. |restart_solr| replace:: sudo service jetty8 restart .. |solr| replace:: Solr .. |restructuredtext| replace:: reStructuredText .. |nginx| replace:: Nginx diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index c022946a9e7..92a7630cbb2 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -5,11 +5,11 @@ Installing CKAN from source =========================== This section describes how to install CKAN from source. Although -:doc:`install-from-package` is simpler, it requires Ubuntu 14.04 64-bit or -Ubuntu 12.04 64-bit. Installing CKAN from source works with other versions of -Ubuntu and with other operating systems (e.g. RedHat, Fedora, CentOS, OS X). -If you install CKAN from source on your own operating system, please share your -experiences on our +:doc:`install-from-package` is simpler, it requires Ubuntu 16.04 64-bit, Ubuntu +14.04 64-bit or Ubuntu 12.04 64-bit. Installing CKAN from source works with +other versions of Ubuntu and with other operating systems (e.g. RedHat, Fedora, +CentOS, OS X). If you install CKAN from source on your own operating system, +please share your experiences on our `How to Install CKAN `_ wiki page. @@ -21,7 +21,11 @@ work on CKAN. -------------------------------- If you're using a Debian-based operating system (such as Ubuntu) install the -required packages with this command:: +required packages with this command for Ubuntu 16.04:: + + sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-8-jdk redis-server + +or for Ubuntu 14.04/12.04:: sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-6-jdk redis-server @@ -41,7 +45,7 @@ virtualenv `The virtual Python environment builder `_ Apache Solr `A search platform `_ Jetty `An HTTP server `_ (used for Solr). -OpenJDK 6 JDK `The Java Development Kit `_ +OpenJDK JDK `The Java Development Kit `_ (used by Jetty) Redis `An in-memory data structure store `_ ===================== =============================================== @@ -321,9 +325,13 @@ If ``javac`` isn't installed, do:: and then restart Solr: -.. parsed-literal:: +For Ubuntu 16.04:: + + sudo service jetty8 restart + +or for Ubuntu 14.04/12.04:: - |restart_solr| + sudo service jetty restart AttributeError: 'module' object has no attribute 'css/main.debug.css' --------------------------------------------------------------------- @@ -331,11 +339,11 @@ AttributeError: 'module' object has no attribute 'css/main.debug.css' This error is likely to show up when `debug` is set to `True`. To fix this error, install frontend dependencies. See :doc:`/contributing/frontend/index`. -After installing the dependencies, run `bin/less` and then start paster server +After installing the dependencies, run ``bin/less`` and then start paster server again. If you do not want to compile CSS, you can also copy the main.css to -main.debug.css to get CKAN running. +main.debug.css to get CKAN running:: cp /usr/lib/ckan/default/src/ckan/ckan/public/base/css/main.css \ /usr/lib/ckan/default/src/ckan/ckan/public/base/css/main.debug.css diff --git a/doc/maintaining/installing/solr.rst b/doc/maintaining/installing/solr.rst index 2bd057537c1..2306336c9af 100644 --- a/doc/maintaining/installing/solr.rst +++ b/doc/maintaining/installing/solr.rst @@ -12,8 +12,8 @@ installed, we need to install and configure Solr. server, but CKAN doesn't require Jetty - you can deploy Solr to another web server, such as Tomcat, if that's convenient on your operating system. -#. Edit the Jetty configuration file (``/etc/default/jetty``) and change the - following variables:: +#. Edit the Jetty configuration file (``/etc/default/jetty8`` or + ``/etc/default/jetty``) and change the following variables:: NO_START=0 # (line 4) JETTY_HOST=127.0.0.1 # (line 16) @@ -26,9 +26,20 @@ installed, we need to install and configure Solr. change it to the relevant host or to 0.0.0.0 (and probably set up your firewall accordingly). - Start the Jetty server:: + Start or restart the Jetty server. - sudo service jetty start + For Ubuntu 16.04:: + + sudo service jetty8 restart + + Or for Ubuntu 14.04/12.04:: + + sudo service jetty restart + + .. note:: + + Ignore any warning that it wasn't already running - some Ubuntu + distributions choose not to start Jetty on install, but it's not important. You should now see a welcome page from Solr if you open http://localhost:8983/solr/ in your web browser (replace localhost with @@ -57,11 +68,15 @@ installed, we need to install and configure Solr. Now restart Solr: - .. parsed-literal:: + For Ubuntu 16.04:: + + sudo service jetty8 restart + + or for Ubuntu 14.04/12.04:: - |restart_solr| + sudo service jetty restart - and check that Solr is running by opening http://localhost:8983/solr/. + Check that Solr is running by opening http://localhost:8983/solr/. #. Finally, change the :ref:`solr_url` setting in your :ref:`config_file` (|production.ini|) to From 0b877d52d48311fd126d5dee32f6e96cafcfc9bf Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sat, 15 Apr 2017 10:16:21 -0400 Subject: [PATCH 29/54] [#3523] copy to: support where params --- ckanext/datastore/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 5f19932a6a9..4b4267cf142 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1130,7 +1130,7 @@ def _execute_single_statement_copy_to(context, sql_string, where_values, buf): }) cursor = context['connection'].connection.cursor() - cursor.copy_expert(sql_string, buf) + cursor.copy_expert(cursor.mogrify(sql_string, where_values), buf) cursor.close() From fac5d80429992c4e91a5a8d3d14949fdee21523a Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sat, 15 Apr 2017 21:40:48 -0400 Subject: [PATCH 30/54] [#3523] faster default records_format=objects output --- ckanext/datastore/db.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 4b4267cf142..5eea236ed7f 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1036,11 +1036,13 @@ def search_data(context, data_dict): records_format = data_dict['records_format'] if records_format == u'objects': sql_fmt = u''' - SELECT row_to_json(j)::text from ( - SELECT {distinct} {select} - FROM "{resource}" {ts_query} - {where} {sort} LIMIT {limit} OFFSET {offset} - ) AS j''' + SELECT array_to_json(array( + SELECT row_to_json(j) from ( + SELECT {distinct} {select} + FROM "{resource}" {ts_query} + {where} {sort} LIMIT {limit} OFFSET {offset} + ) AS j + ))::text''' elif records_format == u'lists': select_columns = u" || ',' || ".join( s for s in query_dict['select'] @@ -1079,6 +1081,10 @@ def search_data(context, data_dict): _execute_single_statement_copy_to( context, sql_string, where_values, buf) records = buf.getvalue() + elif records_format == u'objects': + records = LazyJSONObject( + list(_execute_single_statement( + context, sql_string, where_values))[0][0]) else: records = [ LazyJSONObject(r[0]) for r in From f51610ae78e39b2faff6de271941b0940ae88b03 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sat, 15 Apr 2017 23:11:21 -0400 Subject: [PATCH 31/54] [#3523] faster default dump records_format=lists output --- ckan/lib/lazyjson.py | 2 +- ckanext/datastore/db.py | 33 ++++++++++++++-------------- ckanext/datastore/tests/test_dump.py | 4 ++-- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index d3c0934cc78..05eb8be8a54 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -12,7 +12,7 @@ class LazyJSONObject(RawJSON): unicode strings containing a single JSON object. ''' def __init__(self, json_string): - assert isinstance(json_string, unicode) + assert isinstance(json_string, unicode), json_string self._json_string = json_string self._json_dict = None diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 5eea236ed7f..f0e78dba29b 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1036,21 +1036,21 @@ def search_data(context, data_dict): records_format = data_dict['records_format'] if records_format == u'objects': sql_fmt = u''' - SELECT array_to_json(array( - SELECT row_to_json(j) from ( - SELECT {distinct} {select} - FROM "{resource}" {ts_query} - {where} {sort} LIMIT {limit} OFFSET {offset} - ) AS j - ))::text''' + SELECT array_to_json(array_agg(j))::text FROM ( + SELECT {distinct} {select} + FROM "{resource}" {ts_query} + {where} {sort} LIMIT {limit} OFFSET {offset} + ) AS j''' elif records_format == u'lists': select_columns = u" || ',' || ".join( s for s in query_dict['select'] ).replace('%', '%%') sql_fmt = u''' - SELECT {distinct} '[' || {select} || ']' - FROM "{resource}" {ts_query} - {where} {sort} LIMIT {limit} OFFSET {offset}''' + SELECT '[' || array_to_string(array_agg(j.v), ',') || ']' FROM ( + SELECT {distinct} '[' || {select} || ']' v + FROM "{resource}" {ts_query} + {where} {sort} LIMIT {limit} OFFSET {offset} + ) AS j''' elif records_format == u'csv': sql_fmt = u''' COPY ( @@ -1081,14 +1081,13 @@ def search_data(context, data_dict): _execute_single_statement_copy_to( context, sql_string, where_values, buf) records = buf.getvalue() - elif records_format == u'objects': - records = LazyJSONObject( - list(_execute_single_statement( - context, sql_string, where_values))[0][0]) else: - records = [ - LazyJSONObject(r[0]) for r in - _execute_single_statement(context, sql_string, where_values)] + v = list(_execute_single_statement( + context, sql_string, where_values))[0][0] + if v is None: + records = [] + else: + records = LazyJSONObject(v) data_dict['records'] = records field_info = _get_field_info( diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 2a354463b0b..ad64bc47668 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -159,8 +159,8 @@ def test_dump_json(self): u'{"type":"_text","id":"random_letters"},{"type":"json",' u'"id":"nested"}],\n "records": [\n ' u'[1,"annakarenina","tolstoy","2005-03-01T00:00:00",' - u'["Princess Anna","Sergius"],["a","e","x"],["b", ' - u'{"moo": "moo"}]]\n]}\n') + u'["Princess Anna","Sergius"],["a","e","x"],["b",' + u'{"moo":"moo"}]]\n]}\n') assert_equals(content, expected_content) def test_dump_xml(self): From 903b857307f096da25896fe1eca5103902f31181 Mon Sep 17 00:00:00 2001 From: David Read Date: Sun, 16 Apr 2017 21:38:08 +0100 Subject: [PATCH 32/54] [#3543] Remove references to 12.04 from source install, because it is EOL this month. --- doc/maintaining/installing/install-from-source.rst | 14 +++++++------- doc/maintaining/installing/solr.rst | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index 92a7630cbb2..4e18e630e34 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -5,11 +5,11 @@ Installing CKAN from source =========================== This section describes how to install CKAN from source. Although -:doc:`install-from-package` is simpler, it requires Ubuntu 16.04 64-bit, Ubuntu -14.04 64-bit or Ubuntu 12.04 64-bit. Installing CKAN from source works with -other versions of Ubuntu and with other operating systems (e.g. RedHat, Fedora, -CentOS, OS X). If you install CKAN from source on your own operating system, -please share your experiences on our +:doc:`install-from-package` is simpler, it requires Ubuntu 16.04 64-bit or +Ubuntu 14.04 64-bit. Installing CKAN from source works with other versions of +Ubuntu and with other operating systems (e.g. RedHat, Fedora, CentOS, OS X). If +you install CKAN from source on your own operating system, please share your +experiences on our `How to Install CKAN `_ wiki page. @@ -25,7 +25,7 @@ required packages with this command for Ubuntu 16.04:: sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-8-jdk redis-server -or for Ubuntu 14.04/12.04:: +or for Ubuntu 14.04:: sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-6-jdk redis-server @@ -329,7 +329,7 @@ For Ubuntu 16.04:: sudo service jetty8 restart -or for Ubuntu 14.04/12.04:: +or for Ubuntu 14.04:: sudo service jetty restart diff --git a/doc/maintaining/installing/solr.rst b/doc/maintaining/installing/solr.rst index 2306336c9af..5da3851ce85 100644 --- a/doc/maintaining/installing/solr.rst +++ b/doc/maintaining/installing/solr.rst @@ -32,7 +32,7 @@ installed, we need to install and configure Solr. sudo service jetty8 restart - Or for Ubuntu 14.04/12.04:: + Or for Ubuntu 14.04:: sudo service jetty restart @@ -72,7 +72,7 @@ installed, we need to install and configure Solr. sudo service jetty8 restart - or for Ubuntu 14.04/12.04:: + or for Ubuntu 14.04:: sudo service jetty restart From 5e5f337addfec7e0fdb993cc0d31ffb6171cdc52 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 18 Apr 2017 12:39:42 -0400 Subject: [PATCH 33/54] [#3428] factor out ProgrammingError summarization --- ckanext/datastore/db.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 2c7b2a75b86..c3ed8ad97aa 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -720,8 +720,8 @@ def upsert_data(context, data_dict): "is out of range or was inserted into a text field)." )) except sqlalchemy.exc.InternalError as err: - message = err.args[0].split('\n')[0].decode('utf8') - raise ValidationError({u'records': [message.split(u') ', 1)[-1]]}) + raise ValidationError( + {u'records': [_programming_error_summary(err)]}) elif method in [_UPDATE, _UPSERT]: unique_keys = _get_unique_key(context, data_dict) @@ -1203,8 +1203,7 @@ def _create_triggers(connection, resource_id, triggers): if sql_list: connection.execute(u';\n'.join(sql_list)) except ProgrammingError as pe: - message = pe.args[0].split('\n')[0].decode('utf8') - raise ValidationError({u'triggers': [message.split(u') ', 1)[-1]]}) + raise ValidationError({u'triggers': [_programming_error_summary(pe)]}) def upsert(context, data_dict): @@ -1448,10 +1447,10 @@ def create_function(name, arguments, rettype, definition, or_replace): try: _write_engine_execute(sql) except ProgrammingError as pe: - message = pe.args[0].split('\n')[0].decode('utf8') - key = u'name' if message.startswith( - u'(ProgrammingError) function') else u'definition' - raise ValidationError({key: [message.split(u') ', 1)[-1]]}) + key = ( + u'name' if pe.args[0].startswith('(ProgrammingError) function') + else u'definition') + raise ValidationError({key: [_programming_error_summary(pe)]}) def drop_function(name, if_exists): @@ -1464,8 +1463,7 @@ def drop_function(name, if_exists): try: _write_engine_execute(sql) except ProgrammingError as pe: - message = pe.args[0].split('\n')[0].decode('utf8') - raise ValidationError({u'name': [message.split(u') ', 1)[-1]]}) + raise ValidationError({u'name': [_programming_error_summary(pe)]}) def _write_engine_execute(sql): @@ -1481,3 +1479,14 @@ def _write_engine_execute(sql): raise finally: connection.close() + + +def _programming_error_summary(pe): + u''' + return the text description of e sqlalchemy ProgrammingError or + InternalError without the actual SQL included, for raising as a + ValidationError to send back to API users + ''' + # first line only, after the '(ProgrammingError)' text + message = pe.args[0].split('\n')[0].decode('utf8') + raise message.split(u') ', 1)[-1] From 6bb499f40c7af55893d6df19126a6a60ca0ecb28 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 18 Apr 2017 13:22:22 -0400 Subject: [PATCH 34/54] [#3428] catch errors on upsert + update as well --- ckanext/datastore/db.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index c3ed8ad97aa..1cd3a393187 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -719,7 +719,7 @@ def upsert_data(context, data_dict): toolkit._("The data was invalid (for example: a numeric value " "is out of range or was inserted into a text field)." )) - except sqlalchemy.exc.InternalError as err: + except sqlalchemy.exc.DatabaseError as err: raise ValidationError( {u'records': [_programming_error_summary(err)]}) @@ -782,8 +782,13 @@ def upsert_data(context, data_dict): [u'"{0}"'.format(part) for part in unique_keys]), primary_value=u','.join(["%s"] * len(unique_keys)) ) - results = context['connection'].execute( - sql_string, used_values + [full_text] + unique_values) + try: + results = context['connection'].execute( + sql_string, used_values + [full_text] + unique_values) + except sqlalchemy.exc.DatabaseError as err: + raise ValidationError({ + u'records': [_programming_error_summary(err)], + u'_records_row': num}) # validate that exactly one row has been updated if results.rowcount != 1: @@ -811,9 +816,14 @@ def upsert_data(context, data_dict): for part in unique_keys]), primary_value=u','.join(["%s"] * len(unique_keys)) ) - context['connection'].execute( - sql_string, - (used_values + [full_text] + unique_values) * 2) + try: + context['connection'].execute( + sql_string, + (used_values + [full_text] + unique_values) * 2) + except sqlalchemy.exc.DatabaseError as err: + raise ValidationError({ + u'records': [_programming_error_summary(err)], + u'_records_row': num}) def _get_unique_key(context, data_dict): @@ -1483,8 +1493,8 @@ def _write_engine_execute(sql): def _programming_error_summary(pe): u''' - return the text description of e sqlalchemy ProgrammingError or - InternalError without the actual SQL included, for raising as a + return the text description of a sqlalchemy DatabaseError + without the actual SQL included, for raising as a ValidationError to send back to API users ''' # first line only, after the '(ProgrammingError)' text From c936de81c961afead0834c2a99431c40343c2d06 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 18 Apr 2017 16:05:04 -0400 Subject: [PATCH 35/54] [#3428] fix typo --- ckanext/datastore/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 1cd3a393187..f0090d68e12 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1499,4 +1499,4 @@ def _programming_error_summary(pe): ''' # first line only, after the '(ProgrammingError)' text message = pe.args[0].split('\n')[0].decode('utf8') - raise message.split(u') ', 1)[-1] + return message.split(u') ', 1)[-1] From 8f205ea8e80e2ff659b8e4684cb4c374d82d72b9 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 20 Apr 2017 10:51:09 -0400 Subject: [PATCH 36/54] [#3428] datastore docs: trigger feature paragraph --- doc/maintaining/datastore.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index b1b1790d314..9fd0f995d22 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -245,6 +245,10 @@ Data can be written incrementally to the DataStore through the API. New data can inserted, existing data can be updated or deleted. You can also add a new column to an existing table even if the DataStore resource already contains some data. +Triggers may be added to enforce validation, clean data as it is loaded or +even record record histories. Triggers are PL/pgSQL functions that must be +created by a sysadmin. + You will notice that we tried to keep the layer between the underlying PostgreSQL database and the API as thin as possible to allow you to use the features you would expect from a powerful database management system. From 49ed61273dc30eedd19753b70e8121837a2a9e5b Mon Sep 17 00:00:00 2001 From: Harald von Waldow Date: Thu, 20 Apr 2017 19:35:52 +0200 Subject: [PATCH 37/54] download icon for no_view --- ckan/templates/package/resource_read.html | 2 +- ckan/templates/package/snippets/resource_item.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index be0bba84aa6..a7ffbc0b3b8 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -37,7 +37,7 @@ {{ _('View') }} {% elif res.resource_type == 'api' %} {{ _('API Endpoint') }} - {% elif not res.has_views or not res.can_be_previewed %} + {% elif (not res.has_views or not res.can_be_previewed) and not res.url_type == 'upload' %} {{ _('Go to resource') }} {% else %} {{ _('Download') }} diff --git a/ckan/templates/package/snippets/resource_item.html b/ckan/templates/package/snippets/resource_item.html index 8e437da6f7d..992397e10f4 100644 --- a/ckan/templates/package/snippets/resource_item.html +++ b/ckan/templates/package/snippets/resource_item.html @@ -39,7 +39,7 @@ {% if res.url and h.is_url(res.url) %}
  • - {% if res.has_views %} + {% if res.has_views or res.url_type == 'upload' %} {{ _('Download') }} {% else %} From d7011c652db925483590d87e82608eaccc365e6f Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Fri, 21 Apr 2017 14:55:39 -0400 Subject: [PATCH 38/54] [#3523] faster offset for records_format=lists --- ckanext/datastore/db.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index f0e78dba29b..d67b59e25c1 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -1047,9 +1047,10 @@ def search_data(context, data_dict): ).replace('%', '%%') sql_fmt = u''' SELECT '[' || array_to_string(array_agg(j.v), ',') || ']' FROM ( - SELECT {distinct} '[' || {select} || ']' v - FROM "{resource}" {ts_query} - {where} {sort} LIMIT {limit} OFFSET {offset} + SELECT '[' || {select} || ']' v + FROM ( + SELECT {distinct} * FROM "{resource}" {ts_query} + {where} {sort} LIMIT {limit} OFFSET {offset}) as z ) AS j''' elif records_format == u'csv': sql_fmt = u''' From e43478cae108431690d608130efa9bfdcc78f6fa Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Fri, 21 Apr 2017 16:19:40 -0400 Subject: [PATCH 39/54] [#3523] trade some memory for >2x speedup when dumping csv --- ckanext/datastore/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 27ba7752607..ffd346359be 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -28,7 +28,7 @@ boolean_validator = get_validator('boolean_validator') DUMP_FORMATS = 'csv', 'tsv', 'json', 'xml' -PAGINATE_BY = 10000 +PAGINATE_BY = 32000 class DatastoreController(BaseController): From e2c35e6aa62737df69776a09f213c4e34abfbb1f Mon Sep 17 00:00:00 2001 From: David Read Date: Wed, 26 Apr 2017 10:51:55 +0000 Subject: [PATCH 40/54] [#3543] Fix test --- doc/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index c7ddf563e99..7a5692d42b8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -52,7 +52,6 @@ .. |storage_path| replace:: |storage_parent_dir|/default .. |reload_apache| replace:: sudo service apache2 reload .. |restart_apache| replace:: sudo service apache2 restart -# restart_solr is deprecated .. |restart_solr| replace:: sudo service jetty8 restart .. |solr| replace:: Solr .. |restructuredtext| replace:: reStructuredText From deda5b73ba16da25f3cb769a8774ef61863a4a30 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 2 May 2017 13:09:29 +0300 Subject: [PATCH 41/54] Restrict access to `members` and `bulk_actions` After this fix only users with permission `bulk_update_public` will be able to visit `bulc_process` page and only those who have `group_edit_permissions' will be able to visit `members` page --- ckan/controllers/group.py | 16 +++++++++++++--- ckan/logic/auth/update.py | 30 ++++++++++++++++++++++++------ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index e711fea8561..4c37dadf1bd 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -382,13 +382,16 @@ def bulk_process(self, id): data_dict = {'id': id, 'type': group_type} try: + self._check_access('bulk_update_public', context, data_dict) # Do not query for the group datasets when dictizing, as they will # be ignored and get requested on the controller anyway data_dict['include_datasets'] = False c.group_dict = self._action('group_show')(context, data_dict) c.group = context['group'] - except (NotFound, NotAuthorized): + except NotFound: abort(404, _('Group not found')) + except NotAuthorized: + abort(403, _('User %r not authorized to edit %s') % (c.user, id)) if not c.group_dict['is_organization']: # FIXME: better error @@ -634,14 +637,21 @@ def members(self, id): 'user': c.user} try: + data_dict = {'id': id} + self._check_access('group_edit_permissions', context, data_dict) c.members = self._action('member_list')( context, {'id': id, 'object_type': 'user'} ) - data_dict = {'id': id} data_dict['include_datasets'] = False c.group_dict = self._action('group_show')(context, data_dict) - except (NotFound, NotAuthorized): + except NotFound: abort(404, _('Group not found')) + except NotAuthorized: + abort( + 403, + _('User %r not authorized to edit members of %s') % ( + c.user, id)) + return self._render_template('group/members.html', group_type) def member_new(self, id): diff --git a/ckan/logic/auth/update.py b/ckan/logic/auth/update.py index 3d0e86c25cb..1114f775d21 100644 --- a/ckan/logic/auth/update.py +++ b/ckan/logic/auth/update.py @@ -153,14 +153,32 @@ def group_edit_permissions(context, data_dict): user = context['user'] group = logic_auth.get_group_object(context, data_dict) - authorized = authz.has_user_permission_for_group_or_org(group.id, - user, - 'update') + authorized = authz.has_user_permission_for_group_or_org( + group.id, user, 'update') if not authorized: - return {'success': False, - 'msg': _('User %s not authorized to edit permissions of group %s') % - (str(user), group.id)} + return { + 'success': False, + 'msg': _('User %s not authorized to' + ' edit permissions of group %s') % + (str(user), group.id)} + else: + return {'success': True} + + +def organization_edit_permissions(context, data_dict): + user = context['user'] + group = logic_auth.get_group_object(context, data_dict) + + authorized = authz.has_user_permission_for_group_or_org( + group.id, user, 'update') + + if not authorized: + return { + 'success': False, + 'msg': _('User %s not authorized to edit' + ' permissions of organization %s') % + (str(user), group.id)} else: return {'success': True} From 5dfcef6c72198ec7dfce7d3d43920c178872d318 Mon Sep 17 00:00:00 2001 From: Matt Fullerton Date: Tue, 2 May 2017 12:28:48 +0200 Subject: [PATCH 42/54] Remove exposed ports No need to expose ports externally when using the container names already provides the necessary access to the ports internally from the containers. --- contrib/docker/docker-compose.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 78ca3a948fd..51ffd3bbc56 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -12,17 +12,11 @@ ckan: db: container_name: db image: ckan/postgresql:latest - ports: - - "5432:5432" solr: container_name: solr image: ckan/solr:latest - ports: - - "8983:8983" redis: container_name: redis image: redis:latest - ports: - - "6379:6379" From 6e6394423a9c5856dbc1f1ae54fe33c0c12e0882 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 2 May 2017 14:27:25 +0300 Subject: [PATCH 43/54] test fixes --- ckan/controllers/group.py | 4 ++-- ckan/logic/auth/update.py | 17 ----------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 4c37dadf1bd..40b7ef2b5b3 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -382,7 +382,7 @@ def bulk_process(self, id): data_dict = {'id': id, 'type': group_type} try: - self._check_access('bulk_update_public', context, data_dict) + self._check_access('bulk_update_public', context, {'org_id': id}) # Do not query for the group datasets when dictizing, as they will # be ignored and get requested on the controller anyway data_dict['include_datasets'] = False @@ -638,7 +638,7 @@ def members(self, id): try: data_dict = {'id': id} - self._check_access('group_edit_permissions', context, data_dict) + check_access('group_edit_permissions', context, data_dict) c.members = self._action('member_list')( context, {'id': id, 'object_type': 'user'} ) diff --git a/ckan/logic/auth/update.py b/ckan/logic/auth/update.py index 1114f775d21..ae75f8cedb8 100644 --- a/ckan/logic/auth/update.py +++ b/ckan/logic/auth/update.py @@ -166,23 +166,6 @@ def group_edit_permissions(context, data_dict): return {'success': True} -def organization_edit_permissions(context, data_dict): - user = context['user'] - group = logic_auth.get_group_object(context, data_dict) - - authorized = authz.has_user_permission_for_group_or_org( - group.id, user, 'update') - - if not authorized: - return { - 'success': False, - 'msg': _('User %s not authorized to edit' - ' permissions of organization %s') % - (str(user), group.id)} - else: - return {'success': True} - - @logic.auth_allow_anonymous_access def user_update(context, data_dict): user = context['user'] From f992189d0e3557a66bb39a81802a08b2c385ffd3 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 2 May 2017 15:56:06 +0300 Subject: [PATCH 44/54] fixes in controller tests --- ckan/tests/controllers/test_group.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckan/tests/controllers/test_group.py b/ckan/tests/controllers/test_group.py index 67e06c92ce9..a0fdc00c13f 100644 --- a/ckan/tests/controllers/test_group.py +++ b/ckan/tests/controllers/test_group.py @@ -282,9 +282,12 @@ def test_membership_list(self): group = self._create_group(user_one['name'], other_users) + env = {'REMOTE_USER': user_one['name'].encode('ascii')} + member_list_url = url_for(controller='group', action='members', id=group['id']) - member_list_response = app.get(member_list_url) + member_list_response = app.get( + member_list_url, extra_environ=env) assert_true('2 members' in member_list_response) @@ -375,7 +378,7 @@ def test_remove_member(self): env = {'REMOTE_USER': user_one['name'].encode('ascii')} remove_response = app.post(remove_url, extra_environ=env, status=302) # redirected to member list after removal - remove_response = remove_response.follow() + remove_response = remove_response.follow(extra_environ=env) assert_true('Group member has been deleted.' in remove_response) assert_true('1 members' in remove_response) From 28afb7eec7de7db9a165b5c81def78403a4a08d9 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 11 May 2017 13:59:56 +0100 Subject: [PATCH 45/54] Readd blank line --- doc/maintaining/installing/install-from-source.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index b5e7bb44ded..e8c93a3e133 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -215,6 +215,7 @@ site_url .. include:: solr.rst + .. _postgres-init: ------------------------- From ed2f8aecb5de75f07c862bdb018569ed31a37044 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 11 May 2017 11:18:11 -0400 Subject: [PATCH 46/54] [#3428] datastore_create: more trigger docs --- ckanext/datastore/logic/action.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index ff9d849a206..4f49bb9373f 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -55,7 +55,10 @@ def datastore_create(context, data_dict): :type primary_key: list or comma separated string :param indexes: indexes on table (optional) :type indexes: list or comma separated string - :param triggers: triggers to apply to this table, eg: [ + :param triggers: trigger functions to apply to this table on update/insert. + functions may be created with + :meth:`~ckanext.datastore.logic.action.datastore_function_create`. + eg: [ {"function": "trigger_clean_reference"}, {"function": "trigger_check_codes"}] :type triggers: list of dictionaries From 32c5883a8f7338be2e37bea08a54a9b84258bd86 Mon Sep 17 00:00:00 2001 From: David Read Date: Mon, 15 May 2017 14:29:01 +0100 Subject: [PATCH 47/54] #3544 Fix info about packaging - 16.04 not supported yet for those. --- doc/maintaining/installing/install-from-source.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index e8c93a3e133..25b116b8a94 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -5,8 +5,8 @@ Installing CKAN from source =========================== This section describes how to install CKAN from source. Although -:doc:`install-from-package` is simpler, it requires Ubuntu 16.04 64-bit or -Ubuntu 14.04 64-bit. Installing CKAN from source works with other versions of +:doc:`install-from-package` is simpler, it requires Ubuntu 14.04 64-bit or +Ubuntu 12.04 64-bit. Installing CKAN from source works with other versions of Ubuntu and with other operating systems (e.g. RedHat, Fedora, CentOS, OS X). If you install CKAN from source on your own operating system, please share your experiences on our From 222d207dc6d1632cbf0f9e75b2c49194287d4b80 Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Mon, 15 May 2017 14:58:48 -0400 Subject: [PATCH 48/54] apply triggers to an existing datastore resource --- ckanext/datastore/logic/action.py | 31 +++++++++++++++++++++++++++++++ ckanext/datastore/logic/auth.py | 5 +++++ ckanext/datastore/plugin.py | 2 ++ 3 files changed, 38 insertions(+) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 4f49bb9373f..fdb390bcb34 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -161,6 +161,37 @@ def datastore_create(context, data_dict): return result +def datastore_trigger_each_row(context, data_dict): + ''' update each record with trigger + + The datastore_trigger_each_row API action allows you to apply triggers to + an existing DataStore resource. + + :param resource_id: resource id that the data is going to be stored under. + :type resource_id: string + + **Results:** + + :returns: The rowcount in the table. + :rtype: int + + ''' + res_id = data_dict['resource_id'] + p.toolkit.check_access('datastore_trigger_each_row', context, data_dict) + + connection = db.get_write_engine().connect() + + sql = sqlalchemy.text(u'''update {0} set _id=_id '''.format( + datastore_helpers.identifier(res_id))) + try: + results = connection.execute(sql) + except sqlalchemy.exc.DatabaseError as err: + message = err.args[0].split('\n')[0].decode('utf8') + raise p.toolkit.ValidationError({ + u'records': [message.split(u') ', 1)[-1]]}) + return results.rowcount + + def datastore_upsert(context, data_dict): '''Updates or inserts into a table in the DataStore diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py index 824a73ee46b..6df839247e5 100644 --- a/ckanext/datastore/logic/auth.py +++ b/ckanext/datastore/logic/auth.py @@ -67,3 +67,8 @@ def datastore_function_create(context, data_dict): def datastore_function_delete(context, data_dict): return {'success': False} + + +def datastore_trigger_each_row(context, data_dict): + '''sysadmin-only: functions can be used to skip access checks''' + return {'success': False} diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 33155a76f0b..20caf887b64 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -225,6 +225,7 @@ def get_actions(self): 'datastore_info': action.datastore_info, 'datastore_function_create': action.datastore_function_create, 'datastore_function_delete': action.datastore_function_delete, + 'datastore_trigger_each_row': action.datastore_trigger_each_row, } if not self.legacy_mode: if self.enable_sql_search: @@ -247,6 +248,7 @@ def get_auth_functions(self): 'datastore_change_permissions': auth.datastore_change_permissions, 'datastore_function_create': auth.datastore_function_create, 'datastore_function_delete': auth.datastore_function_delete, + 'datastore_trigger_each_row': auth.datastore_trigger_each_row, } def before_map(self, m): From cf48e035e8f103b80739caa96128d787ff487121 Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Tue, 16 May 2017 09:04:14 -0400 Subject: [PATCH 49/54] rename API --- ckanext/datastore/logic/action.py | 4 ++-- ckanext/datastore/logic/auth.py | 2 +- ckanext/datastore/plugin.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index fdb390bcb34..029bac79072 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -161,10 +161,10 @@ def datastore_create(context, data_dict): return result -def datastore_trigger_each_row(context, data_dict): +def datastore_run_triggers(context, data_dict): ''' update each record with trigger - The datastore_trigger_each_row API action allows you to apply triggers to + The datastore_run_triggers API action allows you to re-apply exisitng triggers to an existing DataStore resource. :param resource_id: resource id that the data is going to be stored under. diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py index 6df839247e5..0c9de0c5201 100644 --- a/ckanext/datastore/logic/auth.py +++ b/ckanext/datastore/logic/auth.py @@ -69,6 +69,6 @@ def datastore_function_delete(context, data_dict): return {'success': False} -def datastore_trigger_each_row(context, data_dict): +def datastore_run_triggers(context, data_dict): '''sysadmin-only: functions can be used to skip access checks''' return {'success': False} diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 20caf887b64..27cfd79a771 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -225,7 +225,7 @@ def get_actions(self): 'datastore_info': action.datastore_info, 'datastore_function_create': action.datastore_function_create, 'datastore_function_delete': action.datastore_function_delete, - 'datastore_trigger_each_row': action.datastore_trigger_each_row, + 'datastore_run_triggers': action.datastore_run_triggers, } if not self.legacy_mode: if self.enable_sql_search: @@ -248,7 +248,7 @@ def get_auth_functions(self): 'datastore_change_permissions': auth.datastore_change_permissions, 'datastore_function_create': auth.datastore_function_create, 'datastore_function_delete': auth.datastore_function_delete, - 'datastore_trigger_each_row': auth.datastore_trigger_each_row, + 'datastore_run_triggers': auth.datastore_run_triggers, } def before_map(self, m): From c58a16d5c19150f53badabf794cb1d6cd8d983e6 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 18 May 2017 14:52:36 +0200 Subject: [PATCH 50/54] email validation for user input --- ckan/logic/schema.py | 5 +++-- ckan/logic/validators.py | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 4a6eff941d3..fe8e05c38fc 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -70,6 +70,7 @@ extra_key_not_in_root_schema, empty_if_not_sysadmin, package_id_does_not_exist, + email_validator ) @@ -146,9 +147,9 @@ def default_create_package_schema(): 'name': [not_empty, unicode, name_validator, package_name_validator], 'title': [if_empty_same_as("name"), unicode], 'author': [ignore_missing, unicode], - 'author_email': [ignore_missing, unicode], + 'author_email': [ignore_missing, unicode, email_validator], 'maintainer': [ignore_missing, unicode], - 'maintainer_email': [ignore_missing, unicode], + 'maintainer_email': [ignore_missing, unicode, email_validator], 'license_id': [ignore_missing, unicode], 'notes': [ignore_missing, unicode], 'url': [ignore_missing, unicode], # , URL(add_http=False)], diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 5173b909167..c1e696e8758 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -831,3 +831,10 @@ def empty_if_not_sysadmin(key, data, errors, context): return empty(key, data, errors, context) + +def email_validator(value, context): + '''Validate email input ''' + if value: + if not re.match("\A(?P[\w\-_]+)@(?P[\w\-_]+).(?P[\w]+)\Z",value,re.IGNORECASE): + raise Invalid(_('Email "%s" is not valid format' % (value))) + return value \ No newline at end of file From 79b4f171ca85c58f8cc8e942a3b80f2618a64d76 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Fri, 19 May 2017 11:18:37 +0200 Subject: [PATCH 51/54] fix code by review notes --- ckan/logic/validators.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index c1e696e8758..1f751e5fe88 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -832,9 +832,16 @@ def empty_if_not_sysadmin(key, data, errors, context): empty(key, data, errors, context) +#pattern from https://html.spec.whatwg.org/#e-mail-state-(type=email) +email_pattern = re.compile(r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9]"\ + "(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]"\ + "(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$") + + def email_validator(value, context): '''Validate email input ''' + if value: - if not re.match("\A(?P[\w\-_]+)@(?P[\w\-_]+).(?P[\w]+)\Z",value,re.IGNORECASE): - raise Invalid(_('Email "%s" is not valid format' % (value))) - return value \ No newline at end of file + if not email_pattern.match(value): + raise Invalid(_('Email {email} is not a valid format').format(email=value)) + return value From 9e4599b468d2c96a37a6d1f58d285d0494fae5c0 Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Mon, 22 May 2017 11:07:51 +0200 Subject: [PATCH 52/54] [#3570] Fix syntax error in resource-view-filters JS module --- ckan/public/base/javascript/modules/resource-view-filters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/public/base/javascript/modules/resource-view-filters.js b/ckan/public/base/javascript/modules/resource-view-filters.js index 0a108acffbd..787e9443e41 100644 --- a/ckan/public/base/javascript/modules/resource-view-filters.js +++ b/ckan/public/base/javascript/modules/resource-view-filters.js @@ -6,7 +6,7 @@ this.ckan.module('resource-view-filters', function (jQuery) { resourceId = self.options.resourceId, fields = self.options.fields, dropdownTemplate = self.options.dropdownTemplate, - addFilterTemplate = '' + self._('Add Filter') + ''; + addFilterTemplate = '' + self._('Add Filter') + '', filtersDiv = $('
    '); var filters = ckan.views.filters.get(); From 0b527b4620c03bec903c59b0871ca5f7f8e763f6 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Tue, 23 May 2017 16:08:52 +0200 Subject: [PATCH 53/54] update test according rfc2606 --- ckan/tests/legacy/functional/api/test_activity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/tests/legacy/functional/api/test_activity.py b/ckan/tests/legacy/functional/api/test_activity.py index 52a2514c1c2..e7a233b4724 100644 --- a/ckan/tests/legacy/functional/api/test_activity.py +++ b/ckan/tests/legacy/functional/api/test_activity.py @@ -139,9 +139,9 @@ def make_package(name=None): 'name': name, 'title': 'My Test Package', 'author': 'test author', - 'author_email': 'test_author@test_author.com', + 'author_email': 'test_author@testauthor.com', 'maintainer': 'test maintainer', - 'maintainer_email': 'test_maintainer@test_maintainer.com', + 'maintainer_email': 'test_maintainer@testmaintainer.com', 'notes': 'some test notes', 'url': 'www.example.com', } From 45573026565ac179483e1a92b5953ed168ff1f4c Mon Sep 17 00:00:00 2001 From: Harald von Waldow Date: Tue, 23 May 2017 22:25:00 +0200 Subject: [PATCH 54/54] Fixed typo in "16. enable PostGIS on the main database" Change of owner for PostGIS table spatial_ref_sys instead of duplicate line. Compare http://docs.ckan.org/projects/ckanext-spatial/en/latest/install.html#ubuntu-14-04-postgresql-9-3-and-postgis-2-1 --- doc/maintaining/upgrading/upgrade-postgres.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/maintaining/upgrading/upgrade-postgres.rst b/doc/maintaining/upgrading/upgrade-postgres.rst index d0c57c3e84a..b43790aaf2c 100644 --- a/doc/maintaining/upgrading/upgrade-postgres.rst +++ b/doc/maintaining/upgrading/upgrade-postgres.rst @@ -212,7 +212,7 @@ Upgrading sudo -u postgres psql --cluster 9.4/main -d |database| -f /usr/share/postgresql/9.4/contrib/postgis-2.1/postgis.sql sudo -u postgres psql --cluster 9.4/main -d |database| -f /usr/share/postgresql/9.4/contrib/postgis-2.1/spatial_ref_sys.sql sudo -u postgres psql --cluster 9.4/main -d |database| -c 'ALTER TABLE geometry_columns OWNER TO ckan_default;' - sudo -u postgres psql --cluster 9.4/main -d |database| -c 'ALTER TABLE geometry_columns OWNER TO ckan_default;' + sudo -u postgres psql --cluster 9.4/main -d |database| -c 'ALTER TABLE spatial_ref_sys OWNER TO ckan_default;' To check if PostGIS was properly installed: