From 36753a211f20973a6d3fcb572dd1e3391760edcf Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 26 Sep 2019 19:07:13 +0300 Subject: [PATCH] Fix datastore_create tests --- ckan/tests/config/test_middleware.py | 4 +- ckan/tests/pytest/fixtures.py | 4 + ckanext/datastore/tests/conftest.py | 10 + ckanext/datastore/tests/test_create.py | 1435 +++++++++++++++--------- 4 files changed, 944 insertions(+), 509 deletions(-) create mode 100644 ckanext/datastore/tests/conftest.py diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index b3530c44751..d381ccf0c34 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -288,7 +288,7 @@ def test_user_objects_in_g_anon_user(self, app): extra_environ={u'REMOTE_USER': str(u'')}, ) assert flask.g.user == u'' - assert flask.g.userobj == None + assert flask.g.userobj is None assert flask.g.author == u'Unknown IP Address' assert flask.g.remote_addr == u'Unknown IP Address' @@ -349,7 +349,7 @@ def test_user_objects_in_c_anon_user(self, app): # tmpl_context available on response assert resp.tmpl_context.user == u'' - assert resp.tmpl_context.userobj == None + assert resp.tmpl_context.userobj is None assert resp.tmpl_context.author == u'Unknown IP Address' assert resp.tmpl_context.remote_addr == u'Unknown IP Address' diff --git a/ckan/tests/pytest/fixtures.py b/ckan/tests/pytest/fixtures.py index a107be963f6..bf12d4225a0 100644 --- a/ckan/tests/pytest/fixtures.py +++ b/ckan/tests/pytest/fixtures.py @@ -26,24 +26,28 @@ def make_app(ckan_config): """ return test_helpers._get_test_app + @pytest.fixture def app(make_app): """Instance of client app. """ return make_app() + @pytest.fixture def reset_db(): """Clear database. """ test_helpers.reset_db() + @pytest.fixture def reset_index(): """Clear search index. """ search.clear_all() + @pytest.fixture def reset_all(reset_db, reset_index): """Clear database and search index. diff --git a/ckanext/datastore/tests/conftest.py b/ckanext/datastore/tests/conftest.py new file mode 100644 index 00000000000..da81bc8b8c8 --- /dev/null +++ b/ckanext/datastore/tests/conftest.py @@ -0,0 +1,10 @@ +import pytest +import ckanext.datastore.tests.helpers as test_helpers + + +@pytest.fixture +def reset_all(reset_all): + engine = test_helpers.db.get_write_engine() + test_helpers.rebuild_all_dbs( + test_helpers.orm.scoped_session( + test_helpers.orm.sessionmaker(bind=engine))) diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 6879a20faa7..969b46cd7f9 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -1,52 +1,100 @@ # encoding: utf-8 import json -from nose.tools import assert_equal, assert_not_equal, raises +import pytest import sqlalchemy.orm as orm -from ckan.tests.helpers import _get_test_app +from nose.tools import assert_equal, assert_not_equal, raises -from ckan.common import config -import ckan.plugins as p import ckan.lib.create_test_data as ctd import ckan.model as model -import ckan.tests.helpers as helpers +import ckan.plugins as p import ckan.tests.factories as factories - +import ckan.tests.helpers as helpers import ckanext.datastore.backend.postgres as db -from ckanext.datastore.tests.helpers import ( - set_url_type, DatastoreFunctionalTestBase, DatastoreLegacyTestBase, - execute_sql, when_was_last_analyze) +from ckan.common import config from ckan.plugins.toolkit import ValidationError +from ckanext.datastore.tests.helpers import set_url_type, execute_sql, when_was_last_analyze + +@pytest.mark.ckan_pytest +@pytest.mark.usefixtures('reset_all', 'app') +class TestDatastoreCreateNewTests: + def _has_index_on_field(self, resource_id, field): + sql = u""" + SELECT + relname + FROM + pg_class + WHERE + pg_class.relname = %s + """ + index_name = db._generate_index_name(resource_id, field) + results = execute_sql(sql, index_name).fetchone() + return bool(results) -class TestDatastoreCreateNewTests(DatastoreFunctionalTestBase): + def _get_index_names(self, resource_id): + sql = u""" + SELECT + i.relname AS index_name + FROM + pg_class t, + pg_class i, + pg_index idx + WHERE + t.oid = idx.indrelid + AND i.oid = idx.indexrelid + AND t.relkind = 'r' + AND t.relname = %s + """ + results = execute_sql(sql, resource_id).fetchall() + return [result[0] for result in results] + + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_create_works_with_empty_array_in_json_field(self): package = factories.Dataset() data = { 'resource': { 'package_id': package['id'] }, - 'fields': [{'id': 'movie', 'type': 'text'}, - {'id': 'directors', 'type': 'json'}], - 'records': [{'movie': 'sideways', 'directors': []}] + 'fields': [{ + 'id': 'movie', + 'type': 'text' + }, { + 'id': 'directors', + 'type': 'json' + }], + 'records': [{ + 'movie': 'sideways', + 'directors': [] + }] } result = helpers.call_action('datastore_create', **data) assert result['resource_id'] is not None + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_create_works_with_empty_object_in_json_field(self): package = factories.Dataset() data = { 'resource': { 'package_id': package['id'] }, - 'fields': [{'id': 'movie', 'type': 'text'}, - {'id': 'director', 'type': 'json'}], - 'records': [{'movie': 'sideways', 'director': {}}] + 'fields': [{ + 'id': 'movie', + 'type': 'text' + }, { + 'id': 'director', + 'type': 'json' + }], + 'records': [{ + 'movie': 'sideways', + 'director': {} + }] } result = helpers.call_action('datastore_create', **data) - assert result['resource_id'] is not None + assert result['resource_id'] is not None + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_create_creates_index_on_primary_key(self): package = factories.Dataset() data = { @@ -61,6 +109,7 @@ def test_create_creates_index_on_primary_key(self): index_names = self._get_index_names(resource_id) assert resource_id + '_pkey' in index_names + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_create_creates_url_with_site_name(self): package = factories.Dataset() data = { @@ -75,6 +124,7 @@ def test_create_creates_url_with_site_name(self): url = resource['url'] assert url.startswith(config.get('ckan.site_url')) + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_create_index_on_specific_fields(self): package = factories.Dataset() data = { @@ -83,15 +133,22 @@ def test_create_index_on_specific_fields(self): 'author': ['tolstoy', 'dostoevsky'], 'package_id': package['id'] }, - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}], + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }], 'indexes': ['author'] } result = helpers.call_action('datastore_create', **data) resource_id = result['resource_id'] assert self._has_index_on_field(resource_id, '"author"') - def test_create_adds_index_on_full_text_search_when_creating_other_indexes(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_adds_index_on_full_text_search_when_creating_other_indexes( + self): package = factories.Dataset() data = { 'resource': { @@ -99,15 +156,22 @@ def test_create_adds_index_on_full_text_search_when_creating_other_indexes(self) 'author': ['tolstoy', 'dostoevsky'], 'package_id': package['id'] }, - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}], + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }], 'indexes': ['author'] } result = helpers.call_action('datastore_create', **data) resource_id = result['resource_id'] assert self._has_index_on_field(resource_id, '"_full_text"') - def test_create_adds_index_on_full_text_search_when_not_creating_other_indexes(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_adds_index_on_full_text_search_when_not_creating_other_indexes( + self): package = factories.Dataset() data = { 'resource': { @@ -115,13 +179,19 @@ def test_create_adds_index_on_full_text_search_when_not_creating_other_indexes(s 'author': ['tolstoy', 'dostoevsky'], 'package_id': package['id'] }, - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}], + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }], } result = helpers.call_action('datastore_create', **data) resource_id = result['resource_id'] assert self._has_index_on_field(resource_id, '"_full_text"') + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_create_add_full_text_search_indexes_on_every_text_field(self): package = factories.Dataset() data = { @@ -130,9 +200,15 @@ def test_create_add_full_text_search_indexes_on_every_text_field(self): 'author': ['tolstoy', 'dostoevsky'], 'package_id': package['id'] }, - 'fields': [{'id': 'boo%k', 'type': 'text'}, - {'id': 'author', 'type': 'text'}], - 'lang': 'english', + 'fields': [{ + 'id': 'boo%k', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }], + 'lang': + 'english', } result = helpers.call_action('datastore_create', **data) resource_id = result['resource_id'] @@ -141,29 +217,33 @@ def test_create_add_full_text_search_indexes_on_every_text_field(self): assert self._has_index_on_field(resource_id, "to_tsvector('english', \"author\")") + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_create_doesnt_add_more_indexes_when_updating_data(self): resource = factories.Resource() data = { 'resource_id': resource['id'], 'force': True, - 'records': [ - {'book': 'annakarenina', 'author': 'tolstoy'} - ] + 'records': [{ + 'book': 'annakarenina', + 'author': 'tolstoy' + }] } result = helpers.call_action('datastore_create', **data) previous_index_names = self._get_index_names(resource['id']) data = { 'resource_id': resource['id'], 'force': True, - 'records': [ - {'book': 'warandpeace', 'author': 'tolstoy'} - ] + 'records': [{ + 'book': 'warandpeace', + 'author': 'tolstoy' + }] } result = helpers.call_action('datastore_create', **data) current_index_names = self._get_index_names(resource['id']) assert_equal(previous_index_names, current_index_names) @raises(p.toolkit.ValidationError) + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_create_duplicate_fields(self): package = factories.Dataset() data = { @@ -172,42 +252,17 @@ def test_create_duplicate_fields(self): 'author': ['tolstoy', 'dostoevsky'], 'package_id': package['id'] }, - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'book', 'type': 'text'}], + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'book', + 'type': 'text' + }], } result = helpers.call_action('datastore_create', **data) - - def _has_index_on_field(self, resource_id, field): - sql = u""" - SELECT - relname - FROM - pg_class - WHERE - pg_class.relname = %s - """ - index_name = db._generate_index_name(resource_id, field) - results = execute_sql(sql, index_name).fetchone() - return bool(results) - - def _get_index_names(self, resource_id): - sql = u""" - SELECT - i.relname AS index_name - FROM - pg_class t, - pg_class i, - pg_index idx - WHERE - t.oid = idx.indrelid - AND i.oid = idx.indexrelid - AND t.relkind = 'r' - AND t.relname = %s - """ - results = execute_sql(sql, resource_id).fetchall() - return [result[0] for result in results] - + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_sets_datastore_active_on_resource_on_create(self): resource = factories.Resource() @@ -216,9 +271,10 @@ def test_sets_datastore_active_on_resource_on_create(self): data = { 'resource_id': resource['id'], 'force': True, - 'records': [ - {'book': 'annakarenina', 'author': 'tolstoy'} - ] + 'records': [{ + 'book': 'annakarenina', + 'author': 'tolstoy' + }] } helpers.call_action('datastore_create', **data) @@ -227,6 +283,7 @@ def test_sets_datastore_active_on_resource_on_create(self): assert_equal(resource['datastore_active'], True) + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_sets_datastore_active_on_resource_on_delete(self): resource = factories.Resource(datastore_active=True) @@ -235,14 +292,16 @@ def test_sets_datastore_active_on_resource_on_delete(self): data = { 'resource_id': resource['id'], 'force': True, - 'records': [ - {'book': 'annakarenina', 'author': 'tolstoy'} - ] + 'records': [{ + 'book': 'annakarenina', + 'author': 'tolstoy' + }] } helpers.call_action('datastore_create', **data) - helpers.call_action('datastore_delete', resource_id=resource['id'], + helpers.call_action('datastore_delete', + resource_id=resource['id'], force=True) resource = helpers.call_action('resource_show', id=resource['id']) @@ -250,6 +309,7 @@ def test_sets_datastore_active_on_resource_on_delete(self): assert_equal(resource['datastore_active'], False) @raises(p.toolkit.ValidationError) + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_create_exceeds_column_name_limit(self): package = factories.Dataset() data = { @@ -257,130 +317,179 @@ def test_create_exceeds_column_name_limit(self): 'package_id': package['id'] }, 'fields': [{ - 'id': 'This is a really long name for a column. Column names ' + 'id': + 'This is a really long name for a column. Column names ' 'in Postgres have a limit of 63 characters', - 'type': 'text' + 'type': + 'text' }] } result = helpers.call_action('datastore_create', **data) + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_calculate_record_count_is_false(self): resource = factories.Resource() data = { - 'resource_id': resource['id'], - 'fields': [{'id': 'name', 'type': 'text'}, - {'id': 'age', 'type': 'text'}], - 'records': [{"name": "Sunita", "age": "51"}, - {"name": "Bowan", "age": "68"}], - 'force': True, + 'resource_id': + resource['id'], + 'fields': [{ + 'id': 'name', + 'type': 'text' + }, { + 'id': 'age', + 'type': 'text' + }], + 'records': [{ + "name": "Sunita", + "age": "51" + }, { + "name": "Bowan", + "age": "68" + }], + 'force': + True, } helpers.call_action('datastore_create', **data) last_analyze = when_was_last_analyze(resource['id']) assert_equal(last_analyze, None) + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_calculate_record_count(self): # how datapusher loads data (send_resource_to_datastore) resource = factories.Resource() data = { - 'resource_id': resource['id'], - 'fields': [{'id': 'name', 'type': 'text'}, - {'id': 'age', 'type': 'text'}], - 'records': [{"name": "Sunita", "age": "51"}, - {"name": "Bowan", "age": "68"}], - 'calculate_record_count': True, - 'force': True, + 'resource_id': + resource['id'], + 'fields': [{ + 'id': 'name', + 'type': 'text' + }, { + 'id': 'age', + 'type': 'text' + }], + 'records': [{ + "name": "Sunita", + "age": "51" + }, { + "name": "Bowan", + "age": "68" + }], + 'calculate_record_count': + True, + 'force': + True, } helpers.call_action('datastore_create', **data) last_analyze = when_was_last_analyze(resource['id']) assert_not_equal(last_analyze, None) -class TestDatastoreCreate(DatastoreLegacyTestBase): +@pytest.mark.ckan_pytest +class TestDatastoreCreate: sysadmin_user = None normal_user = None - @classmethod - def setup_class(cls): - cls.app = _get_test_app() - super(TestDatastoreCreate, cls).setup_class() + @pytest.fixture(autouse=True) + def create_test_data(self, reset_all): ctd.CreateTestData.create() - cls.sysadmin_user = model.User.get('testsysadmin') - cls.normal_user = model.User.get('annafan') + self.sysadmin_user = model.User.get('testsysadmin') + self.normal_user = model.User.get('annafan') engine = db.get_write_engine() - cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) + self.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( - model.Package.get('annakarenina').resources, cls.sysadmin_user) + model.Package.get('annakarenina').resources, self.sysadmin_user) - def test_create_requires_auth(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_requires_auth(self, app): resource = model.Package.get('annakarenina').resources[0] - data = { - 'resource_id': resource.id - } + data = {'resource_id': resource.id} postparams = '%s=1' % json.dumps(data) - res = self.app.post('/api/action/datastore_create', params=postparams, - status=403) + res = app.post('/api/action/datastore_create', + params=postparams, + status=403) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_empty_fails(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_empty_fails(self, app): postparams = '%s=1' % json.dumps({}) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_invalid_alias_name(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_invalid_alias_name(self, app): resource = model.Package.get('annakarenina').resources[0] data = { - 'resource_id': resource.id, - 'aliases': u'foo"bar', - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}] + 'resource_id': + resource.id, + 'aliases': + u'foo"bar', + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False data = { - 'resource_id': resource.id, - 'aliases': u'fo%25bar', # alias with percent - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}] + 'resource_id': + resource.id, + 'aliases': + u'fo%25bar', # alias with percent + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_duplicate_alias_name(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_duplicate_alias_name(self, app): resource = model.Package.get('annakarenina').resources[0] - data = { - 'resource_id': resource.id, - 'aliases': u'myalias' - } + data = {'resource_id': resource.id, 'aliases': u'myalias'} postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=200) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=200) res_dict = json.loads(res.body) assert res_dict['success'] is True # try to create another table with the same alias resource = model.Package.get('annakarenina').resources[1] - data = { - 'resource_id': resource.id, - 'aliases': u'myalias' - } + data = {'resource_id': resource.id, 'aliases': u'myalias'} postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False @@ -392,71 +501,119 @@ def test_create_duplicate_alias_name(self): } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_invalid_field_type(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_invalid_field_type(self, app): resource = model.Package.get('annakarenina').resources[0] data = { - 'resource_id': resource.id, - 'fields': [{'id': 'book', 'type': 'int['}, # this is invalid - {'id': 'author', 'type': 'INVALID'}] + 'resource_id': + resource.id, + 'fields': [ + { + 'id': 'book', + 'type': 'int[' + }, # this is invalid + { + 'id': 'author', + 'type': 'INVALID' + } + ] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_invalid_field_name(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_invalid_field_name(self, app): resource = model.Package.get('annakarenina').resources[0] auth = {'Authorization': str(self.sysadmin_user.apikey)} - invalid_names = ['_author', '"author', '', ' author', 'author ', - '\tauthor', 'author\n'] + invalid_names = [ + '_author', '"author', '', ' author', 'author ', '\tauthor', + 'author\n' + ] for field_name in invalid_names: data = { - 'resource_id': resource.id, - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': field_name, 'type': 'text'}] + 'resource_id': + resource.id, + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': field_name, + 'type': 'text' + }] } postparams = '%s=1' % json.dumps(data) - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_invalid_record_field(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_invalid_record_field(self, app): resource = model.Package.get('annakarenina').resources[0] data = { - 'resource_id': resource.id, - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, - {'book': 'warandpeace', 'published': '1869'}] + 'resource_id': + resource.id, + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }], + 'records': [{ + 'book': 'annakarenina', + 'author': 'tolstoy' + }, { + 'book': 'warandpeace', + 'published': '1869' + }] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_bad_records(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_bad_records(self, app): resource = model.Package.get('annakarenina').resources[0] data = { - 'resource_id': resource.id, - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}], + 'resource_id': + resource.id, + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }], 'records': ['bad'] # treat author as null } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False @@ -464,64 +621,104 @@ def test_bad_records(self): resource = model.Package.get('annakarenina').resources[0] data = { - 'resource_id': resource.id, - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy'}, - [], - {'book': 'warandpeace'}] # treat author as null + 'resource_id': + resource.id, + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }], + 'records': [{ + 'book': 'annakarenina', + 'author': 'tolstoy' + }, [], { + 'book': 'warandpeace' + }] # treat author as null } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False assert_equal(res_dict['error']['__type'], 'Validation Error') - def test_create_invalid_index(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_invalid_index(self, app): resource = model.Package.get('annakarenina').resources[0] data = { - 'resource_id': resource.id, - 'indexes': 'book, dummy', - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}] + 'resource_id': + resource.id, + 'indexes': + 'book, dummy', + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_invalid_unique_index(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_invalid_unique_index(self, app): resource = model.Package.get('annakarenina').resources[0] data = { - 'resource_id': resource.id, - 'primary_key': 'dummy', - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}] + 'resource_id': + resource.id, + 'primary_key': + 'dummy', + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_create_alias_twice(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_alias_twice(self, app): resource = model.Package.get('annakarenina').resources[1] data = { - 'resource_id': resource.id, - 'aliases': 'new_alias', - 'fields': [{'id': 'book', 'type': 'text'}, - {'id': 'author', 'type': 'text'}] + 'resource_id': + resource.id, + 'aliases': + 'new_alias', + 'fields': [{ + 'id': 'book', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'text' + }] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['success'] is True, res_dict @@ -529,40 +726,64 @@ def test_create_alias_twice(self): data = { 'resource_id': resource.id, 'aliases': 'new_alias', - 'fields': [{'id': 'more books', 'type': 'text'}] + 'fields': [{ + 'id': 'more books', + 'type': 'text' + }] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False, res_dict - def test_create_basic(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_basic(self, app): resource = model.Package.get('annakarenina').resources[0] aliases = [u'great_list_of_books', u'another_list_of_b\xfcks'] data = { - 'resource_id': resource.id, - 'aliases': aliases, - 'fields': [{'id': 'boo%k', 'type': 'text'}, # column with percent - {'id': 'author', 'type': 'json'}], + 'resource_id': + resource.id, + 'aliases': + aliases, + 'fields': [ + { + 'id': 'boo%k', + 'type': 'text' + }, # column with percent + { + 'id': 'author', + 'type': 'json' + } + ], 'indexes': [['boo%k', 'author'], 'author'], - 'records': [{'boo%k': 'crime', 'author': ['tolstoy', 'dostoevsky']}, - {'boo%k': 'annakarenina', 'author': ['tolstoy', 'putin']}, - {'boo%k': 'warandpeace'}] # treat author as null + 'records': [{ + 'boo%k': 'crime', + 'author': ['tolstoy', 'dostoevsky'] + }, { + 'boo%k': 'annakarenina', + 'author': ['tolstoy', 'putin'] + }, { + 'boo%k': 'warandpeace' + }] # treat author as null } ### Firstly test to see whether resource has no datastore table yet postparams = '%s=1' % json.dumps({'id': resource.id}) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/resource_show', params=postparams, - extra_environ=auth) + res = app.post('/api/action/resource_show', + params=postparams, + extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['result']['datastore_active'] is False postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['success'] is True @@ -576,8 +797,8 @@ def test_create_basic(self): assert results.rowcount == 3 for i, row in enumerate(results): assert data['records'][i].get('boo%k') == row['boo%k'] - assert data['records'][i].get('author') == ( - json.loads(row['author'][0]) if row['author'] else None) + assert data['records'][i].get('author') == (json.loads( + row['author'][0]) if row['author'] else None) results = c.execute(''' select * from "{0}" where _full_text @@ to_tsquery('warandpeace') @@ -594,8 +815,13 @@ def test_create_basic(self): c = self.Session.connection() for alias in aliases: - results = [row for row in c.execute(u'select * from "{0}"'.format(resource.id))] - results_alias = [row for row in c.execute(u'select * from "{0}"'.format(alias))] + results = [ + row for row in c.execute(u'select * from "{0}"'.format( + resource.id)) + ] + results_alias = [ + row for row in c.execute(u'select * from "{0}"'.format(alias)) + ] assert results == results_alias @@ -608,21 +834,26 @@ def test_create_basic(self): # check to test to see if resource now has a datastore table postparams = '%s=1' % json.dumps({'id': resource.id}) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/resource_show', params=postparams, - extra_environ=auth) + res = app.post('/api/action/resource_show', + params=postparams, + extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['result']['datastore_active'] ####### insert again simple data2 = { 'resource_id': resource.id, - 'records': [{'boo%k': 'hagji murat', 'author': ['tolstoy']}] + 'records': [{ + 'boo%k': 'hagji murat', + 'author': ['tolstoy'] + }] } postparams = '%s=1' % json.dumps(data2) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['success'] is True @@ -636,8 +867,8 @@ def test_create_basic(self): all_data = data['records'] + data2['records'] for i, row in enumerate(results): assert all_data[i].get('boo%k') == row['boo%k'] - assert all_data[i].get('author') == ( - json.loads(row['author'][0]) if row['author'] else None) + assert all_data[i].get('author') == (json.loads(row['author'][0]) + if row['author'] else None) c = self.Session.connection() results = c.execute(''' @@ -648,16 +879,21 @@ def test_create_basic(self): ####### insert again extra field data3 = { - 'resource_id': resource.id, - 'records': [{'boo%k': 'crime and punsihment', - 'author': ['dostoevsky'], 'rating': 2}], + 'resource_id': + resource.id, + 'records': [{ + 'boo%k': 'crime and punsihment', + 'author': ['dostoevsky'], + 'rating': 2 + }], 'indexes': ['rating'] } postparams = '%s=1' % json.dumps(data3) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['success'] is True @@ -669,24 +905,32 @@ def test_create_basic(self): all_data = data['records'] + data2['records'] + data3['records'] for i, row in enumerate(results): - assert all_data[i].get('boo%k') == row['boo%k'], (i, all_data[i].get('boo%k'), row['boo%k']) - assert all_data[i].get('author') == (json.loads(row['author'][0]) if row['author'] else None) - - results = c.execute('''select * from "{0}" where _full_text @@ to_tsquery('dostoevsky') '''.format(resource.id)) + assert all_data[i].get('boo%k') == row['boo%k'], ( + i, all_data[i].get('boo%k'), row['boo%k']) + assert all_data[i].get('author') == (json.loads(row['author'][0]) + if row['author'] else None) + + results = c.execute( + '''select * from "{0}" where _full_text @@ to_tsquery('dostoevsky') ''' + .format(resource.id)) self.Session.remove() assert results.rowcount == 2 ####### insert again which will fail because of unique book name data4 = { 'resource_id': resource.id, - 'records': [{'boo%k': 'warandpeace'}], + 'records': [{ + 'boo%k': 'warandpeace' + }], 'primary_key': 'boo%k' } postparams = '%s=1' % json.dumps(data4) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False @@ -696,14 +940,18 @@ def test_create_basic(self): data5 = { 'resource_id': resource.id, 'aliases': 'another_alias', # replaces aliases - 'records': [{'boo%k': 'warandpeace'}], + 'records': [{ + 'boo%k': 'warandpeace' + }], 'primary_key': '' } postparams = '%s=1' % json.dumps(data5) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, expect_errors=True) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + expect_errors=True) res_dict = json.loads(res.body) assert res_dict['success'] is True, res_dict @@ -724,41 +972,73 @@ def test_create_basic(self): ####### insert array type data6 = { - 'resource_id': resource.id, - 'fields': [{'id': 'boo%k', 'type': 'text'}, - {'id': 'author', 'type': 'json'}, - {'id': 'rating', 'type': 'int'}, - {'id': 'characters', 'type': '_text'}], # this is an array of strings - 'records': [{'boo%k': 'the hobbit', - 'author': ['tolkien'], 'characters': ['Bilbo', 'Gandalf']}], + 'resource_id': + resource.id, + 'fields': [{ + 'id': 'boo%k', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'json' + }, { + 'id': 'rating', + 'type': 'int' + }, { + 'id': 'characters', + 'type': '_text' + }], # this is an array of strings + 'records': [{ + 'boo%k': 'the hobbit', + 'author': ['tolkien'], + 'characters': ['Bilbo', 'Gandalf'] + }], 'indexes': ['characters'] } postparams = '%s=1' % json.dumps(data6) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, expect_errors=True) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + expect_errors=True) res_dict = json.loads(res.body) assert res_dict['success'] is True, res_dict ####### insert type that requires additional lookup data7 = { - 'resource_id': resource.id, - 'fields': [{'id': 'boo%k', 'type': 'text'}, - {'id': 'author', 'type': 'json'}, - {'id': 'rating', 'type': 'int'}, - {'id': 'characters', 'type': '_text'}, - {'id': 'location', 'type': 'int[2]'}], - 'records': [{'boo%k': 'lord of the rings', - 'author': ['tolkien'], 'location': [3, -42]}], + 'resource_id': + resource.id, + 'fields': [{ + 'id': 'boo%k', + 'type': 'text' + }, { + 'id': 'author', + 'type': 'json' + }, { + 'id': 'rating', + 'type': 'int' + }, { + 'id': 'characters', + 'type': '_text' + }, { + 'id': 'location', + 'type': 'int[2]' + }], + 'records': [{ + 'boo%k': 'lord of the rings', + 'author': ['tolkien'], + 'location': [3, -42] + }], 'indexes': ['characters'] } postparams = '%s=1' % json.dumps(data7) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, expect_errors=True) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + expect_errors=True) res_dict = json.loads(res.body) assert res_dict['success'] is True, res_dict @@ -766,37 +1046,58 @@ def test_create_basic(self): ####### insert with parameter id rather than resource_id which is a shortcut data8 = { 'id': resource.id, - # insert with percent - 'records': [{'boo%k': 'warandpeace', 'author': '99% good'}] + # insert with percent + 'records': [{ + 'boo%k': 'warandpeace', + 'author': '99% good' + }] } postparams = '%s=1' % json.dumps(data8) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, expect_errors=True) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + expect_errors=True) res_dict = json.loads(res.body) assert res_dict['success'] is True, res_dict - def test_create_datastore_resource_on_dataset(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_datastore_resource_on_dataset(self, app): pkg = model.Package.get('annakarenina') data = { 'resource': { 'package_id': pkg.id, }, - 'fields': [{'id': 'boo%k', 'type': 'text'}, # column with percent - {'id': 'author', 'type': 'json'}], + 'fields': [ + { + 'id': 'boo%k', + 'type': 'text' + }, # column with percent + { + 'id': 'author', + 'type': 'json' + } + ], 'indexes': [['boo%k', 'author'], 'author'], - 'records': [{'boo%k': 'crime', 'author': ['tolstoy', 'dostoevsky']}, - {'boo%k': 'annakarenina', 'author': ['tolstoy', 'putin']}, - {'boo%k': 'warandpeace'}] # treat author as null + 'records': [{ + 'boo%k': 'crime', + 'author': ['tolstoy', 'dostoevsky'] + }, { + 'boo%k': 'annakarenina', + 'author': ['tolstoy', 'putin'] + }, { + 'boo%k': 'warandpeace' + }] # treat author as null } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.normal_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['success'] is True @@ -804,147 +1105,244 @@ def test_create_datastore_resource_on_dataset(self): assert res['fields'] == data['fields'], res['fields'] # Get resource details - data = { - 'id': res['resource_id'] - } + data = {'id': res['resource_id']} postparams = '%s=1' % json.dumps(data) - res = self.app.post('/api/action/resource_show', params=postparams) + res = app.post('/api/action/resource_show', params=postparams) res_dict = json.loads(res.body) assert res_dict['result']['datastore_active'] is True - - def test_guess_types(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_guess_types(self, app): resource = model.Package.get('annakarenina').resources[1] - data = { - 'resource_id': resource.id - } + data = {'resource_id': resource.id} postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_delete', params=postparams, - extra_environ=auth, status="*") # ignore status + res = app.post('/api/action/datastore_delete', + params=postparams, + extra_environ=auth, + status="*") # ignore status res_dict = json.loads(res.body) data = { - 'resource_id': resource.id, - 'fields': [{'id': 'author', 'type': 'json'}, - {'id': 'count'}, - {'id': 'book'}, - {'id': 'date'}], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy', - 'count': 1, 'date': '2005-12-01', 'count2': 0.5}, - {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}, - {'book': 'warandpeace'}] # treat author as null + 'resource_id': + resource.id, + 'fields': [{ + 'id': 'author', + 'type': 'json' + }, { + 'id': 'count' + }, { + 'id': 'book' + }, { + 'id': 'date' + }], + 'records': [{ + 'book': 'annakarenina', + 'author': 'tolstoy', + 'count': 1, + 'date': '2005-12-01', + 'count2': 0.5 + }, { + 'book': 'crime', + 'author': ['tolstoy', 'dostoevsky'] + }, { + 'book': 'warandpeace' + }] # treat author as null } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth) res_dict = json.loads(res.body) c = self.Session.connection() results = c.execute('''select * from "{0}" '''.format(resource.id)) - types = [db._pg_types[field[1]] for field in results.cursor.description] + types = [ + db._pg_types[field[1]] for field in results.cursor.description + ] - assert types == [u'int4', u'tsvector', u'nested', u'int4', u'text', u'timestamp', u'float8'], types + assert types == [ + u'int4', u'tsvector', u'nested', u'int4', u'text', u'timestamp', + u'float8' + ], types assert results.rowcount == 3 for i, row in enumerate(results): assert data['records'][i].get('book') == row['book'] - assert data['records'][i].get('author') == ( - json.loads(row['author'][0]) if row['author'] else None) + assert data['records'][i].get('author') == (json.loads( + row['author'][0]) if row['author'] else None) self.Session.remove() ### extend types data = { - 'resource_id': resource.id, - 'fields': [{'id': 'author', 'type': 'text'}, - {'id': 'count'}, - {'id': 'book'}, - {'id': 'date'}, - {'id': 'count2'}, - {'id': 'extra', 'type':'text'}, - {'id': 'date2'}, - ], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy', - 'count': 1, 'date': '2005-12-01', 'count2': 2, - 'nested': [1, 2], 'date2': '2005-12-01'}] + 'resource_id': + resource.id, + 'fields': [ + { + 'id': 'author', + 'type': 'text' + }, + { + 'id': 'count' + }, + { + 'id': 'book' + }, + { + 'id': 'date' + }, + { + 'id': 'count2' + }, + { + 'id': 'extra', + 'type': 'text' + }, + { + 'id': 'date2' + }, + ], + 'records': [{ + 'book': 'annakarenina', + 'author': 'tolstoy', + 'count': 1, + 'date': '2005-12-01', + 'count2': 2, + 'nested': [1, 2], + 'date2': '2005-12-01' + }] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth) res_dict = json.loads(res.body) c = self.Session.connection() results = c.execute('''select * from "{0}" '''.format(resource.id)) self.Session.remove() - types = [db._pg_types[field[1]] for field in results.cursor.description] - - assert types == [u'int4', # id - u'tsvector', # fulltext - u'nested', # author - u'int4', # count - u'text', # book - u'timestamp', # date - u'float8', # count2 - u'text', # extra - u'timestamp', # date2 - u'nested', # count3 - ], types + types = [ + db._pg_types[field[1]] for field in results.cursor.description + ] + + assert types == [ + u'int4', # id + u'tsvector', # fulltext + u'nested', # author + u'int4', # count + u'text', # book + u'timestamp', # date + u'float8', # count2 + u'text', # extra + u'timestamp', # date2 + u'nested', # count3 + ], types ### fields resupplied in wrong order data = { - 'resource_id': resource.id, - 'fields': [{'id': 'author', 'type': 'text'}, - {'id': 'count'}, - {'id': 'date'}, # date and book in wrong order - {'id': 'book'}, - {'id': 'count2'}, - {'id': 'extra', 'type':'text'}, - {'id': 'date2'}, - ], - 'records': [{'book': 'annakarenina', 'author': 'tolstoy', - 'count': 1, 'date': '2005-12-01', 'count2': 2, - 'count3': 432, 'date2': '2005-12-01'}] + 'resource_id': + resource.id, + 'fields': [ + { + 'id': 'author', + 'type': 'text' + }, + { + 'id': 'count' + }, + { + 'id': 'date' + }, # date and book in wrong order + { + 'id': 'book' + }, + { + 'id': 'count2' + }, + { + 'id': 'extra', + 'type': 'text' + }, + { + 'id': 'date2' + }, + ], + 'records': [{ + 'book': 'annakarenina', + 'author': 'tolstoy', + 'count': 1, + 'date': '2005-12-01', + 'count2': 2, + 'count3': 432, + 'date2': '2005-12-01' + }] } postparams = '%s=1' % json.dumps(data) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False - def test_datastore_create_with_invalid_data_value(self): + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_datastore_create_with_invalid_data_value(self, app): """datastore_create() should return an error for invalid data.""" resource = factories.Resource(url_type="datastore") data_dict = { - "resource_id": resource["id"], - "fields": [{"id": "value", "type": "numeric"}], + "resource_id": + resource["id"], + "fields": [{ + "id": "value", + "type": "numeric" + }], "records": [ - {"value": 0}, - {"value": 1}, - {"value": 2}, - {"value": 3}, - {"value": " "}, # Invalid numeric value. - {"value": 5}, - {"value": 6}, - {"value": 7}, + { + "value": 0 + }, + { + "value": 1 + }, + { + "value": 2 + }, + { + "value": 3 + }, + { + "value": " " + }, # Invalid numeric value. + { + "value": 5 + }, + { + "value": 6 + }, + { + "value": 7 + }, ], - "method": "insert", + "method": + "insert", } postparams = '%s=1' % json.dumps(data_dict) auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) + res = app.post('/api/action/datastore_create', + params=postparams, + extra_environ=auth, + status=409) res_dict = json.loads(res.body) assert res_dict['success'] is False @@ -952,211 +1350,234 @@ def test_datastore_create_with_invalid_data_value(self): assert res_dict['error']['message'].startswith('The data was invalid') -class TestDatastoreFunctionCreate(DatastoreFunctionalTestBase): +@pytest.mark.ckan_pytest +@pytest.mark.usefixtures('reset_all', 'app') +class TestDatastoreFunctionCreate: + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_nop_trigger(self): - helpers.call_action( - u'datastore_function_create', - name=u'test_nop', - rettype=u'trigger', - definition=u'BEGIN RETURN NEW; END;') + helpers.call_action(u'datastore_function_create', + name=u'test_nop', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_invalid_definition(self): - try: - helpers.call_action( - u'datastore_function_create', - name=u'test_invalid_def', - rettype=u'trigger', - definition=u'HELLO WORLD') - except ValidationError as ve: - assert_equal( - ve.error_dict, - {u'definition': - [u'syntax error at or near "HELLO"']}) - else: - assert 0, u'no validation error' + with pytest.raises(ValidationError) as error: + helpers.call_action(u'datastore_function_create', + name=u'test_invalid_def', + rettype=u'trigger', + definition=u'HELLO WORLD') + assert error.value.error_dict == { + u'definition': [u'syntax error at or near "HELLO"'] + } + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_redefined_trigger(self): - helpers.call_action( - u'datastore_function_create', - name=u'test_redefined', - rettype=u'trigger', - definition=u'BEGIN RETURN NEW; END;') - try: - helpers.call_action( - u'datastore_function_create', - name=u'test_redefined', - rettype=u'trigger', - definition=u'BEGIN RETURN NEW; END;') - except ValidationError as ve: - assert_equal( - ve.error_dict, - {u'name':[ - u'function "test_redefined" already exists ' - u'with same argument types']}) - else: - assert 0, u'no validation error' + helpers.call_action(u'datastore_function_create', + name=u'test_redefined', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + with pytest.raises(ValidationError) as error: + helpers.call_action(u'datastore_function_create', + name=u'test_redefined', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + assert error.value.error_dict == { + u'name': [ + u'function "test_redefined" already exists ' + u'with same argument types' + ] + } + @pytest.mark.ckan_config('ckan.plugins', 'datastore') def test_redefined_with_or_replace_trigger(self): - helpers.call_action( - u'datastore_function_create', - name=u'test_replaceme', - rettype=u'trigger', - definition=u'BEGIN RETURN NEW; END;') - helpers.call_action( - u'datastore_function_create', - name=u'test_replaceme', - or_replace=True, - rettype=u'trigger', - definition=u'BEGIN RETURN NEW; END;') - - -class TestDatastoreCreateTriggers(DatastoreFunctionalTestBase): - def test_create_with_missing_trigger(self): + helpers.call_action(u'datastore_function_create', + name=u'test_replaceme', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + helpers.call_action(u'datastore_function_create', + name=u'test_replaceme', + or_replace=True, + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + + +@pytest.mark.ckan_pytest +@pytest.mark.usefixtures('reset_all') +class TestDatastoreCreateTriggers: + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_with_missing_trigger(self, app): ds = factories.Dataset() - try: - app = self._get_test_app() + with pytest.raises(ValidationError) as error: with app.flask_app.test_request_context(): - helpers.call_action( - u'datastore_create', - resource={u'package_id': ds['id']}, - fields=[{u'id': u'spam', u'type': u'text'}], - records=[{u'spam': u'SPAM'}, {u'spam': u'EGGS'}], - triggers=[{u'function': u'no_such_trigger_function'}]) - except ValidationError as ve: - assert_equal( - ve.error_dict, - {u'triggers':[ - u'function no_such_trigger_function() does not exist']}) - else: - assert 0, u'no validation error' - - def test_create_trigger_applies_to_records(self): + helpers.call_action(u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{ + u'id': u'spam', + u'type': u'text' + }], + records=[{ + u'spam': u'SPAM' + }, { + u'spam': u'EGGS' + }], + triggers=[{ + u'function': + u'no_such_trigger_function' + }]) + assert error.value.error_dict == { + u'triggers': + [u'function no_such_trigger_function() does not exist'] + } + + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_trigger_applies_to_records(self, app): ds = factories.Dataset() - helpers.call_action( - u'datastore_function_create', - name=u'spamify_trigger', - rettype=u'trigger', - definition=u''' + helpers.call_action(u'datastore_function_create', + name=u'spamify_trigger', + rettype=u'trigger', + definition=u''' BEGIN NEW.spam := 'spam spam ' || NEW.spam || ' spam'; RETURN NEW; END;''') - app = self._get_test_app() with app.flask_app.test_request_context(): - res = helpers.call_action( - u'datastore_create', - resource={u'package_id': ds['id']}, - fields=[{u'id': u'spam', u'type': u'text'}], - records=[{u'spam': u'SPAM'}, {u'spam': u'EGGS'}], - triggers=[{u'function': u'spamify_trigger'}]) + res = helpers.call_action(u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{ + u'id': u'spam', + u'type': u'text' + }], + records=[{ + u'spam': u'SPAM' + }, { + u'spam': u'EGGS' + }], + triggers=[{ + u'function': u'spamify_trigger' + }]) assert_equal( - helpers.call_action( - u'datastore_search', - fields=[u'spam'], - resource_id=res['resource_id'])['records'], - [ - {u'spam': u'spam spam SPAM spam'}, - {u'spam': u'spam spam EGGS spam'}]) - - def test_upsert_trigger_applies_to_records(self): + helpers.call_action(u'datastore_search', + fields=[u'spam'], + resource_id=res['resource_id'])['records'], + [{ + u'spam': u'spam spam SPAM spam' + }, { + u'spam': u'spam spam EGGS spam' + }]) + + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_upsert_trigger_applies_to_records(self, app): ds = factories.Dataset() - helpers.call_action( - u'datastore_function_create', - name=u'more_spam_trigger', - rettype=u'trigger', - definition=u''' + helpers.call_action(u'datastore_function_create', + name=u'more_spam_trigger', + rettype=u'trigger', + definition=u''' BEGIN NEW.spam := 'spam spam ' || NEW.spam || ' spam'; RETURN NEW; END;''') - app = self._get_test_app() with app.flask_app.test_request_context(): - res = helpers.call_action( - u'datastore_create', - resource={u'package_id': ds['id']}, - fields=[{u'id': u'spam', u'type': u'text'}], - triggers=[{u'function': u'more_spam_trigger'}]) - helpers.call_action( - u'datastore_upsert', - method=u'insert', - resource_id=res['resource_id'], - records=[{u'spam': u'BEANS'}, {u'spam': u'SPAM'}]) + res = helpers.call_action(u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{ + u'id': u'spam', + u'type': u'text' + }], + triggers=[{ + u'function': + u'more_spam_trigger' + }]) + helpers.call_action(u'datastore_upsert', + method=u'insert', + resource_id=res['resource_id'], + records=[{ + u'spam': u'BEANS' + }, { + u'spam': u'SPAM' + }]) assert_equal( - helpers.call_action( - u'datastore_search', - fields=[u'spam'], - resource_id=res['resource_id'])['records'], - [ - {u'spam': u'spam spam BEANS spam'}, - {u'spam': u'spam spam SPAM spam'}]) - - def test_create_trigger_exception(self): + helpers.call_action(u'datastore_search', + fields=[u'spam'], + resource_id=res['resource_id'])['records'], + [{ + u'spam': u'spam spam BEANS spam' + }, { + u'spam': u'spam spam SPAM spam' + }]) + + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_create_trigger_exception(self, app): ds = factories.Dataset() - helpers.call_action( - u'datastore_function_create', - name=u'spamexception_trigger', - rettype=u'trigger', - definition=u''' + helpers.call_action(u'datastore_function_create', + name=u'spamexception_trigger', + rettype=u'trigger', + definition=u''' BEGIN IF NEW.spam != 'spam' THEN RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam; END IF; RETURN NEW; END;''') - try: - app = self._get_test_app() + with pytest.raises(ValidationError) as error: with app.flask_app.test_request_context(): - helpers.call_action( - u'datastore_create', - resource={u'package_id': ds['id']}, - fields=[{u'id': u'spam', u'type': u'text'}], - records=[{u'spam': u'spam'}, {u'spam': u'EGGS'}], - triggers=[{u'function': u'spamexception_trigger'}]) - except ValidationError as ve: - assert_equal( - ve.error_dict, - {u'records':[ - u'"EGGS"? Yeeeeccch!']}) - else: - assert 0, u'no validation error' - - def test_upsert_trigger_exception(self): + helpers.call_action(u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{ + u'id': u'spam', + u'type': u'text' + }], + records=[{ + u'spam': u'spam' + }, { + u'spam': u'EGGS' + }], + triggers=[{ + u'function': + u'spamexception_trigger' + }]) + assert error.value.error_dict == {u'records': [u'"EGGS"? Yeeeeccch!']} + + @pytest.mark.ckan_config('ckan.plugins', 'datastore') + def test_upsert_trigger_exception(self, app): ds = factories.Dataset() - helpers.call_action( - u'datastore_function_create', - name=u'spamonly_trigger', - rettype=u'trigger', - definition=u''' + helpers.call_action(u'datastore_function_create', + name=u'spamonly_trigger', + rettype=u'trigger', + definition=u''' BEGIN IF NEW.spam != 'spam' THEN RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam; END IF; RETURN NEW; END;''') - app = self._get_test_app() with app.flask_app.test_request_context(): - res = helpers.call_action( - u'datastore_create', - resource={u'package_id': ds['id']}, - fields=[{u'id': u'spam', u'type': u'text'}], - triggers=[{u'function': u'spamonly_trigger'}]) - try: - helpers.call_action( - u'datastore_upsert', - method=u'insert', - resource_id=res['resource_id'], - records=[{u'spam': u'spam'}, {u'spam': u'BEANS'}]) - except ValidationError as ve: - assert_equal( - ve.error_dict, - {u'records':[ - u'"BEANS"? Yeeeeccch!']}) - else: - assert 0, u'no validation error' + res = helpers.call_action(u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{ + u'id': u'spam', + u'type': u'text' + }], + triggers=[{ + u'function': + u'spamonly_trigger' + }]) + with pytest.raises(ValidationError) as error: + helpers.call_action(u'datastore_upsert', + method=u'insert', + resource_id=res['resource_id'], + records=[{ + u'spam': u'spam' + }, { + u'spam': u'BEANS' + }]) + assert error.value.error_dict == { + u'records': [u'"BEANS"? Yeeeeccch!'] + }