Merge pull request #4407 from ckan/4407-datastore-upsert-dry-run

datastore_upsert: dry_run parameter

ckanext/datastore/backend/postgres.py

@@ -1414,7 +1414,10 @@ def upsert(context, data_dict):
         context['connection'].execute(
             u'SET LOCAL statement_timeout TO {0}'.format(timeout))
         upsert_data(context, data_dict)
-        trans.commit()
+        if data_dict.get(u'dry_run', False):
+            trans.rollback()
+        else:
+            trans.commit()
         return _unrename_json_field(data_dict)
     except IntegrityError as e:
         if e.orig.pgcode == _PG_ERR_CODE['unique_violation']:

ckanext/datastore/logic/action.py

@@ -229,6 +229,9 @@ def datastore_upsert(context, data_dict):
                    Possible options are: upsert, insert, update
                    (optional, default: upsert)
     :type method: string
+    :param dry_run: set to True to abort transaction instead of committing,
+                    e.g. to check for validation or type errors.
+    :type dry_run: bool (optional, default: False)
 
     **Results:**
 

ckanext/datastore/logic/schema.py

@@ -135,6 +135,7 @@ def datastore_upsert_schema():
         'id': [ignore_missing],
         'method': [ignore_missing, text_type, OneOf(
             ['upsert', 'insert', 'update'])],
+        'dry_run': [ignore_missing, boolean_validator],
         '__junk': [empty],
         '__before': [rename('id', 'resource_id')]
     }

ckanext/datastore/tests/test_upsert.py

@@ -12,6 +12,7 @@
 import ckan.tests.legacy as tests
 import ckan.tests.helpers as helpers
 import ckan.tests.factories as factories
+from ckan.plugins.toolkit import ValidationError
 
 from ckan.common import config
 
@@ -22,7 +23,7 @@
 assert_equal = nose.tools.assert_equal
 
 
-class TestDatastoreUpsertNewTests(DatastoreFunctionalTestBase):
+class TestDatastoreUpsert(DatastoreFunctionalTestBase):
     def test_upsert_doesnt_crash_with_json_field(self):
         resource = factories.Resource()
         data = {
@@ -67,15 +68,82 @@ def test_upsert_doesnt_crash_with_json_field_with_string_value(self):
         }
         helpers.call_action('datastore_upsert', **data)
 
-
-class TestDatastoreUpsert(DatastoreLegacyTestBase):
+    def test_dry_run(self):
+        ds = factories.Dataset()
+        table = helpers.call_action(
+            u'datastore_create',
+            resource={u'package_id': ds['id']},
+            fields=[{u'id': u'spam', u'type': u'text'}],
+            primary_key=u'spam')
+        helpers.call_action(
+            u'datastore_upsert',
+            resource_id=table['resource_id'],
+            records=[{u'spam': u'SPAM'}, {u'spam': u'EGGS'}],
+            dry_run=True)
+        result = helpers.call_action(
+            u'datastore_search',
+            resource_id=table['resource_id'])
+        assert_equal(result['records'], [])
+
+    def test_dry_run_type_error(self):
+        ds = factories.Dataset()
+        table = helpers.call_action(
+            u'datastore_create',
+            resource={u'package_id': ds['id']},
+            fields=[{u'id': u'spam', u'type': u'numeric'}],
+            primary_key=u'spam')
+        try:
+            helpers.call_action(
+                u'datastore_upsert',
+                resource_id=table['resource_id'],
+                records=[{u'spam': u'SPAM'}, {u'spam': u'EGGS'}],
+                dry_run=True)
+        except ValidationError as ve:
+            assert_equal(ve.error_dict['records'],
+                [u'invalid input syntax for type numeric: "SPAM"'])
+        else:
+            assert 0, 'error not raised'
+
+    def test_dry_run_trigger_error(self):
+        ds = factories.Dataset()
+        helpers.call_action(
+            u'datastore_function_create',
+            name=u'spamexception_trigger',
+            rettype=u'trigger',
+            definition=u'''
+                BEGIN
+                IF NEW.spam != 'spam' THEN
+                    RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam;
+                END IF;
+                RETURN NEW;
+                END;''')
+        table = helpers.call_action(
+            u'datastore_create',
+            resource={u'package_id': ds['id']},
+            fields=[{u'id': u'spam', u'type': u'text'}],
+            primary_key=u'spam',
+            triggers=[{u'function': u'spamexception_trigger'}])
+        try:
+            helpers.call_action(
+                u'datastore_upsert',
+                resource_id=table['resource_id'],
+                records=[{u'spam': u'EGGS'}],
+                dry_run=True)
+        except ValidationError as ve:
+            assert_equal(ve.error_dict['records'],
+                [u'"EGGS"? Yeeeeccch!'])
+        else:
+            assert 0, 'error not raised'
+
+
+class TestDatastoreUpsertLegacyTests(DatastoreLegacyTestBase):
     sysadmin_user = None
     normal_user = None
 
     @classmethod
     def setup_class(cls):
         cls.app = helpers._get_test_app()
-        super(TestDatastoreUpsert, cls).setup_class()
+        super(TestDatastoreUpsertLegacyTests, cls).setup_class()
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
@@ -313,14 +381,14 @@ def test_upsert_works_with_empty_list_in_json_field(self):
 
 
 
-class TestDatastoreInsert(DatastoreLegacyTestBase):
+class TestDatastoreInsertLegacyTests(DatastoreLegacyTestBase):
     sysadmin_user = None
     normal_user = None
 
     @classmethod
     def setup_class(cls):
         cls.app = helpers._get_test_app()
-        super(TestDatastoreInsert, cls).setup_class()
+        super(TestDatastoreInsertLegacyTests, cls).setup_class()
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
@@ -408,14 +476,14 @@ def test_insert_basic(self):
         assert results.rowcount == 3
 
 
-class TestDatastoreUpdate(DatastoreLegacyTestBase):
+class TestDatastoreUpdateLegacyTests(DatastoreLegacyTestBase):
     sysadmin_user = None
     normal_user = None
 
     @classmethod
     def setup_class(cls):
         cls.app = helpers._get_test_app()
-        super(TestDatastoreUpdate, cls).setup_class()
+        super(TestDatastoreUpdateLegacyTests, cls).setup_class()
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')