Storage fields validators

Makefile

@@ -20,6 +20,7 @@ SED := $(shell which gsed sed | head -n1)
 CKAN_CLI := $(shell which ckan | head -n1)
 
 TEST_INI_PATH := ./test.ini
+TEST_PATH :=
 SENTINELS := .make-status
 
 PYTHON_VERSION := $(shell $(PYTHON) -c 'import sys; print(sys.version_info[0])')
@@ -252,7 +253,8 @@ $(SENTINELS)/tests-passed: $(SENTINELS)/test-setup $(shell find $(PACKAGE_DIR) -
 		--ckan-ini=$(TEST_INI_PATH) \
 		--doctest-modules \
 		--ignore $(PACKAGE_DIR)/cli.py  \
-		$(PACKAGE_DIR)
+		-s \
+		$(PACKAGE_DIR)/$(TEST_PATH)
 	@touch $@
 
 # Help related variables and targets

ckanext/external_storage/plugin.py

@@ -4,19 +4,95 @@
 from ckanext.authz_service.authzzie import Authzzie
 from ckanext.authz_service.interfaces import IAuthorizationBindings
 
-from . import actions, authz, helpers
+from . import actions, authz, helpers, validators
 from .blueprints import blueprint
 from .download_handler import download_handler
 from .interfaces import IResourceDownloadHandler
 
 
-class ExternalStoragePlugin(plugins.SingletonPlugin):
+class ExternalStoragePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm):
     plugins.implements(plugins.IConfigurer)
     plugins.implements(plugins.ITemplateHelpers)
     plugins.implements(plugins.IBlueprint)
     plugins.implements(plugins.IActions)
     plugins.implements(IAuthorizationBindings)
     plugins.implements(IResourceDownloadHandler, inherit=True)
+    plugins.implements(plugins.IValidators)
+    plugins.implements(plugins.IDatasetForm)
+
+    # IDatasetForm
+    def create_package_schema(self):
+        # let's grab the default schema in our plugin
+        schema = super(ExternalStoragePlugin, self).create_package_schema()
+
+        schema['resources'].update({
+            'url_type': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_validator('upload_has_sha256'),
+                toolkit.get_validator('upload_has_size'),
+                toolkit.get_validator('upload_has_lfs_prefix')
+                ],
+            'sha256': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_validator('valid_sha256')
+            ],
+            'size': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_validator('is_positive_integer'),
+            ],
+            'lfs_prefix': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_validator('valid_lfs_prefix'),
+            ]
+        })
+
+        return schema
+
+    def update_package_schema(self):
+        schema = super(ExternalStoragePlugin, self).update_package_schema()
+
+        schema['resources'].update({
+            'url_type': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_validator('upload_has_sha256'),
+                toolkit.get_validator('upload_has_size'),
+                toolkit.get_validator('upload_has_lfs_prefix')
+                ],
+            'sha256': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_validator('valid_sha256')
+            ],
+            'size': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_validator('is_positive_integer'),
+            ],
+            'lfs_prefix': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_validator('valid_lfs_prefix'),
+            ]
+        })
+
+        return schema
+
+    def is_fallback(self):
+        # Return True to register this plugin as the default handler for
+        # package types not handled by any other IDatasetForm plugin.
+        return True
+
+    def package_types(self):
+        # This plugin doesn't handle any special package types, it just
+        # registers itself as the default (above).
+        return []
+
+    # IValidators
+    def get_validators(self):
+        return {
+            u'upload_has_sha256': validators.upload_has_sha256,
+            u'upload_has_size': validators.upload_has_size,
+            u'upload_has_lfs_prefix': validators.upload_has_lfs_prefix,
+            u'valid_sha256': validators.valid_sha256,
+            u'valid_lfs_prefix': validators.valid_lfs_prefix,
+        }
 
     # IConfigurer
 

ckanext/external_storage/tests/test_actions.py

@@ -0,0 +1,135 @@
+import ckan.plugins.toolkit as toolkit
+import pytest
+from ckan.tests import factories
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_validation_error_if_not_sha256():
+    with pytest.raises(toolkit.ValidationError):
+        factories.Dataset(
+            resources=[
+                {
+                    'url': '/my/file.csv',
+                    'url_type': 'upload',
+                    'size': 12345,
+                    'lfs_prefix': 'lfs/prefix'
+                }
+            ]
+        )
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_validation_error_if_not_size_on_uploads():
+    with pytest.raises(toolkit.ValidationError):
+        factories.Dataset(
+            resources=[
+                {
+                    'url': '/my/file.csv',
+                    'url_type': 'upload',
+                    'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919',
+                    'lfs_prefix': 'lfs/prefix'
+                }
+            ]
+        )
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_validation_error_if_not_lfs_prefix_on_uploads():
+    with pytest.raises(toolkit.ValidationError):
+        factories.Dataset(
+            resources=[
+                {
+                    'url': '/my/file.csv',
+                    'url_type': 'upload',
+                    'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919',
+                    'size': 123456
+                }
+            ]
+        )
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_no_validation_error_if_not_upload():
+    factories.Dataset(
+            resources=[{'url': 'https://www.example.com', 'url_type': ''}]
+        )
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_no_validation_error_if_all_fields_are_set():
+    dataset = factories.Dataset(
+        resources=[
+            {
+                'url': '/my/file.csv',
+                'url_type': 'upload',
+                'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919',
+                'size': 12345,
+                'lfs_prefix': 'lfs/prefix'
+            }
+        ]
+    )
+
+    assert dataset['resources'][0]['sha256'] == 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919'
+    assert dataset['resources'][0]['size'] == 12345
+    assert dataset['resources'][0]['lfs_prefix'] == 'lfs/prefix'
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_validation_error_if_wrong_sha256():
+    with pytest.raises(toolkit.ValidationError):
+        factories.Dataset(
+            resources=[
+                {
+                    'url': '/my/file.csv',
+                    'url_type': 'upload',
+                    'sha256': 'wrong_sha256',
+                    'size': 123456,
+                    'lfs_prefix': 'lfs/prefix'
+                }
+            ]
+        )
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_validation_error_if_size_not_positive_integer():
+    with pytest.raises(toolkit.ValidationError):
+        factories.Dataset(
+            resources=[
+                {
+                    'url': '/my/file.csv',
+                    'url_type': 'upload',
+                    'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919',
+                    'size': -12,
+                    'lfs_prefix': 'lfs/prefix'
+                }
+            ]
+        )
+
+    with pytest.raises(toolkit.ValidationError):
+        factories.Dataset(
+            resources=[
+                {
+                    'url': '/my/file.csv',
+                    'url_type': 'upload',
+                    'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919',
+                    'size': 0,
+                    'lfs_prefix': 'lfs/prefix'
+                }
+            ]
+        )
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_validation_error_if_empty_lfs_prefix():
+    with pytest.raises(toolkit.ValidationError):
+        factories.Dataset(
+            resources=[
+                {
+                    'url': '/my/file.csv',
+                    'url_type': 'upload',
+                    'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919',
+                    'size': 123456,
+                    'lfs_prefix': ''
+                }
+            ]
+        )

ckanext/external_storage/tests/test_validators.py

@@ -0,0 +1,96 @@
+import pytest
+from ckan.plugins.toolkit import Invalid
+
+from ckanext.external_storage import validators
+
+
+def test_upload_has_sha256():
+    key = ('resources', 0, 'url_type')
+    flattened_data = {
+        (u'resources', 0, u'url_type'): u'upload',
+        (u'resources', 0, u'url'): u'/my/file.csv',
+        (u'resources', 0, u'sha256'): u'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919'
+    }
+    validators.upload_has_sha256(key, flattened_data, {}, {})
+
+    with pytest.raises(Invalid):
+        flattened_data = {
+            (u'resources', 0, u'url_type'): u'upload',
+            (u'resources', 0, u'url'): u'/my/file.csv',
+        }
+        validators.upload_has_sha256(key, flattened_data, {}, {})
+
+
+def test_upload_has_size():
+    key = ('resources', 0, 'url_type')
+    flattened_data = {
+        (u'resources', 0, u'url_type'): u'upload',
+        (u'resources', 0, u'url'): u'/my/file.csv',
+        (u'resources', 0, u'size'): 123456
+    }
+    validators.upload_has_size(key, flattened_data, {}, {})
+
+    with pytest.raises(Invalid):
+        flattened_data = {
+            (u'resources', 0, u'url_type'): u'upload',
+            (u'resources', 0, u'url'): u'/my/file.csv',
+        }
+        validators.upload_has_size(key, flattened_data, {}, {})
+
+
+def test_upload_has_lfs_prefix():
+    key = ('resources', 0, 'url_type')
+    flattened_data = {
+        (u'resources', 0, u'url_type'): u'upload',
+        (u'resources', 0, u'url'): u'/my/file.csv',
+        (u'resources', 0, u'lfs_prefix'): u'lfs/prefix'
+    }
+    validators.upload_has_lfs_prefix(key, flattened_data, {}, {})
+
+    with pytest.raises(Invalid):
+        flattened_data = {
+            (u'resources', 0, u'url_type'): u'upload',
+            (u'resources', 0, u'url'): u'/my/file.csv'
+        }
+        validators.upload_has_lfs_prefix(key, flattened_data, {}, {})
+
+
+def test_valid_sha256():
+    validators.valid_sha256('cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919')
+
+    with pytest.raises(Invalid):
+        validators.valid_sha256('wrong_sha256')
+
+
+def test_valid_lfs_prefix():
+    validators.valid_lfs_prefix("lfs/prefix")
+
+    with pytest.raises(Invalid):
+        validators.valid_lfs_prefix("")
+
+
+def test_sha256_doesnt_raise_if_not_upload():
+    key = ('resources', 0, 'url_type')
+    flattened_data = {
+        (u'resources', 0, u'url_type'): u'',
+        (u'resources', 0, u'url'): u'https://www.google.com',
+    }
+    validators.upload_has_sha256(key, flattened_data, {}, {})
+
+
+def test_size_doesnt_raise_if_not_upload():
+    key = ('resources', 0, 'url_type')
+    flattened_data = {
+        (u'resources', 0, u'url_type'): u'',
+        (u'resources', 0, u'url'): u'https://www.google.com',
+    }
+    validators.upload_has_size(key, flattened_data, {}, {})
+
+
+def test_lfs_prefix_doesnt_raise_if_not_upload():
+    key = ('resources', 0, 'url_type')
+    flattened_data = {
+        (u'resources', 0, u'url_type'): u'',
+        (u'resources', 0, u'url'): u'https://www.google.com',
+    }
+    validators.upload_has_lfs_prefix(key, flattened_data, {}, {})