Add custom metadata in organization

udata/core/dataset/models.py

@@ -552,30 +552,29 @@ def clean(self):
             self.frequency = LEGACY_FREQUENCIES[self.frequency]
 
         for key, value in self.extras.items():
-            if 'custom' in key:
-                if not self.organization:
-                    raise MongoEngineValidationError(
-                        'Custom metadatas are only accessible to dataset owned by on organization.')
-                custom_meta = key.split(':')[1]
-                org_custom = self.organization.extras.get('custom')
-                if not org_custom:
-                    raise MongoEngineValidationError(
-                        'Dataset\'s organization does not have any custom metadata.')
-                custom_present = False
-                custom_type = None
-                for custom in org_custom:
-                    if custom['title'] == custom_meta:
-                        custom_present = True
-                        custom_type = custom['type']
-                        break
-                if not custom_present:
-                    raise MongoEngineValidationError(
-                        'Dataset\'s organization did not define the requested custom metadata.')
-                if custom_type == 'choice':
-                    custom_type = 'list'
-                if not isinstance(value, locate(custom_type)):
-                    raise MongoEngineValidationError(
-                        'Custom metadata is not of the right type.')
+            if not key.startswith('custom:'):
+                continue
+            if not self.organization:
+                raise MongoEngineValidationError(
+                    'Custom metadatas are only accessible to dataset owned by on organization.')
+            custom_meta = key.split(':')[1]
+            org_custom = self.organization.extras.get('custom', [])
+            custom_present = False
+            for custom in org_custom:
+                if custom['title'] != custom_meta:
+                    continue
+                custom_present = True
+                if custom['type'] == 'choice':
+                    if value not in custom['choices']:
+                        raise MongoEngineValidationError(
+                            'Custom metadata choice is not defined by organization.')
+                else:
+                    if not isinstance(value, locate(custom['type'])):
+                        raise MongoEngineValidationError(
+                            'Custom metadata is not of the right type.')
+            if not custom_present:
+                raise MongoEngineValidationError(
+                    'Dataset\'s organization did not define the requested custom metadata.')
 
     def url_for(self, *args, **kwargs):
         return endpoint_for('datasets.show', 'api.dataset', dataset=self, *args, **kwargs)

udata/core/organization/apiv2.py

@@ -43,7 +43,7 @@ class OrganizationExtrasAPI(API):
     def get(self, org):
         '''Get an organization extras given its identifier'''
         if org.deleted:
-            apiv2.abort(410, 'Dataset has been deleted')
+            apiv2.abort(410, 'Organization has been deleted')
         return org.extras
 
     @apiv2.secure
@@ -79,10 +79,10 @@ def delete(self, org):
         if org.deleted:
             apiv2.abort(410, 'Organization has been deleted')
         EditOrganizationPermission(org).test()
-        try:
-            for key in data:
+        for key in data:
+            try:
                 del org.extras[key]
-        except KeyError:
-            apiv2.abort(404, 'Key not found in existing extras')
+            except KeyError:
+                pass
         org.save()
         return org.extras, 204

udata/models/extras_fields.py

@@ -71,16 +71,32 @@ def validate(self, values):
 
         errors = {}
 
-        expected_keys = ["title", "description", "type", "choices"]
+        mandatory_keys = ["title", "description", "type"]
+        optional_keys = ["choices"]
         valid_types = ["str", "int", "float", "bool", "datetime", "date", "choice"]
 
         for elem in values.get('custom', []):
-            # Check if the dictionary contains the expected keys and only them
-            if all(key in expected_keys for key in elem.keys()):
+            mandatory_keys_set = set(mandatory_keys)
+            optional_keys_set = set(optional_keys)
+            dict_keys_set = set(elem.keys())
+            # Check if all mandatory keys are in the dictionary
+            if not mandatory_keys_set.issubset(dict_keys_set):
+                errors['custom'] = 'The dictionary does not contain the mandatory keys.'
+            # Combine mandatory and optional keys
+            all_allowed_keys = mandatory_keys_set.union(optional_keys_set)
+            # Check if there are any keys in the dictionary that are neither mandatory nor optional
+            extra_keys = dict_keys_set - all_allowed_keys
+            # If there are no extra keys, the dictionary is valid
+            if len(extra_keys) == 0:
                 if elem.get("type") not in valid_types:
-                    errors['type'] = 'Value should be one of: {types}'.format(types=valid_types)
+                    errors['type'] = ('Type \'{type}\' of \'{title}\' should be one of: {types}'
+                                      .format(type=elem.get("type"), title=elem.get("title"), types=valid_types))
             else:
-                errors['custom'] = 'The dictionary does not contain the expected keys or contains extra keys.'
+                errors['custom'] = 'The dictionary does contains extra keys than allowed ones.'
+
+            # If the "choices" key exists, check that it's not an empty list
+            if "choices" in elem and not elem["choices"]:
+                errors['custom'] = 'The \'choices\' keys cannot be an empty list.'
 
         if errors:
-            self.error('Unknown badges types', errors=errors)
+            self.error('Custom extras error', errors=errors)

udata/settings.py

@@ -74,7 +74,7 @@ class Defaults(object):
 
     # Flask security settings
 
-    SESSION_COOKIE_SECURE = False
+    SESSION_COOKIE_SECURE = True
     SESSION_COOKIE_SAMESITE = None  # Can be set to 'Lax' or 'Strict'. See https://flask.palletsprojects.com/en/2.3.x/security/#security-cookie
 
     # Flask-Security-Too settings

udata/tests/apiv2/test_datasets.py

@@ -298,7 +298,8 @@ def test_dataset_custom_extras_choices(self):
                 {
                     "title": "color",
                     "description": "the colors of the dataset (Hex code)",
-                    "type": "choice"
+                    "type": "choice",
+                    "choices": ["yellow", "blue"]
                 }
             ]
         }
@@ -310,15 +311,15 @@ def test_dataset_custom_extras_choices(self):
         }
         response = self.put(url_for('apiv2.dataset_extras', dataset=dataset), data)
         self.assert400(response)
-        assert 'Custom metadata is not of the right type' in response.json['message']
+        assert 'Custom metadata choice is not defined by organization' in response.json['message']
 
         data = {
-            'custom:color': ['eea393', 'ff5733']
+            'custom:color': 'yellow'
         }
         response = self.put(url_for('apiv2.dataset_extras', dataset=dataset), data)
         self.assert200(response)
         dataset.reload()
-        assert dataset.extras['custom:color'] == ['eea393', 'ff5733']
+        assert dataset.extras['custom:color'] == 'yellow'
 
 
 class DatasetResourceExtrasAPITest(APITestCase):