feat(dataset): dats_file field changed from string to json

chord_metadata_service/chord/api_views.py

@@ -1,5 +1,5 @@
-import logging
 import json
+import logging
 
 from asgiref.sync import async_to_sync, sync_to_async
 
@@ -84,7 +84,7 @@ def dats(self, _request, *_args, **_kwargs):
         Return the DATS file as a JSON response or an error if not found.
         """
         dataset = self.get_object()
-        return Response(json.loads(dataset.dats_file))
+        return Response(dataset.dats_file)
 
     @action(detail=True, methods=["get"])
     def resources(self, _request, *_args, **_kwargs):
@@ -107,6 +107,24 @@ async def destroy(self, request, *args, **kwargs):
         logger.info(f"Cleanup: removed {n_removed} objects in total")
         return Response(status=status.HTTP_204_NO_CONTENT)
 
+    def create(self, request, *args, **kwargs):
+        """
+        Creates a Dataset.
+        If the request's dats_file is a string, it will be parsed to JSON.
+        """
+        dats_file = request.data.get('dats_file')
+        if isinstance(dats_file, str):
+            try:
+                dats_file = json.loads(dats_file)
+            except json.JSONDecodeError:
+                error_msg = ("Submitted dataset.dats_file data is not a valid JSON string. "
+                             "Make sure the string value is JSON compatible, or submit dats_file as a JSON object.")
+                logger.error(error_msg)
+                return Response(error_msg, status.HTTP_400_BAD_REQUEST)
+            # Set dats_file request value to JSON
+            request.data['dats_file'] = dats_file
+        return super().create(request, *args, **kwargs)
+
 
 class ProjectJsonSchemaViewSet(CHORDPublicModelViewSet):
     """

chord_metadata_service/chord/migrations/0007_v6_1_0.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-01-16 19:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('chord', '0006_v4_0_0'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dataset',
+            name='dats_file',
+            field=models.JSONField(blank=True, null=True, help_text='Content of a valid DATS file, in JSON format, that specifies the dataset provenance.'),
+        )
+    ]

chord_metadata_service/chord/models.py

@@ -143,8 +143,9 @@ def resources(self):
                                 help_text="Tags associated with the dataset, which will help in its discovery.")
     version = models.CharField(max_length=200, blank=True, default=version_default,
                                help_text="A release point for the dataset when applicable.")
-    dats_file = models.TextField(blank=True, help_text="Content of a valid DATS file, in JSON format, that specifies"
-                                 " the dataset provenance.")
+    dats_file = models.JSONField(blank=True, null=True,
+                                 help_text="Content of a valid DATS file, in JSON format, "
+                                           "that specifies the dataset provenance.")
     extra_properties = models.JSONField(blank=True, null=True,
                                         help_text="Extra properties that do not fit in the previous "
                                         "specified attributes.")

chord_metadata_service/chord/tests/test_api.py

@@ -62,7 +62,17 @@ def setUp(self) -> None:
         self.project = r.json()
 
         self.valid_payloads = [
-            valid_dataset_1(self.project["identifier"])
+            valid_dataset_1(self.project["identifier"]),
+            {
+                **valid_dataset_1(self.project["identifier"]),
+                "title": "Dataset 2",
+                "dats_file": {},  # Valid dats_file JSON object
+            },
+            {
+                **valid_dataset_1(self.project["identifier"]),
+                "title": "Dataset 3",
+                "dats_file": "{}",  # Valid dats_file JSON string
+            }
         ]
 
         self.dats_valid_payload = dats_dataset(self.project["identifier"], VALID_DATS_CREATORS)
@@ -78,7 +88,12 @@ def setUp(self) -> None:
                 "title": "Dataset 1",
                 "description": "Test Dataset",
                 "project": None
-            }
+            },
+            {
+                **valid_dataset_1(self.project["identifier"]),
+                "title": "Dataset 4",
+                "dats_file": "INVALID_JSON_STRING",
+            },
         ]
 
     def test_create_dataset(self):
@@ -95,7 +110,7 @@ def test_create_dataset(self):
             self.assertEqual(Dataset.objects.count(), len(self.valid_payloads))
 
     def test_dats(self):
-        payload = {**self.dats_valid_payload, 'dats_file': json.dumps({})}
+        payload = {**self.dats_valid_payload, 'dats_file': {}}
         r = self.client.post('/api/datasets', data=json.dumps(payload),
                              content_type="application/json")
         r_invalid = self.client.post('/api/datasets', data=json.dumps(self.dats_invalid_payload),
@@ -109,7 +124,7 @@ def test_dats(self):
         url = f'/api/datasets/{dataset_id}/dats'
         response = self.client.get(url)
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertDictEqual(response.data, json.loads(payload['dats_file']))
+        self.assertDictEqual(response.data, payload['dats_file'])
 
     def test_resources(self):
         resource = {

chord_metadata_service/restapi/api_views.py

@@ -1,5 +1,3 @@
-import json
-
 from adrf.decorators import api_view as api_view_async
 from drf_spectacular.utils import extend_schema, inline_serializer
 from django.conf import settings
@@ -329,13 +327,6 @@ def public_dataset(_request):
         "extra_properties", "identifier"
     )
 
-    # convert dats_file json content to dict
-    datasets = [
-        {
-            **d,
-            "dats_file": json.loads(d["dats_file"]) if d["dats_file"] else None
-        } for d in datasets]
-
     return Response({
         "datasets": datasets
     })

chord_metadata_service/restapi/tests/test_api.py

@@ -319,7 +319,7 @@ def setUp(self) -> None:
         project = ch_m.Project.objects.create(title="Test project", description="test description")
         dats_path = os.path.join(os.path.dirname(__file__), "example_dats_provenance.json")
         with open(dats_path) as f:
-            dats_content = f.read()
+            dats_content = json.loads(f.read())
 
         ch_m.Dataset.objects.create(
             title="Dataset 1",