OpenConceptLab/ocl_issues#1415 Implement automated import scripts for…

… FHIR HL7 content

README.md

@@ -74,7 +74,7 @@ Run api with:
 
 In order to import FHIR resources run:
 
-`docker-compose run --no-deps --rm -v $(pwd)/../oclfhir-tests/definitions.json:/fhir api python tools/fhir_import.py -s /fhir -t http://api:8000/orgs/test -s 891b4b17feab99f3ff7e5b5d04ccc5da7aa96da6 -c http://api:8000/orgs/test -r CodeSystem`
+`docker-compose run --no-deps --rm -v $(pwd)/../fhir_imports:/fhir api python tools/import.py -f /fhir -t http://api:8000/orgs/test -s 891b4b17feab99f3ff7e5b5d04ccc5da7aa96da6 -c http://api:8000/orgs/test
 
 For help run:
 

core/code_systems/serializers.py

@@ -1,13 +1,15 @@
+import ast
 import logging
 from collections import OrderedDict
 
 from rest_framework import serializers
 from rest_framework.fields import CharField, BooleanField, IntegerField, SerializerMethodField, ChoiceField, \
-    DateTimeField, JSONField
+    DateTimeField
 
 from core import settings
 from core.code_systems.constants import RESOURCE_TYPE
 from core.common.constants import HEAD
+from core.common.fhir_helpers import delete_empty_fields
 from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer
 from core.concepts.models import Concept, ConceptName
 from core.concepts.serializers import ConceptDetailSerializer
@@ -128,7 +130,8 @@ def to_internal_value(self, data):
                 break
 
         if not found:
-            ret['names'].append({'name': data['display'], 'locale': settings.DEFAULT_LOCALE, 'locale_preferred': True})
+            ret['names'].append({'name': data.get('display', data.get('code', None)), 'locale': settings.DEFAULT_LOCALE,
+                                 'locale_preferred': True})
 
         return ret
 
@@ -155,6 +158,17 @@ def to_representation(self, value):
             limit = 1000
         return CodeSystemConceptSerializer(value.concepts.order_by('id')[:limit], many=True).data
 
+class TextField(ReadSerializerMixin, serializers.Serializer):
+    status = ChoiceField(choices=['generated', 'extensions', 'additional', 'empty'], required=True)
+    div = CharField(required=True)
+
+    def to_internal_value(self, data):
+        validated_data = super().to_internal_value(data)
+        return dict(validated_data)
+
+    def to_representation(self, instance):
+        obj = ast.literal_eval(instance)
+        return super().to_representation(obj)
 
 class CodeSystemDetailSerializer(serializers.ModelSerializer):
     resourceType = SerializerMethodField(method_name='get_resource_type')
@@ -179,7 +193,7 @@ class CodeSystemDetailSerializer(serializers.ModelSerializer):
     collectionReference = CharField(source='collection_reference', required=False)
     hierarchyMeaning = CharField(source='hierarchy_meaning', required=False)
     revisionDate = DateTimeField(source='revision_date', required=False)
-    text = JSONField(required=False)
+    text = TextField(required=False)
 
     class Meta:
         model = Source
@@ -233,6 +247,7 @@ def get_meta(obj):
     def to_representation(self, instance):
         try:
             rep = super().to_representation(instance)
+            delete_empty_fields(rep)
             IdentifierSerializer.include_ocl_identifier(instance.uri, RESOURCE_TYPE, rep)
         except (Exception, ):
             msg = f'Failed to represent "{instance.uri}" as {RESOURCE_TYPE}'
@@ -278,9 +293,9 @@ def create(self, validated_data):
         source.version = '0.1' if version == HEAD else version
 
         source.id = None  # pylint: disable=invalid-name
-        errors = Source.persist_new_version(source, user)
+        # Make it synchronous for now so that the list of concepts is included in the response
+        errors = Source.persist_new_version(source, user, sync=True)
         self._errors.update(errors)
-
         return source
 
     def update(self, instance, validated_data):
@@ -323,7 +338,7 @@ def update(self, instance, validated_data):
         source.version = source_version
         source.released = source_released
         source.id = None
-        errors = Source.persist_new_version(source, user)
+        # Make it synchronous for now so that the list of concepts is included in the response
+        errors = Source.persist_new_version(source, user, sync=True)
         self._errors.update(errors)
-
         return source

core/common/fhir_helpers.py

@@ -28,3 +28,10 @@ def translate_fhir_query(fhir_query_fields, query_params, queryset):
             kwargs = {query_field: query_value}
             queryset = queryset.filter(**kwargs)
     return queryset
+
+def delete_empty_fields(obj):
+    for field in list(obj.keys()):
+        if obj[field] is None or obj[field] == {} or obj[field] == []:
+            del obj[field]
+        elif isinstance(obj[field], dict):
+            delete_empty_fields(obj[field])

core/common/tests.py

@@ -18,7 +18,7 @@
 from moto import mock_s3
 from requests.auth import HTTPBasicAuth
 from rest_framework.exceptions import ValidationError
-from rest_framework.test import APITestCase
+from rest_framework.test import APITestCase, APITransactionTestCase
 
 from core.collections.models import CollectionReference
 from core.common.constants import HEAD
@@ -205,6 +205,14 @@ def create_lookup_concept_classes(user=None, org=None):
             names=[ConceptNameFactory.build(name="English")]
         )
 
+class OCLAPITransactionTestCase(APITransactionTestCase, BaseTestCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        call_command("loaddata", "core/fixtures/base_entities.yaml")
+        call_command("loaddata", "core/fixtures/toggles.json")
+        org = Organization.objects.get(id=1)
+        org.members.add(1)
 
 class OCLAPITestCase(APITestCase, BaseTestCase):
     @classmethod

core/concept_maps/serializers.py

@@ -5,6 +5,7 @@
 from rest_framework.fields import CharField, SerializerMethodField, \
     DateTimeField
 
+from core.common.fhir_helpers import delete_empty_fields
 from core.concept_maps.constants import RESOURCE_TYPE
 from core.common.constants import HEAD
 from core.common.serializers import StatusField, IdentifierSerializer
@@ -137,6 +138,7 @@ def get_meta(obj):
     def to_representation(self, instance):
         try:
             rep = super().to_representation(instance)
+            delete_empty_fields(rep)
             IdentifierSerializer.include_ocl_identifier(instance.uri, RESOURCE_TYPE, rep)
         except (Exception, ):
             msg = f'Failed to represent "{instance.uri}" as {RESOURCE_TYPE}'

core/integration_tests/test_utils.py

@@ -0,0 +1,35 @@
+import json
+import os
+
+from jsonpath_ng import parse
+
+
+def load_json(file, parent_dir=None):
+    if parent_dir:
+        if parent_dir.startswith(os.path.pathsep):
+            file_path = parent_dir
+        else:
+            module_dir = os.path.dirname(__file__)  # get current directory
+            file_path = os.path.join(module_dir, parent_dir)
+    else:
+        file_path = os.path.dirname(__file__)
+    file_path = os.path.join(file_path, file)
+    with open(file_path) as f:
+        json_file = json.load(f)
+    return json_file
+
+
+def update_json(json_input, path, value):
+    jsonpath_expr = parse(path)
+    jsonpath_expr.update_or_create(json_input, value)
+
+
+def find(json_input, path):
+    jsonpath_expr = parse(path)
+    return jsonpath_expr.find(json_input)
+
+
+def ignore_json_paths(self, json_input, json_response, paths):
+    for path in paths:
+        self.update(json_input, path, None)
+        self.update(json_response, path, None)

core/settings.py

@@ -34,7 +34,7 @@
 SECRET_KEY = '=q1%fd62$x!35xzzlc3lix3g!s&!2%-1d@5a=rm!n4lu74&6)p'
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = os.environ.get('DEBUG') == 'TRUE'
+DEBUG = False #os.environ.get('DEBUG') == 'TRUE'
 
 ALLOWED_HOSTS = ['*']
 

core/urls.py

@@ -91,3 +91,6 @@
     path('manage/bulkimport/', BulkImportView.as_view(), name='bulk_import_urls'),
     path('toggles/', include('core.toggles.urls'), name='toggles'),
 ]
+
+handler500 = 'rest_framework.exceptions.server_error'
+handler400 = 'rest_framework.exceptions.bad_request'

core/value_sets/serializers.py

@@ -1,14 +1,15 @@
 import logging
 
 from rest_framework import serializers
-from rest_framework.fields import CharField, DateField, SerializerMethodField, ChoiceField, DateTimeField, JSONField, \
+from rest_framework.fields import CharField, DateField, SerializerMethodField, ChoiceField, DateTimeField, \
     BooleanField, ListField, URLField
 
-from core.code_systems.serializers import CodeSystemConceptSerializer
+from core.code_systems.serializers import CodeSystemConceptSerializer, TextField
 from core.collections.models import Collection, Expansion
 from core.collections.parsers import CollectionReferenceParser
 from core.collections.serializers import CollectionCreateOrUpdateSerializer
 from core.common.constants import HEAD
+from core.common.fhir_helpers import delete_empty_fields
 from core.common.serializers import StatusField, IdentifierSerializer, ReadSerializerMixin
 from core.orgs.models import Organization
 from core.parameters.serializers import ParametersSerializer
@@ -155,7 +156,7 @@ class ValueSetDetailSerializer(serializers.ModelSerializer):
     identifier = IdentifierSerializer(many=True, required=False)
     date = DateTimeField(source='revision_date', required=False)
     compose = ComposeValueSetField(source='*', required=False)
-    text = JSONField(required=False)
+    text = TextField(required=False)
 
     class Meta:
         model = Collection
@@ -189,7 +190,8 @@ def create(self, validated_data):
         collection.id = None  # pylint: disable=invalid-name
         collection.version = collection_version
         collection.expansion_uri = None
-        errors = Collection.persist_new_version(collection, user)
+        # Persist synchronously in order to return complete results in the reponse
+        errors = Collection.persist_new_version(collection, user, sync=True)
         self._errors.update(errors)
 
         return collection
@@ -238,14 +240,16 @@ def update(self, instance, validated_data):
         collection.released = collection_released
         collection.id = None
         collection.expansion_uri = None
-        errors = Collection.persist_new_version(collection, user)
+        # Persist synchronously in order to include complete results in the response
+        errors = Collection.persist_new_version(collection, user, sync=True)
         self._errors.update(errors)
 
         return collection
 
     def to_representation(self, instance):
         try:
             rep = super().to_representation(instance)
+            delete_empty_fields(rep)
             IdentifierSerializer.include_ocl_identifier(instance.uri, RESOURCE_TYPE, rep)
         except (Exception, ):
             msg = f'Failed to represent "{instance.uri}" as {RESOURCE_TYPE}'

core/value_sets/tests.py

@@ -52,7 +52,7 @@ def test_public_can_find_globally_without_compose(self):
 
         self.assertEqual(
             resource['identifier'][0]['value'], f'/orgs/{self.org.mnemonic}/ValueSet/{self.collection.mnemonic}/')
-        self.assertEqual(resource['compose'], None)
+        self.assertFalse('compose' in resource)
 
     def test_public_can_find_globally(self):
         self.collection.add_references([
@@ -118,7 +118,7 @@ def test_can_create_empty(self):
         resource = response.data
         self.assertEqual(resource['version'], 'v1')
         self.assertEqual(resource['identifier'][0]['value'], f'/users/{self.user.username}/ValueSet/c2/')
-        self.assertEqual(resource['compose'], None)
+        self.assertFalse('compose' in resource)
 
     def test_can_create_with_compose(self):
         response = self.client.post(
@@ -258,7 +258,7 @@ def test_can_update_empty(self):
         resource = response.data
         self.assertEqual(resource['version'], 'v2')
         self.assertEqual(resource['identifier'][0]['value'], f'/orgs/{self.org.mnemonic}/ValueSet/c1/')
-        self.assertEqual(resource['compose'], None)
+        self.assertFalse('compose' in resource)
 
     def test_update_with_compose(self):
         self.collection.add_references([

requirements.txt

@@ -36,3 +36,4 @@ django-dirtyfields==1.9.2
 jsonpath-ng==1.5.3
 mozilla-django-oidc==3.0.0
 django-celery-beat==2.5.0
+jsondiff==2.0.0