OpenConceptLab/ocl_issues#1399 | names/descriptions migrations to spl…

…it names and descriptions
OpenConceptLab · Jan 11, 2023 · 9d8b0c3 · 9d8b0c3
1 parent 7de739b
commit 9d8b0c3
Show file tree

Hide file tree

Showing 29 changed files with 795 additions and 408 deletions.
diff --git a/core/code_systems/serializers.py b/core/code_systems/serializers.py
@@ -8,7 +8,7 @@
 from core.code_systems.constants import RESOURCE_TYPE
 from core.common.constants import HEAD
 from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer
-from core.concepts.models import Concept, LocalizedText
+from core.concepts.models import Concept, ConceptName
 from core.concepts.serializers import ConceptDetailSerializer
 from core.orgs.models import Organization
 from core.sources.models import Source
@@ -34,7 +34,7 @@ class CodeSystemConceptDesignationSerializer(serializers.ModelSerializer):
     use = CodeSystemConceptDesignationUseSerializer(source='*', required=False)
 
     class Meta:
-        model = LocalizedText
+        model = ConceptName
         fields = ('language', 'value', 'use')
 
 

diff --git a/core/code_systems/tests.py b/core/code_systems/tests.py
@@ -4,7 +4,7 @@
 
 from core.common.tests import OCLTestCase
 from core.concepts.models import Concept
-from core.concepts.tests.factories import ConceptFactory, LocalizedTextFactory
+from core.concepts.tests.factories import ConceptFactory
 from core.orgs.tests.factories import OrganizationFactory
 from core.sources.models import Source
 from core.sources.tests.factories import OrganizationSourceFactory, UserSourceFactory
@@ -21,7 +21,7 @@ def setUp(self):
         self.org_source_v1 = OrganizationSourceFactory.build(
             version='v1', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent)
         Source.persist_new_version(self.org_source_v1, self.org_source.created_by)
-        self.concept_1 = ConceptFactory(parent=self.org_source, names=[LocalizedTextFactory(name="concept_1_name")])
+        self.concept_1 = ConceptFactory(parent=self.org_source, names=1, name__name="concept_1_name")
         self.concept_2 = ConceptFactory(parent=self.org_source)
         self.org_source_v2 = OrganizationSourceFactory.build(
             version='v2', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent)

diff --git a/core/collections/tests/tests.py b/core/collections/tests/tests.py
@@ -17,7 +17,7 @@
 from core.common.tests import OCLTestCase, OCLAPITestCase
 from core.concepts.documents import ConceptDocument
 from core.concepts.models import Concept
-from core.concepts.tests.factories import ConceptFactory, LocalizedTextFactory
+from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory
 from core.mappings.documents import MappingDocument
 from core.mappings.tests.factories import MappingFactory
 from core.orgs.tests.factories import OrganizationFactory
@@ -184,8 +184,8 @@ def test_validate_reference_already_exists(self):
         expansion = ExpansionFactory(collection_version=collection)
         collection.expansion_uri = expansion.uri
         collection.save()
-        ch_locale = LocalizedTextFactory(locale_preferred=True, locale='ch')
-        en_locale = LocalizedTextFactory(locale_preferred=True, locale='en')
+        ch_locale = ConceptNameFactory.build(locale_preferred=True, locale='ch')
+        en_locale = ConceptNameFactory.build(locale_preferred=True, locale='en')
         concept = ConceptFactory(names=[ch_locale, en_locale])
         reference = CollectionReference(expression=concept.uri, collection=collection)
         reference.save()
@@ -205,8 +205,8 @@ def test_validate_reference_already_exists(self):
         )
 
     def test_validate_openmrs_schema_duplicate_locale_type(self):
-        ch_locale = LocalizedTextFactory(locale_preferred=True, locale='ch')
-        en_locale = LocalizedTextFactory(locale_preferred=True, locale='en')
+        ch_locale = ConceptNameFactory.build(locale_preferred=True, locale='ch')
+        en_locale = ConceptNameFactory.build(locale_preferred=True, locale='en')
         concept1 = ConceptFactory(names=[ch_locale, en_locale])
         collection = OrganizationCollectionFactory(custom_validation_schema=CUSTOM_VALIDATION_SCHEMA_OPENMRS)
         expansion = ExpansionFactory(collection_version=collection)
@@ -217,6 +217,8 @@ def test_validate_openmrs_schema_duplicate_locale_type(self):
             expression=concept1.uri, collection=collection, system=concept1.parent.uri, version='HEAD')
         concept1_reference.save()
 
+        ch_locale = ConceptNameFactory.build(name=ch_locale.name, locale_preferred=True, locale='ch')
+        en_locale = ConceptNameFactory.build(name=en_locale.name, locale_preferred=True, locale='en')
         concept2 = ConceptFactory(names=[ch_locale, en_locale])
         concept2_reference = CollectionReference(
             expression=concept2.uri, collection=collection, system=concept2.parent.uri, version='HEAD')
@@ -230,7 +232,7 @@ def test_validate_openmrs_schema_duplicate_locale_type(self):
         )
 
     def test_validate_openmrs_schema_matching_name_locale(self):
-        ch_locale = LocalizedTextFactory(locale_preferred=False, locale='ch')
+        ch_locale = ConceptNameFactory.build(locale_preferred=False, locale='ch')
         concept1 = ConceptFactory(names=[ch_locale])
         collection = OrganizationCollectionFactory(custom_validation_schema=CUSTOM_VALIDATION_SCHEMA_OPENMRS)
         expansion = ExpansionFactory(collection_version=collection)
@@ -241,8 +243,8 @@ def test_validate_openmrs_schema_matching_name_locale(self):
             expression=concept1.uri, collection=collection, system=concept1.parent.uri, version='HEAD')
         concept1_reference.save()
 
-        en_locale1 = LocalizedTextFactory(locale='en', locale_preferred=False, name='name')
-        en_locale2 = LocalizedTextFactory(locale='en', locale_preferred=True, name='name')
+        en_locale1 = ConceptNameFactory.build(locale='en', locale_preferred=False, name='name')
+        en_locale2 = ConceptNameFactory.build(locale='en', locale_preferred=True, name='name')
         concept2 = ConceptFactory(names=[en_locale1, en_locale2])
         concept2_reference = CollectionReference(
             expression=concept2.uri, collection=collection, system=concept2.parent.uri, version='HEAD')

diff --git a/core/common/tasks.py b/core/common/tasks.py
@@ -482,16 +482,16 @@ def delete_duplicate_locales(start_from=None):  # pragma: no cover
 
 @app.task
 def delete_dormant_locales():  # pragma: no cover
-    from core.concepts.models import LocalizedText
-    queryset = LocalizedText.get_dormant_queryset()
+    from core.concepts.models import ConceptName
+    queryset = ConceptName.get_dormant_queryset()
     total = queryset.count()
     logger.info('%s Dormant locales found. Deleting in batches...' % total)  # pylint: disable=logging-not-lazy,consider-using-f-string
 
     batch_size = 1000
     for start in range(0, total, batch_size):
         end = min(start + batch_size, total)
         logger.info('Iterating locales %d - %d to delete...' % (start + 1, end))  # pylint: disable=logging-not-lazy,consider-using-f-string
-        LocalizedText.objects.filter(id__in=queryset.order_by('id')[start:end].values('id')).delete()
+        ConceptName.objects.filter(id__in=queryset.order_by('id')[start:end].values('id')).delete()
 
     return 1
 

diff --git a/core/common/tests.py b/core/common/tests.py
@@ -3,6 +3,8 @@
 import uuid
 from unittest.mock import patch, Mock, mock_open
 
+import factory
+
 import boto3
 from botocore.exceptions import ClientError
 from colour_runner.django_runner import ColourRunnerMixin
@@ -49,7 +51,7 @@ class BaseTestCase(SetupTestEnvironment):
     @staticmethod
     def create_lookup_concept_classes(user=None, org=None):
         from core.sources.tests.factories import OrganizationSourceFactory
-        from core.concepts.tests.factories import LocalizedTextFactory, ConceptFactory
+        from core.concepts.tests.factories import ConceptNameFactory, ConceptFactory
 
         org = org or Organization.objects.get(mnemonic='OCL')
         user = user or UserProfile.objects.get(username='ocladmin')
@@ -71,129 +73,129 @@ def create_lookup_concept_classes(user=None, org=None):
 
         ConceptFactory(
             version=HEAD, updated_by=user, parent=classes_source, concept_class="Concept Class",
-            names=[LocalizedTextFactory(name="Diagnosis")]
+            names=[ConceptNameFactory.build(name="Diagnosis")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=classes_source, concept_class="Concept Class",
-            names=[LocalizedTextFactory(name="Drug")]
+            names=[ConceptNameFactory.build(name="Drug")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=classes_source, concept_class="Concept Class",
-            names=[LocalizedTextFactory(name="Test")]
+            names=[ConceptNameFactory.build(name="Test")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=classes_source, concept_class="Concept Class",
-            names=[LocalizedTextFactory(name="Procedure")]
+            names=[ConceptNameFactory.build(name="Procedure")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=datatypes_source, concept_class="Datatype",
-            names=[LocalizedTextFactory(name="None"), LocalizedTextFactory(name="N/A")]
+            names=[ConceptNameFactory.build(name="None"), ConceptNameFactory.build(name="N/A")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=datatypes_source, concept_class="Datatype",
-            names=[LocalizedTextFactory(name="Numeric")]
+            names=[ConceptNameFactory.build(name="Numeric")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=datatypes_source, concept_class="Datatype",
-            names=[LocalizedTextFactory(name="Coded")]
+            names=[ConceptNameFactory.build(name="Coded")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=datatypes_source, concept_class="Datatype",
-            names=[LocalizedTextFactory(name="Text")]
+            names=[ConceptNameFactory.build(name="Text")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=nametypes_source, concept_class="NameType",
-            names=[LocalizedTextFactory(name="FULLY_SPECIFIED"), LocalizedTextFactory(name="Fully Specified")]
+            names=[ConceptNameFactory.build(name="FULLY_SPECIFIED"), ConceptNameFactory.build(name="Fully Specified")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=nametypes_source, concept_class="NameType",
-            names=[LocalizedTextFactory(name="Short"), LocalizedTextFactory(name="SHORT")]
+            names=[ConceptNameFactory.build(name="Short"), ConceptNameFactory.build(name="SHORT")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=nametypes_source, concept_class="NameType",
-            names=[LocalizedTextFactory(name="INDEX_TERM"), LocalizedTextFactory(name="Index Term")]
+            names=[ConceptNameFactory.build(name="INDEX_TERM"), ConceptNameFactory.build(name="Index Term")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=nametypes_source, concept_class="NameType",
-            names=[LocalizedTextFactory(name="None")]
+            names=[ConceptNameFactory.build(name="None")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=descriptiontypes_source, concept_class="DescriptionType",
-            names=[LocalizedTextFactory(name="None")]
+            names=[ConceptNameFactory.build(name="None")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=descriptiontypes_source, concept_class="DescriptionType",
-            names=[LocalizedTextFactory(name="FULLY_SPECIFIED")]
+            names=[ConceptNameFactory.build(name="FULLY_SPECIFIED")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=descriptiontypes_source, concept_class="DescriptionType",
-            names=[LocalizedTextFactory(name="Definition")]
+            names=[ConceptNameFactory.build(name="Definition")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=maptypes_source, concept_class="MapType",
-            names=[LocalizedTextFactory(name="SAME-AS"), LocalizedTextFactory(name="Same As")]
+            names=[ConceptNameFactory.build(name="SAME-AS"), ConceptNameFactory.build(name="Same As")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=maptypes_source, concept_class="MapType",
-            names=[LocalizedTextFactory(name="Is Subset of")]
+            names=[ConceptNameFactory.build(name="Is Subset of")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=maptypes_source, concept_class="MapType",
-            names=[LocalizedTextFactory(name="Different")]
+            names=[ConceptNameFactory.build(name="Different")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=maptypes_source, concept_class="MapType",
             names=[
-                LocalizedTextFactory(name="BROADER-THAN"), LocalizedTextFactory(name="Broader Than"),
-                LocalizedTextFactory(name="BROADER_THAN")
+                ConceptNameFactory.build(name="BROADER-THAN"), ConceptNameFactory.build(name="Broader Than"),
+                ConceptNameFactory.build(name="BROADER_THAN")
             ]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=maptypes_source, concept_class="MapType",
             names=[
-                LocalizedTextFactory(name="NARROWER-THAN"), LocalizedTextFactory(name="Narrower Than"),
-                LocalizedTextFactory(name="NARROWER_THAN")
+                ConceptNameFactory.build(name="NARROWER-THAN"), ConceptNameFactory.build(name="Narrower Than"),
+                ConceptNameFactory.build(name="NARROWER_THAN")
             ]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=maptypes_source, concept_class="MapType",
-            names=[LocalizedTextFactory(name="Q-AND-A")]
+            names=[ConceptNameFactory.build(name="Q-AND-A")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=maptypes_source, concept_class="MapType",
-            names=[LocalizedTextFactory(name="More specific than")]
+            names=[ConceptNameFactory.build(name="More specific than")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=maptypes_source, concept_class="MapType",
-            names=[LocalizedTextFactory(name="Less specific than")]
+            names=[ConceptNameFactory.build(name="Less specific than")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=maptypes_source, concept_class="MapType",
-            names=[LocalizedTextFactory(name="Something Else")]
+            names=[ConceptNameFactory.build(name="Something Else")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=locales_source, concept_class="Locale",
-            names=[LocalizedTextFactory(name="en")]
+            names=[ConceptNameFactory.build(name="en")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=locales_source, concept_class="Locale",
-            names=[LocalizedTextFactory(name="es")]
+            names=[ConceptNameFactory.build(name="es")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=locales_source, concept_class="Locale",
-            names=[LocalizedTextFactory(name="fr")]
+            names=[ConceptNameFactory.build(name="fr")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=locales_source, concept_class="Locale",
-            names=[LocalizedTextFactory(name="tr")]
+            names=[ConceptNameFactory.build(name="tr")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=locales_source, concept_class="Locale",
-            names=[LocalizedTextFactory(name="Abkhazian")]
+            names=[ConceptNameFactory.build(name="Abkhazian")]
         )
         ConceptFactory(
             version=HEAD, updated_by=user, parent=locales_source, concept_class="Locale",
-            names=[LocalizedTextFactory(name="English")]
+            names=[ConceptNameFactory.build(name="English")]
         )
 
 
@@ -213,6 +215,13 @@ def setUpClass(cls):
         call_command("loaddata", "core/fixtures/base_entities.yaml")
         call_command("loaddata", "core/fixtures/auth_groups.yaml")
 
+    @staticmethod
+    def factory_to_params(factory_klass, **kwargs):
+        return {
+            **factory.build(dict, FACTORY_CLASS=factory_klass),
+            **kwargs
+        }
+
 
 class S3Test(TestCase):
     @mock_s3

diff --git a/core/common/views.py b/core/common/views.py
@@ -877,8 +877,8 @@ class ConceptDormantLocalesView(APIView):  # pragma: no cover
 
     @staticmethod
     def get(_, **kwargs):  # pylint: disable=unused-argument
-        from core.concepts.models import LocalizedText
-        count = LocalizedText.dormants()
+        from core.concepts.models import ConceptName
+        count = ConceptName.dormants()
         return Response(count, status=status.HTTP_200_OK)
 
     @staticmethod

diff --git a/core/concept_maps/tests.py b/core/concept_maps/tests.py
@@ -1,7 +1,7 @@
 from rest_framework.test import APIClient
 
 from core.common.tests import OCLTestCase
-from core.concepts.tests.factories import ConceptFactory, LocalizedTextFactory
+from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory
 from core.mappings.tests.factories import MappingFactory
 from core.orgs.tests.factories import OrganizationFactory
 from core.sources.models import Source
@@ -20,7 +20,7 @@ def setUp(self):
             version='v1', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent)
         Source.persist_new_version(self.org_source_v1, self.org_source.created_by)
         self.concept_1 = ConceptFactory(parent=self.org_source, mnemonic='concept_1',
-                                        names=[LocalizedTextFactory(name="concept_1_name")])
+                                        names=[ConceptNameFactory(name="concept_1_name")])
         self.concept_2 = ConceptFactory(parent=self.org_source, mnemonic='concept_2')
 
 
@@ -34,7 +34,7 @@ def setUp(self):
         self.org_source_B = OrganizationSourceFactory(organization=self.org, canonical_url='/some/url/B',
                                                        full_name='source name B')
         self.concept_B_1 = ConceptFactory(parent=self.org_source_B, mnemonic='concept_B_1',
-                                          names=[LocalizedTextFactory(name="concept_B_1_name")])
+                                          names=[ConceptNameFactory(name="concept_B_1_name")])
         self.concept_B_2 = ConceptFactory(parent=self.org_source_B, mnemonic='concept_B_2')
         self.org_source_B_v1 = OrganizationSourceFactory.build(
             version='v1', mnemonic=self.org_source_B.mnemonic, organization=self.org_source_B.parent)

diff --git a/core/concepts/documents.py b/core/concepts/documents.py
@@ -68,9 +68,7 @@ def prepare_name(instance):
 
     @staticmethod
     def prepare_locale(instance):
-        return list(
-            instance.names.filter(locale__isnull=False).distinct('locale').values_list('locale', flat=True)
-        )
+        return list(set(instance.names.filter(locale__isnull=False).values_list('locale', flat=True)))
 
     @staticmethod
     def prepare_synonyms(instance):
@@ -114,11 +112,11 @@ def prepare_extras(instance):
     @staticmethod
     def prepare_name_types(instance):
         return list(
-            instance.names.filter(type__isnull=False).distinct('type').values_list('type', flat=True)
+            set(instance.names.filter(type__isnull=False).values_list('type', flat=True))
         )
 
     @staticmethod
     def prepare_description_types(instance):
         return list(
-            instance.descriptions.filter(type__isnull=False).distinct('type').values_list('type', flat=True)
+            set(instance.descriptions.filter(type__isnull=False).values_list('type', flat=True))
         )