OpenConceptLab/ocl_issues#637 | flatten extras to separate fields for…

… indexing
OpenConceptLab · Mar 17, 2021 · a9d6325 · a9d6325
1 parent f4f9451
commit a9d6325
Show file tree

Hide file tree

Showing 8 changed files with 92 additions and 7 deletions.
diff --git a/core/collections/documents.py b/core/collections/documents.py
@@ -3,7 +3,7 @@
 from pydash import get
 
 from core.collections.models import Collection
-from core.common.utils import jsonify_safe
+from core.common.utils import jsonify_safe, flatten_dict
 
 
 @registry.register_document
@@ -48,6 +48,8 @@ def prepare_extras(instance):
 
         if instance.extras:
             value = jsonify_safe(instance.extras)
+            if isinstance(value, dict):
+                value = flatten_dict(value)
 
         return value or {}
 

diff --git a/core/common/tests.py b/core/common/tests.py
@@ -20,7 +20,7 @@
     compact_dict_by_values, to_snake_case, flower_get, task_exists, parse_bulk_import_task_id,
     to_camel_case,
     drop_version, is_versioned_uri, separate_version, to_parent_uri, jsonify_safe, es_get,
-    get_resource_class_from_resource_name)
+    get_resource_class_from_resource_name, flatten_dict)
 from core.concepts.models import Concept, LocalizedText
 from core.mappings.models import Mapping
 from core.orgs.models import Organization
@@ -638,6 +638,60 @@ def test_get_resource_class_from_resource_name(self):
         for name in ['user', 'USer', 'user_profile', 'USERS']:
             self.assertEqual(get_resource_class_from_resource_name(name).__name__, 'UserProfile')
 
+    def test_flatten_dict(self):
+        self.assertEqual(flatten_dict(dict(foo='bar')), dict(foo='bar'))
+        self.assertEqual(flatten_dict(dict(foo=1)), dict(foo='1'))
+        self.assertEqual(flatten_dict(dict(foo=1.1)), dict(foo='1.1'))
+        self.assertEqual(flatten_dict(dict(foo=True)), dict(foo='True'))
+        self.assertEqual(
+            flatten_dict(dict(foo=True, bar=dict(tao=dict(te='ching')))),
+            dict(foo='True', bar__tao__te='ching')
+        )
+        # self.assertEqual(
+        #     flatten_dict(
+        #         {
+        #             'path': [
+        #                 {'text': 'MedicationStatement', 'linkid': '/MedicationStatement'},
+        #                 {'text': 'Family Planning Modern Method', 'linkid': '/MedicationStatement/method'}
+        #             ],
+        #             'header_concept_id': 'ModernMethod',
+        #             'questionnaire_choice_value': 'LA27919-2'
+        #         }
+        #     ),
+        #     dict(
+        #         path__0__text='MedicationStatement', path__0__linkid='/MedicationStatement',
+        #         path__1__text='Family Planning Modern Method', path__1__linkid='/MedicationStatement/method',
+        #         header_concept_id='ModernMethod', questionnaire_choice_value='LA27919-2'
+        #     )
+        # )
+        #
+        # self.assertEqual(
+        #     flatten_dict(
+        #         {
+        #             'path': [
+        #                 {'text': 'MedicationStatement', 'linkid': '/MedicationStatement'},
+        #                 {'text': 'Family Planning Modern Method', 'linkid': '/MedicationStatement/method'}
+        #             ],
+        #             'Applicable Periods': ['FY19', 'FY18'],
+        #             'foobar': [1],
+        #             'bar': [],
+        #             'header_concept_id': 'ModernMethod',
+        #             'questionnaire_choice_value': 'LA27919-2'
+        #         }
+        #     ),
+        #     {
+        #         'path__0__text': 'MedicationStatement',
+        #         'path__0__linkid': '/MedicationStatement',
+        #         'path__1__text': 'Family Planning Modern Method',
+        #         'path__1__linkid': '/MedicationStatement/method',
+        #         'Applicable Periods__0': 'FY19',
+        #         'Applicable Periods__1': 'FY18',
+        #         'foobar__0': '1',
+        #         'header_concept_id': 'ModernMethod',
+        #         'questionnaire_choice_value': 'LA27919-2',
+        #     }
+        # )
+
 
 class BaseModelTest(OCLTestCase):
     def test_model_name(self):

diff --git a/core/common/utils.py b/core/common/utils.py
@@ -4,6 +4,7 @@
 import tempfile
 import uuid
 import zipfile
+from collections import MutableMapping  # pylint: disable=no-name-in-module
 from urllib import parse
 
 import requests
@@ -538,3 +539,21 @@ def get_content_type_from_resource_name(resource):
         return ContentType.objects.get_for_model(model)
 
     return None
+
+
+def flatten_dict(dikt, parent_key='', sep='__'):
+    items = []
+    for key, val in dikt.items():
+        new_key = parent_key + sep + key if parent_key else key
+
+        if isinstance(val, MutableMapping):
+            items.extend(flatten_dict(val, new_key, sep=sep).items())
+        # elif isinstance(val, list):
+        #     for i, _v in enumerate(val, start=0):
+        #         if isinstance(_v, dict):
+        #             items.extend(flatten_dict(_v, "{}__{}".format(key, i)).items())
+        #         else:
+        #             items.append(("{}__{}".format(key, i), str(_v)))
+        else:
+            items.append((new_key, str(val)))
+    return dict(items)
diff --git a/core/concepts/documents.py b/core/concepts/documents.py
@@ -1,7 +1,7 @@
 from django_elasticsearch_dsl import Document, fields
 from django_elasticsearch_dsl.registries import registry
 
-from core.common.utils import jsonify_safe
+from core.common.utils import jsonify_safe, flatten_dict
 from core.concepts.models import Concept
 
 
@@ -64,5 +64,7 @@ def prepare_extras(instance):
 
         if instance.extras:
             value = jsonify_safe(instance.extras)
+            if isinstance(value, dict):
+                value = flatten_dict(value)
 
         return value or {}
diff --git a/core/mappings/documents.py b/core/mappings/documents.py
@@ -2,7 +2,7 @@
 from django_elasticsearch_dsl.registries import registry
 from pydash import get
 
-from core.common.utils import jsonify_safe
+from core.common.utils import jsonify_safe, flatten_dict
 from core.mappings.models import Mapping
 
 
@@ -92,5 +92,7 @@ def prepare_extras(instance):
 
         if instance.extras:
             value = jsonify_safe(instance.extras)
+            if isinstance(value, dict):
+                value = flatten_dict(value)
 
         return value or {}
diff --git a/core/orgs/documents.py b/core/orgs/documents.py
@@ -1,7 +1,7 @@
 from django_elasticsearch_dsl import Document, fields
 from django_elasticsearch_dsl.registries import registry
 
-from core.common.utils import jsonify_safe
+from core.common.utils import jsonify_safe, flatten_dict
 from core.orgs.models import Organization
 
 
@@ -32,6 +32,8 @@ def prepare_extras(instance):
 
         if instance.extras:
             value = jsonify_safe(instance.extras)
+            if isinstance(value, dict):
+                value = flatten_dict(value)
 
         return value or {}
 

diff --git a/core/sources/documents.py b/core/sources/documents.py
@@ -2,7 +2,7 @@
 from django_elasticsearch_dsl.registries import registry
 from pydash import get
 
-from core.common.utils import jsonify_safe
+from core.common.utils import jsonify_safe, flatten_dict
 from core.sources.models import Source
 
 
@@ -49,6 +49,8 @@ def prepare_extras(instance):
 
         if instance.extras:
             value = jsonify_safe(instance.extras)
+            if isinstance(value, dict):
+                value = flatten_dict(value)
 
         return value or {}
 

diff --git a/core/users/documents.py b/core/users/documents.py
@@ -1,7 +1,7 @@
 from django_elasticsearch_dsl import Document, fields
 from django_elasticsearch_dsl.registries import registry
 
-from core.common.utils import jsonify_safe
+from core.common.utils import jsonify_safe, flatten_dict
 from core.users.models import UserProfile
 
 
@@ -32,6 +32,8 @@ def prepare_extras(instance):
 
         if instance.extras:
             value = jsonify_safe(instance.extras)
+            if isinstance(value, dict):
+                value = flatten_dict(value)
 
         return value or {}