From a9d63259358318e26460e260a1db99c5583f5f45 Mon Sep 17 00:00:00 2001 From: Sny Date: Wed, 17 Mar 2021 08:46:14 +0530 Subject: [PATCH] OpenConceptLab/ocl_issues#637 | flatten extras to separate fields for indexing --- core/collections/documents.py | 4 ++- core/common/tests.py | 56 ++++++++++++++++++++++++++++++++++- core/common/utils.py | 19 ++++++++++++ core/concepts/documents.py | 4 ++- core/mappings/documents.py | 4 ++- core/orgs/documents.py | 4 ++- core/sources/documents.py | 4 ++- core/users/documents.py | 4 ++- 8 files changed, 92 insertions(+), 7 deletions(-) diff --git a/core/collections/documents.py b/core/collections/documents.py index 106b821e5..c13667245 100644 --- a/core/collections/documents.py +++ b/core/collections/documents.py @@ -3,7 +3,7 @@ from pydash import get from core.collections.models import Collection -from core.common.utils import jsonify_safe +from core.common.utils import jsonify_safe, flatten_dict @registry.register_document @@ -48,6 +48,8 @@ def prepare_extras(instance): if instance.extras: value = jsonify_safe(instance.extras) + if isinstance(value, dict): + value = flatten_dict(value) return value or {} diff --git a/core/common/tests.py b/core/common/tests.py index 73209a248..1213e8a58 100644 --- a/core/common/tests.py +++ b/core/common/tests.py @@ -20,7 +20,7 @@ compact_dict_by_values, to_snake_case, flower_get, task_exists, parse_bulk_import_task_id, to_camel_case, drop_version, is_versioned_uri, separate_version, to_parent_uri, jsonify_safe, es_get, - get_resource_class_from_resource_name) + get_resource_class_from_resource_name, flatten_dict) from core.concepts.models import Concept, LocalizedText from core.mappings.models import Mapping from core.orgs.models import Organization @@ -638,6 +638,60 @@ def test_get_resource_class_from_resource_name(self): for name in ['user', 'USer', 'user_profile', 'USERS']: self.assertEqual(get_resource_class_from_resource_name(name).__name__, 'UserProfile') + def test_flatten_dict(self): + self.assertEqual(flatten_dict(dict(foo='bar')), dict(foo='bar')) + self.assertEqual(flatten_dict(dict(foo=1)), dict(foo='1')) + self.assertEqual(flatten_dict(dict(foo=1.1)), dict(foo='1.1')) + self.assertEqual(flatten_dict(dict(foo=True)), dict(foo='True')) + self.assertEqual( + flatten_dict(dict(foo=True, bar=dict(tao=dict(te='ching')))), + dict(foo='True', bar__tao__te='ching') + ) + # self.assertEqual( + # flatten_dict( + # { + # 'path': [ + # {'text': 'MedicationStatement', 'linkid': '/MedicationStatement'}, + # {'text': 'Family Planning Modern Method', 'linkid': '/MedicationStatement/method'} + # ], + # 'header_concept_id': 'ModernMethod', + # 'questionnaire_choice_value': 'LA27919-2' + # } + # ), + # dict( + # path__0__text='MedicationStatement', path__0__linkid='/MedicationStatement', + # path__1__text='Family Planning Modern Method', path__1__linkid='/MedicationStatement/method', + # header_concept_id='ModernMethod', questionnaire_choice_value='LA27919-2' + # ) + # ) + # + # self.assertEqual( + # flatten_dict( + # { + # 'path': [ + # {'text': 'MedicationStatement', 'linkid': '/MedicationStatement'}, + # {'text': 'Family Planning Modern Method', 'linkid': '/MedicationStatement/method'} + # ], + # 'Applicable Periods': ['FY19', 'FY18'], + # 'foobar': [1], + # 'bar': [], + # 'header_concept_id': 'ModernMethod', + # 'questionnaire_choice_value': 'LA27919-2' + # } + # ), + # { + # 'path__0__text': 'MedicationStatement', + # 'path__0__linkid': '/MedicationStatement', + # 'path__1__text': 'Family Planning Modern Method', + # 'path__1__linkid': '/MedicationStatement/method', + # 'Applicable Periods__0': 'FY19', + # 'Applicable Periods__1': 'FY18', + # 'foobar__0': '1', + # 'header_concept_id': 'ModernMethod', + # 'questionnaire_choice_value': 'LA27919-2', + # } + # ) + class BaseModelTest(OCLTestCase): def test_model_name(self): diff --git a/core/common/utils.py b/core/common/utils.py index 125eb04f6..6aec7312d 100644 --- a/core/common/utils.py +++ b/core/common/utils.py @@ -4,6 +4,7 @@ import tempfile import uuid import zipfile +from collections import MutableMapping # pylint: disable=no-name-in-module from urllib import parse import requests @@ -538,3 +539,21 @@ def get_content_type_from_resource_name(resource): return ContentType.objects.get_for_model(model) return None + + +def flatten_dict(dikt, parent_key='', sep='__'): + items = [] + for key, val in dikt.items(): + new_key = parent_key + sep + key if parent_key else key + + if isinstance(val, MutableMapping): + items.extend(flatten_dict(val, new_key, sep=sep).items()) + # elif isinstance(val, list): + # for i, _v in enumerate(val, start=0): + # if isinstance(_v, dict): + # items.extend(flatten_dict(_v, "{}__{}".format(key, i)).items()) + # else: + # items.append(("{}__{}".format(key, i), str(_v))) + else: + items.append((new_key, str(val))) + return dict(items) diff --git a/core/concepts/documents.py b/core/concepts/documents.py index cdacdfdeb..ffee1ef96 100644 --- a/core/concepts/documents.py +++ b/core/concepts/documents.py @@ -1,7 +1,7 @@ from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry -from core.common.utils import jsonify_safe +from core.common.utils import jsonify_safe, flatten_dict from core.concepts.models import Concept @@ -64,5 +64,7 @@ def prepare_extras(instance): if instance.extras: value = jsonify_safe(instance.extras) + if isinstance(value, dict): + value = flatten_dict(value) return value or {} diff --git a/core/mappings/documents.py b/core/mappings/documents.py index df8267359..4c7966ae7 100644 --- a/core/mappings/documents.py +++ b/core/mappings/documents.py @@ -2,7 +2,7 @@ from django_elasticsearch_dsl.registries import registry from pydash import get -from core.common.utils import jsonify_safe +from core.common.utils import jsonify_safe, flatten_dict from core.mappings.models import Mapping @@ -92,5 +92,7 @@ def prepare_extras(instance): if instance.extras: value = jsonify_safe(instance.extras) + if isinstance(value, dict): + value = flatten_dict(value) return value or {} diff --git a/core/orgs/documents.py b/core/orgs/documents.py index 68c9b1e99..45322b479 100644 --- a/core/orgs/documents.py +++ b/core/orgs/documents.py @@ -1,7 +1,7 @@ from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry -from core.common.utils import jsonify_safe +from core.common.utils import jsonify_safe, flatten_dict from core.orgs.models import Organization @@ -32,6 +32,8 @@ def prepare_extras(instance): if instance.extras: value = jsonify_safe(instance.extras) + if isinstance(value, dict): + value = flatten_dict(value) return value or {} diff --git a/core/sources/documents.py b/core/sources/documents.py index 91aff6225..6b1dd8f77 100644 --- a/core/sources/documents.py +++ b/core/sources/documents.py @@ -2,7 +2,7 @@ from django_elasticsearch_dsl.registries import registry from pydash import get -from core.common.utils import jsonify_safe +from core.common.utils import jsonify_safe, flatten_dict from core.sources.models import Source @@ -49,6 +49,8 @@ def prepare_extras(instance): if instance.extras: value = jsonify_safe(instance.extras) + if isinstance(value, dict): + value = flatten_dict(value) return value or {} diff --git a/core/users/documents.py b/core/users/documents.py index e051f8111..6c6b883a6 100644 --- a/core/users/documents.py +++ b/core/users/documents.py @@ -1,7 +1,7 @@ from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry -from core.common.utils import jsonify_safe +from core.common.utils import jsonify_safe, flatten_dict from core.users.models import UserProfile @@ -32,6 +32,8 @@ def prepare_extras(instance): if instance.extras: value = jsonify_safe(instance.extras) + if isinstance(value, dict): + value = flatten_dict(value) return value or {}