Expansions | indexing async

OpenConceptLab · Jan 19, 2022 · b0b2b29 · b0b2b29
1 parent c705bab
commit b0b2b29
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 14 deletions.
diff --git a/core/collections/models.py b/core/collections/models.py
@@ -15,7 +15,8 @@
     DEFAULT_REPOSITORY_TYPE, ACCESS_TYPE_VIEW, ACCESS_TYPE_EDIT,
     ACCESS_TYPE_NONE, SEARCH_PARAM)
 from core.common.models import ConceptContainerModel, BaseResourceModel
-from core.common.tasks import seed_children_to_expansion
+from core.common.tasks import seed_children_to_expansion, batch_index_resources, index_expansion_concepts, \
+    index_expansion_mappings
 from core.common.utils import is_valid_uri, drop_version, to_owner_uri, generate_temp_version, api_get
 from core.concepts.constants import LOCALES_FULLY_SPECIFIED
 from core.concepts.models import Concept
@@ -594,13 +595,11 @@ def apply_parameters(self, queryset):
 
     def index_concepts(self):
         if self.concepts.exists():
-            from core.concepts.documents import ConceptDocument
-            self.batch_index(self.concepts, ConceptDocument)
+            index_expansion_concepts.apply_async((self.id, ), queue='indexing')
 
     def index_mappings(self):
         if self.mappings.exists():
-            from core.mappings.documents import MappingDocument
-            self.batch_index(self.mappings, MappingDocument)
+            index_expansion_mappings.apply_async((self.id, ), queue='indexing')
 
     def index_all(self):
         self.index_concepts()
@@ -610,10 +609,8 @@ def delete_expressions(self, expressions):
         self.concepts.set(self.concepts.exclude(uri__in=expressions))
         self.mappings.set(self.mappings.exclude(uri__in=expressions))
 
-        from core.concepts.documents import ConceptDocument
-        from core.mappings.documents import MappingDocument
-        self.batch_index(Concept.objects.filter(uri__in=expressions), ConceptDocument)
-        self.batch_index(Mapping.objects.filter(uri__in=expressions), MappingDocument)
+        batch_index_resources.apply_async(('concept', dict(uri__in=expressions)), queue='indexing')
+        batch_index_resources.apply_async(('mapping', dict(uri__in=expressions)), queue='indexing')
 
     def add_references(self, references, index=True):
         if isinstance(references, CollectionReference):

diff --git a/core/common/tasks.py b/core/common/tasks.py
@@ -473,6 +473,24 @@ def batch_index_resources(resource, filters):
     return 1
 
 
+@app.task
+def index_expansion_concepts(expansion_id):
+    from core.collections.models import Expansion
+    expansion = Expansion.objects.filter(id=expansion_id).first()
+    if expansion:
+        from core.concepts.documents import ConceptDocument
+        expansion.batch_index(expansion.concepts, ConceptDocument)
+
+
+@app.task
+def index_expansion_mappings(expansion_id):
+    from core.collections.models import Expansion
+    expansion = Expansion.objects.filter(id=expansion_id).first()
+    if expansion:
+        from core.mappings.documents import MappingDocument
+        expansion.batch_index(expansion.mappings, MappingDocument)
+
+
 @app.task
 def make_hierarchy(concept_map):  # pragma: no cover
     from core.concepts.models import Concept

diff --git a/core/importers/models.py b/core/importers/models.py
@@ -13,7 +13,7 @@
 from core.collections.models import Collection
 from core.common.constants import HEAD
 from core.common.services import RedisService
-from core.common.tasks import bulk_import_parts_inline, delete_organization
+from core.common.tasks import bulk_import_parts_inline, delete_organization, batch_index_resources
 from core.common.utils import drop_version, is_url_encoded_string, encode_string, to_parent_uri
 from core.concepts.models import Concept
 from core.mappings.models import Mapping
@@ -568,11 +568,11 @@ def process(self):
                 )
                 for ref in added_references:
                     if ref.concepts:
-                        for concept in ref.concepts:
-                            concept.index()
+                        batch_index_resources.apply_async(
+                            ('concept', dict(id__in=list(ref.concepts.values_list('id', flat=True)))), queue='indexing')
                     if ref.mappings:
-                        for mapping in ref.mappings:
-                            mapping.index()
+                        batch_index_resources.apply_async(
+                            ('mapping', dict(id__in=list(ref.mappings.values_list('id', flat=True)))), queue='indexing')
 
                 return CREATED
             return PERMISSION_DENIED