Batch indexing on version creation and reference delete

OpenConceptLab · Mar 10, 2021 · 0fbd64a · 0fbd64a
1 parent b22d7c5
commit 0fbd64a
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 7 deletions.
diff --git a/core/collections/models.py b/core/collections/models.py
@@ -309,8 +309,8 @@ def delete_references(self, expressions):
 
         from core.concepts.documents import ConceptDocument
         from core.mappings.documents import MappingDocument
-        ConceptDocument().update(Concept.objects.filter(uri__in=expressions))
-        MappingDocument().update(Mapping.objects.filter(uri__in=expressions))
+        self.batch_index(Concept.objects.filter(uri__in=expressions), ConceptDocument)
+        self.batch_index(Mapping.objects.filter(uri__in=expressions), MappingDocument)
 
     @staticmethod
     def __get_children_from_expressions(expressions):

diff --git a/core/common/models.py b/core/common/models.py
@@ -643,7 +643,7 @@ def seed_concepts(self, index=True):
             self.concepts.set(concepts)
             if index:
                 from core.concepts.documents import ConceptDocument
-                ConceptDocument().update(self.concepts.all(), parallel=True)
+                self.batch_index(self.concepts, ConceptDocument)
 
     def seed_mappings(self, index=True):
         head = self.head
@@ -657,14 +657,25 @@ def seed_mappings(self, index=True):
             self.mappings.set(mappings)
             if index:
                 from core.mappings.documents import MappingDocument
-                MappingDocument().update(self.mappings.all(), parallel=True)
+                self.batch_index(self.mappings, MappingDocument)
+
+    @staticmethod
+    def batch_index(queryset, document):
+        count = queryset.count()
+        batch_size = 100
+        offset = 0
+        limit = batch_size
+        while offset < count:
+            document().update(queryset.all()[offset:limit], parallel=True)
+            offset = limit
+            limit += batch_size
 
     def index_children(self):
         from core.concepts.documents import ConceptDocument
-        ConceptDocument().update(self.concepts.all(), parallel=True)
-
         from core.mappings.documents import MappingDocument
-        MappingDocument().update(self.mappings.all(), parallel=True)
+
+        self.batch_index(self.concepts, ConceptDocument)
+        self.batch_index(self.mappings, MappingDocument)
 
     def add_processing(self, process_id):
         if self.id: