Merge pull request #100 from RockefellerArchiveCenter/issue-99

Uses rac_es helpers for bulk indexing
RockefellerArchiveCenter · May 21, 2020 · 8bdd66a · 8bdd66a
2 parents 5bc2abe + df790e8
commit 8bdd66a
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 8 deletions.
diff --git a/indexer/cron.py b/indexer/cron.py
@@ -13,7 +13,8 @@ class BaseCron(CronJobBase):
     def do(self):
         start = datetime.now()
         action = "Full" if self.clean else "Incremental"
-        object_type = self.object_type if self.object_type else "All"
+        object_type = self.object_type if self.object_type else "all"
+        indexed = []
         print("{} indexing of {} records started at {}".format(action, object_type, start))
         try:
             indexed = Indexer().add(object_type=self.object_type, clean=self.clean)

diff --git a/indexer/indexers.py b/indexer/indexers.py
@@ -45,10 +45,10 @@ def __init__(self):
             BaseDescriptionComponent.init()
         self.pisces_client = ElectronBond(baseurl=settings.PISCES['baseurl'])
 
-    def prepare_data(self, object_type, data):
-        doc = OBJECT_TYPES[object_type](**data["data"])
-        doc.meta.id = data["es_id"]
-        return doc.to_dict(True)
+    def prepare_data(self, obj_type, clean):
+        for obj in self.fetch_objects(obj_type, clean):
+            doc = OBJECT_TYPES[obj_type](**obj)
+            yield doc.prepare_streaming_dict(obj["id"])
 
     @silk_profile()
     def fetch_objects(self, object_type, clean):
@@ -62,8 +62,7 @@ def add(self, object_type=None, clean=False, **kwargs):
         indexed_ids = []
         object_types = [object_type] if object_type else OBJECT_TYPES
         for obj_type in object_types:
-            objects = self.fetch_objects(obj_type, clean)
-            for ok, result in streaming_bulk(self.connection, (self.prepare_data(obj_type, obj) for obj in objects), refresh=True):
+            for ok, result in streaming_bulk(self.connection, self.prepare_data(obj_type, clean), refresh=True):
                 action, result = result.popitem()
                 if not ok:
                     update_pisces(indexed_ids, "indexed")

diff --git a/requirements.txt b/requirements.txt
@@ -32,6 +32,7 @@ Pygments==2.5.2
 python-dateutil==2.8.1
 pytz==2019.3
 PyYAML==5.3
+rac_es==0.2.1
 requests==2.23.0
 shortuuid==0.5.0
 six==1.12.0
@@ -44,4 +45,3 @@ wrapt==1.12.0
 yarl==1.4.2
 zipp==3.0.0
 zope.interface==4.7.1
-git+https://github.com/RockefellerArchiveCenter/rac_es#egg=rac_es