Merge pull request #121 from kolusask/mti_output

Process MTI mesh_terms
gyorilab · Jul 16, 2020 · 053485d · 053485d
2 parents 77113bb + b17a1e7
commit 053485d
Show file tree

Hide file tree

Showing 5 changed files with 248 additions and 158 deletions.
diff --git a/indra_db/managers/dump_manager.py b/indra_db/managers/dump_manager.py
@@ -236,6 +236,22 @@ def dump(self, continuing=False):
         return
 
 
+class StatementHashMeshId(Dumper):
+    name = 'mti_mesh_ids'
+    fmi = 'pkl'
+
+    def dump(self, continuing=False):
+        if self.use_principal:
+            ro = get_db(self.db_label)
+        else:
+            ro = get_ro(self.db_label)
+
+        q = ro.select_all([ro.MeshMeta.mk_hash, ro.MeshMeta.mesh_num])
+
+        s3 = boto3.client('s3')
+        s3.put_object(Body=pickle.dumps(q.all()), **self.get_s3_path().kw())
+
+
 def load_readonly_dump(db_label, ro_label, dump_file):
     principal_db = get_db(db_label)
     readonly_db = get_ro(ro_label)
@@ -359,7 +375,6 @@ def main():
         ro_dumper = Readonly.from_list(starter.manifest)
         if not args.allow_continue or not ro_dumper:
             logger.info("Generating readonly schema (est. a long time)")
-            assert False, 'moooo'
             ro_dumper = Readonly(date_stamp=starter.date_stamp)
             ro_dumper.dump(continuing=args.allow_continue)
         else:
@@ -380,6 +395,12 @@ def main():
         else:
             logger.info("Statement dump exists, skipping.")
 
+        if not args.allow_continue \
+                or not StatementHashMeshId.from_list(starter.manifest):
+            logger.info("Dumping hash-mesh tuples.")
+            StatementHashMeshId(date_stamp=starter.date_stamp)\
+                .dump(continuing=args.continuing)
+
         if not args.allow_continue or not Belief.from_list(starter.manifest):
             logger.info("Dumping belief.")
             Belief(date_stamp=starter.date_stamp)\