Merge remote-tracking branch 'newgene/master'

biothings · Sep 19, 2022 · c4c6100 · c4c6100
2 parents 21482ce + 9ca3c30
commit c4c6100
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 5 deletions.
diff --git a/biothings/hub/databuild/differ.py b/biothings/hub/databuild/differ.py
@@ -10,6 +10,7 @@
 
 from biothings.utils.common import timesofar, get_timestamp, \
     dump, rmdashfr, loadobj, md5sum
+from biothings.utils.serializer import to_json, to_json_file
 from biothings.utils.mongo import id_feeder, get_target_db, get_previous_collection
 from biothings.utils.hub_db import get_src_build
 from biothings.utils.loggers import get_logger
@@ -297,7 +298,7 @@ def mapping_diffed(f):
                             got_error = err
                     self.metadata["diff"]["mapping_file"] = res["mapping_file"]
                     diff_stats["mapping_changed"] = True
-                    json.dump(self.metadata,
+                    to_json_file(self.metadata,
                               open(self.metadata_filename, "w"),
                               indent=True)
 
@@ -394,7 +395,7 @@ def diffed(f):
             self.logger.info(
                 "Finished calculating diff for the old collection. Total number of docs deleted: {}"
                 .format(diff_stats["delete"]))
-            json.dump(self.metadata,
+            to_json_file(self.metadata,
                       open(self.metadata_filename, "w"),
                       indent=True)
 
@@ -478,7 +479,7 @@ def merged(f, cnt):
                                  job={"step": "diff-reduce"})
             res = await merge_diff()
             self.metadata["diff"]["files"] = res
-            json.dump(self.metadata,
+            to_json_file(self.metadata,
                       open(self.metadata_filename, "w"),
                       indent=True)
             if got_error:
@@ -520,7 +521,7 @@ def posted(f):
 
             job.add_done_callback(posted)
             await job
-            json.dump(self.metadata,
+            to_json_file(self.metadata,
                       open(self.metadata_filename, "w"),
                       indent=True)
             if got_error:
@@ -1322,7 +1323,7 @@ def rebuild_diff_file_list(self, diff_folder):
                     metadata["diff"]["mapping"] = info
                 else:
                     metadata["diff"]["files"].append(info)
-            json.dump(metadata,
+            to_json_file(metadata,
                       open(os.path.join(diff_folder, "metadata.json"), "w"),
                       indent=True)
             self.logger.info(

diff --git a/biothings/utils/serializer.py b/biothings/utils/serializer.py
@@ -33,6 +33,11 @@ def to_json(data, indent=False, sort_keys=False):
     return orjson.dumps(data, default=orjson_default, option=option).decode()
 
 
+def to_json_file(data, fobj, indent=False, sort_keys=False):
+    json_str = to_json(data, indent=indent, sort_keys=sort_keys)
+    fobj.write(json_str)
+
+
 def to_yaml(data, stream=None, Dumper=yaml.SafeDumper, default_flow_style=False):
     # Author: Cyrus Afrasiabi