Add support for compressed exports at merge step

Knowledge-Graph-Hub · Aug 10, 2020 · 4e03403 · 4e03403
1 parent 6fb5109
commit 4e03403
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 4 deletions.
diff --git a/kg_covid_19/merge_utils/merge_kg.py b/kg_covid_19/merge_utils/merge_kg.py
@@ -114,9 +114,10 @@ def load_and_merge(yaml_file: str) -> nx.MultiDiGraph:
                 destination_transformer.save()
             elif destination['type'] in get_file_types():
                 destination_transformer = get_transformer(destination['type'])(merged_graph)
-                if destination['type'] in {'nt', 'ttl'}:
+                mode = 'w:gz' if destination['type'] in {'tsv'} else None
+                if destination['type'] in {'nt', 'nt.gz', 'ttl'}:
                     destination_transformer.set_property_types(PROPERTY_TYPES)
-                destination_transformer.save(destination['filename'], output_format=destination['type'])
+                destination_transformer.save(destination['filename'], output_format=destination['type'], mode=mode)
             else:
                 logging.error("type {} not yet supported for KGX load-and-merge operation.".format(destination['type']))
 

diff --git a/merge.yaml b/merge.yaml
@@ -92,8 +92,8 @@ destination:
       type: tsv
       filename: data/merged/merged-kg
    merged-kg-nt:
-      type: nt
-      filename: data/merged/merged-kg.nt
+      type: nt.gz
+      filename: data/merged/merged-kg.nt.gz
 #   merged-kg-neo4j:
 #      type: neo4j
 #      uri: http://localhost:8484