Merge a1c1b94 into df67f63

kg_covid_19/load_utils/merge_kg.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from typing import Dict, List
 import yaml
 import networkx as nx
@@ -35,6 +36,16 @@ def load_and_merge(yaml_file: str) -> nx.MultiDiGraph:
     config = parse_load_config(yaml_file)
     transformers: List = []
 
+    # make sure all files exist before we start load
+    for key in config['target']:
+        target = config['target'][key]
+        logging.info("Checking that file exist for {}".format(key))
+        if target['type'] in get_file_types():
+            for f in target['filename']:
+                if not os.path.exists(f) or not os.path.isfile(f):
+                    raise FileNotFoundError("File {} for transform {}  in yaml file {} "
+                                            "doesn't exist! Dying.", f, key, yaml_file)
+
     # read all the sources defined in the YAML
     for key in config['target']:
         target = config['target'][key]

tests/resources/merge_MISSING_FILE.yaml

@@ -0,0 +1,10 @@
+---
+target:
+   drug-central:
+      type: tsv
+      filename:
+         - data/transformed/drug_central/nodes.tsv
+         - data/transformed/drug_central/DOESNT_EXIST.tsv
+destination:
+   type: tsv
+   filename: merged-kg

tests/resources/merge_valid.yaml

@@ -0,0 +1,10 @@
+---
+target:
+   drug-central:
+      type: tsv
+      filename:
+         - data/transformed/drug_central/nodes.tsv
+         - data/transformed/drug_central/edges.tsv
+destination:
+   type: tsv
+   filename: merged-kg

tests/test_run.py

@@ -2,8 +2,7 @@
 from click.testing import CliRunner
 from unittest import mock
 
-from run import transform
-from run import download
+from run import download, transform, load
 
 
 class TestRun(TestCase):
@@ -18,9 +17,16 @@ def test_download(self, mock_get):
         # this really just makes sure request.get get called somewhere downstream
         self.assertTrue(mock_get.called)
 
-    @skip
     def test_transform(self):
         result = self.runner.invoke(cli=transform,
                                     args=['-i', 'tests/data/raw'])
-        self.assertEqual(result.exit_code, 0)
+        self.assertNotEqual(result.exit_code, 0)
+
+    def test_merge_missing_file_error(self):
+        with self.assertRaises(FileNotFoundError) as context:
+           self.runner.invoke(catch_exceptions=False,
+                                        cli=load,
+                                        args=['-y',
+                                              'tests/resources/merge_MISSING_FILE.yaml']
+                                        )