Cleanup workflow tests (#205)

* using tmp_path

* using pathlib.Path in operartor and provenance

* renamed TreeWorkflow to Workflow

rook/operator.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 from copy import deepcopy
+import pathlib
 
 from rook.director import wrap_director
 from rook.utils.input_utils import resolve_to_file_paths
@@ -17,8 +18,12 @@ class Operator(object):
     prefix = NotImplemented
 
     def __init__(self, output_dir):
+        if isinstance(output_dir, pathlib.Path):
+            output_dir_ = output_dir.as_posix()
+        else:
+            output_dir_ = output_dir
         self.config = {
-            "output_dir": output_dir,
+            "output_dir": output_dir_,
             # "apply_fixes": apply_fixes,
             # 'original_files': original_files
             # 'chunk_rules': dconfig.chunk_rules,

rook/provenance.py

@@ -2,6 +2,7 @@
 import uuid
 import json
 from datetime import datetime
+import pathlib
 
 from prov.identifier import Namespace
 import prov.model as prov
@@ -29,7 +30,10 @@
 
 class Provenance(object):
     def __init__(self, output_dir):
-        self.output_dir = output_dir
+        if isinstance(output_dir, pathlib.Path):
+            self.output_dir = output_dir
+        else:
+            self.output_dir = pathlib.Path(output_dir)
         self.doc = None
         self._identifier = None
         self._workflow = None
@@ -168,14 +172,14 @@ def _execution_activity(self, identifier, label=None, attributes=None):
         return activity
 
     def write_json(self):
-        outfile = os.path.join(self.output_dir, "provenance.json")
-        self.doc.serialize(outfile, format="json")
+        outfile = self.output_dir / "provenance.json"
+        self.doc.serialize(outfile.as_posix(), format="json")
         return outfile
 
     def write_png(self):
-        outfile = os.path.join(self.output_dir, "provenance.png")
+        outfile = self.output_dir / "provenance.png"
         figure = prov_to_dot(self.doc)
-        figure.write_png(outfile)
+        figure.write_png(outfile.as_posix())
         return outfile
 
     def get_provn(self):

rook/workflow.py

@@ -64,7 +64,7 @@ def build_tree(wfdoc):
 
 class WorkflowRunner(object):
     def __init__(self, output_dir):
-        self.workflow = TreeWorkflow(output_dir)
+        self.workflow = Workflow(output_dir)
 
     def run(self, path):
         wfdoc = load_wfdoc(path)
@@ -98,7 +98,7 @@ def _run(self, wfdoc):
         raise NotImplementedError("implemented in subclass")
 
 
-class TreeWorkflow(BaseWorkflow):
+class Workflow(BaseWorkflow):
     def validate(self, wfdoc):
         if "doc" not in wfdoc:
             raise WorkflowValidationError("doc missing")

tests/test_workflow.py

@@ -12,9 +12,9 @@
 TREE_WF_5 = resource_file("subset_wf_5.json")
 
 
-def test_validate_tree_wf():
+def test_validate_tree_wf(tmp_path):
     wfdoc = workflow.load_wfdoc(TREE_WF)
-    wf = workflow.TreeWorkflow(output_dir=tempfile.mkdtemp())
+    wf = workflow.Workflow(output_dir=tmp_path)
     assert wf.validate(wfdoc) is True
 
 
@@ -36,40 +36,40 @@ def test_build_tree():
     ]
 
 
-def test_run_tree_wf():
-    wf = workflow.WorkflowRunner(output_dir=tempfile.mkdtemp())
+def test_run_tree_wf(tmp_path):
+    wf = workflow.WorkflowRunner(output_dir=tmp_path)
     output = wf.run(TREE_WF)
     assert "tas_mon_HadGEM2-ES_rcp85_r1i1p1_20850101-21200101_avg-year.nc" in output[0]
 
 
-def test_run_tree_wf_2():
-    wf = workflow.WorkflowRunner(output_dir=tempfile.mkdtemp())
+def test_run_tree_wf_2(tmp_path):
+    wf = workflow.WorkflowRunner(output_dir=tmp_path)
     output = wf.run(TREE_WF_2)
     assert "tas_mon_HadGEM2-ES_rcp85_r1i1p1_20900101-21000101_avg-year.nc" in output[0]
 
 
-def test_run_wf_cmip6_subset_average():
+def test_run_wf_cmip6_subset_average(tmp_path):
     wfdoc = resource_file("wf_cmip6_subset_average.json")
-    wf = workflow.WorkflowRunner(output_dir=tempfile.mkdtemp())
+    wf = workflow.WorkflowRunner(output_dir=tmp_path)
     output = wf.run(wfdoc)
     assert (
         "rlds_Amon_IPSL-CM6A-LR_historical_r1i1p1f1_gr_19850101-20140101_avg-year.nc"
         in output[0]
     )
 
 
-def test_run_tree_wf_5():
-    wf = workflow.WorkflowRunner(output_dir=tempfile.mkdtemp())
+def test_run_tree_wf_5(tmp_path):
+    wf = workflow.WorkflowRunner(output_dir=tmp_path)
     output = wf.run(TREE_WF_5)
     assert (
         "rlds_Amon_IPSL-CM6A-LR_historical_r1i1p1f1_gr_19950101-20000101_avg-year.nc"
         in output[0]
     )
 
 
-def test_wf_average_latlon_cmip6():
+def test_wf_average_latlon_cmip6(tmp_path):
     wfdoc = resource_file("wf_average_latlon_cmip6.json")
-    wf = workflow.WorkflowRunner(output_dir=tempfile.mkdtemp())
+    wf = workflow.WorkflowRunner(output_dir=tmp_path)
     output = wf.run(wfdoc)
     # print(output)
     assert (
@@ -78,9 +78,9 @@ def test_wf_average_latlon_cmip6():
     )
 
 
-def test_wf_c3s_cmip6_collection_only():
+def test_wf_c3s_cmip6_collection_only(tmp_path):
     wfdoc = resource_file("wf_c3s_cmip6_subset_collection_only.json")
-    wf = workflow.WorkflowRunner(output_dir=tempfile.mkdtemp())
+    wf = workflow.WorkflowRunner(output_dir=tmp_path)
     output = wf.run(wfdoc)
     expected_url = (
         "https://data.mips.copernicus-climate.eu/thredds/fileServer/esg_c3s-cmip6/"
@@ -90,9 +90,9 @@ def test_wf_c3s_cmip6_collection_only():
     assert output[0] == expected_url
 
 
-def test_wf_c3s_cmip6_original_files():
+def test_wf_c3s_cmip6_original_files(tmp_path):
     wfdoc = resource_file("wf_c3s_cmip6_subset_original_files.json")
-    wf = workflow.WorkflowRunner(output_dir=tempfile.mkdtemp())
+    wf = workflow.WorkflowRunner(output_dir=tmp_path)
     output = wf.run(wfdoc)
     expected_url = (
         "https://data.mips.copernicus-climate.eu/thredds/fileServer/esg_c3s-cmip6/"