Fix prov multiple files (#114)

* add all output files to prov * added prov test for multiple files * update daops * fixed inventory test
roocs · Feb 10, 2021 · f25e1df · f25e1df
1 parent cd3cade
commit f25e1df
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 13 deletions.
diff --git a/rook/provenance.py b/rook/provenance.py
@@ -94,13 +94,14 @@ def add_operator(self, operator, parameters, collection, output):
             self.doc.start(op, starter=self.sw_daops, trigger=self.sw_rook)
 
         # Generated output file
-        ds_out = os.path.basename(output[0])
-        # ds_out_attrs = {
-        #     'prov:type': 'provone:Data',
-        #     'prov:value': f'{ds_out}',
-        # }
-        op_out = self.doc.entity(f":{ds_out}")
-        self.doc.wasDerivedFrom(op_out, op_in, activity=op)
+        for out in output:
+            ds_out = os.path.basename(out)
+            # ds_out_attrs = {
+            #     'prov:type': 'provone:Data',
+            #     'prov:value': f'{ds_out}',
+            # }
+            op_out = self.doc.entity(f":{ds_out}")
+            self.doc.wasDerivedFrom(op_out, op_in, activity=op)
 
     def write_json(self):
         outfile = os.path.join(self.output_dir, "provenance.json")

diff --git a/setup.py b/setup.py
@@ -53,8 +53,8 @@
     install_requires=[
         reqs,
         "daops @ git+https://github.com/roocs/daops.git",
-        "clisops @ git+https://github.com/roocs/clisops.git",
-        "roocs-utils @ git+https://github.com/roocs/roocs-utils.git",
+        # "clisops @ git+https://github.com/roocs/clisops.git",
+        # "roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs-utils",
     ],
     extras_require={
         "dev": dev_reqs,  # pip install ".[dev]"

diff --git a/tests/common.py b/tests/common.py
@@ -17,14 +17,20 @@
 
 def write_roocs_cfg():
     cfg_templ = """
+    [clisops:write]
+    file_size_limit = 100KB
+
     [project:cmip5]
     base_dir = {{ base_dir }}/mini-esgf-data/test_data/badc/cmip5/data/cmip5
+    use_inventory = False
 
     [project:cmip6]
     base_dir = {{ base_dir }}/mini-esgf-data/test_data/badc/cmip6/data/CMIP6
+    use_inventory = False
 
     [project:cordex]
     base_dir = {{ base_dir }}/mini-esgf-data/test_data/badc/cordex/data/cordex
+    use_inventory = False
 
     [project:c3s-cmip5]
     base_dir = {{ base_dir }}/mini-esgf-data/test_data/gws/nopw/j04/cp4cds1_vol1/data/c3s-cmip5

diff --git a/tests/test_wps_subset.py b/tests/test_wps_subset.py
@@ -8,17 +8,22 @@
 from .common import PYWPS_CFG, get_output
 
 
-# Would expect this to raise an error
-def test_wps_subset_no_inv():
+def test_wps_subset_cmip6_no_inv():
     client = client_for(Service(processes=[Subset()], cfgfiles=[PYWPS_CFG]))
-    datainputs = "collection=c3s-cmip5.output1.ICHEC.EC-EARTH.historical.day.atmos.day.r1i1p1.tas.latest"
+    datainputs = "collection=c3s-cmip6.CMIP.MPI-M.MPI-ESM1-2-HR.historical.r1i1p1f1.SImon.siconc.gn.latest"
     datainputs += ";time=1860-01-01/1900-12-30"
     resp = client.get(
         "?service=WPS&request=Execute&version=1.0.0&identifier=subset&datainputs={}".format(
             datainputs
         )
     )
-    assert resp.status_code == 200
+    # assert resp.status_code == 200
+    # print(resp.data)
+    # assert_response_success(resp)
+    assert (
+        "Process error: Some or all of the requested collection are not in the list of available data"
+        in str(resp.data)
+    )
 
 
 def test_wps_subset_cmip6():
@@ -55,6 +60,32 @@ def test_wps_subset_cmip6_prov():
     )
 
 
+def test_wps_subset_cmip6_multiple_files_prov():
+    client = client_for(Service(processes=[Subset()], cfgfiles=[PYWPS_CFG]))
+    datainputs = "collection=CMIP6.CMIP.MPI-M.MPI-ESM1-2-HR.historical.r1i1p1f1.SImon.siconc.gn.latest"
+    datainputs += ";time=1850-01-01/2013-12-30"
+    resp = client.get(
+        "?service=WPS&request=Execute&version=1.0.0&identifier=subset&datainputs={}".format(
+            datainputs
+        )
+    )
+    assert_response_success(resp)
+    doc = prov.read(get_output(resp.xml)["prov"][len("file://"):])
+    print(doc.get_provn())
+    assert (
+        'activity(subset, -, -, [time="1850-01-01/2013-12-30", apply_fixes="0" %% xsd:boolean])'
+        in doc.get_provn()
+    )
+    assert (
+        'wasDerivedFrom(siconc_SImon_MPI-ESM1-2-HR_historical_r1i1p1f1_gn_18500116-18960316.nc, CMIP6.CMIP.MPI-M.MPI-ESM1-2-HR.historical.r1i1p1f1.SImon.siconc.gn.latest, subset, -, -)'  # noqa
+        in doc.get_provn()
+    )
+    assert (
+        'wasDerivedFrom(siconc_SImon_MPI-ESM1-2-HR_historical_r1i1p1f1_gn_18960416-19420616.nc, CMIP6.CMIP.MPI-M.MPI-ESM1-2-HR.historical.r1i1p1f1.SImon.siconc.gn.latest, subset, -, -)'  # noqa
+        in doc.get_provn()
+    )
+
+
 def test_wps_subset_cmip6_original_files():
     client = client_for(Service(processes=[Subset()], cfgfiles=[PYWPS_CFG]))
     datainputs = "collection=c3s-cmip6.CMIP.IPSL.IPSL-CM6A-LR.historical.r1i1p1f1.Amon.rlds.gr.v20180803"