Merge pull request #96 from roocs/fix-download

fix build_metalink for urls returned by workflow
roocs · Jan 25, 2021 · 2127a61 · 2127a61
2 parents f402f51 + bb5d632
commit 2127a61
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 7 deletions.
diff --git a/rook/processes/wps_orchestrate.py b/rook/processes/wps_orchestrate.py
@@ -83,8 +83,7 @@ def _handler(self, request, response):
         # )
 
         ml4 = build_metalink("workflow-result", "Workflow result as NetCDF files.",
-                             self.workdir, file_uris,
-                             as_urls=False)
+                             self.workdir, file_uris)
 
         # for ncfile in output:
         #     mf = MetaFile("NetCDF file", "NetCDF file", fmt=FORMATS.NETCDF)

diff --git a/rook/processes/wps_subset.py b/rook/processes/wps_subset.py
@@ -151,8 +151,7 @@ def _handler(self, request, response):
         director = wrap_director(collection, inputs, run_subset)
 
         ml4 = build_metalink("subset-result", "Subsetting result as NetCDF files.",
-                             self.workdir, director.output_uris,
-                             as_urls=director.use_original_files)
+                             self.workdir, director.output_uris)
 
         populate_response(response, 'subset', self.workdir, inputs, collection, ml4)
         return response
diff --git a/rook/utils/metalink_utils.py b/rook/utils/metalink_utils.py
@@ -1,3 +1,5 @@
+from urllib.parse import urlparse
+
 from pywps.inout.outputs import MetaFile, MetaLink4
 from pywps import FORMATS
 
@@ -6,8 +8,7 @@
 }
 
 
-def build_metalink(identity, description, workdir, file_uris, file_type='NetCDF',
-                   as_urls=False):
+def build_metalink(identity, description, workdir, file_uris, file_type='NetCDF'):
 
     ml4 = MetaLink4(identity, description, workdir=workdir)
     file_desc = f'{file_type} file'
@@ -16,7 +17,7 @@ def build_metalink(identity, description, workdir, file_uris, file_type='NetCDF'
     for file_uri in file_uris:
         mf = MetaFile(file_desc, file_desc, fmt=file_type_map.get(file_type, file_type))
 
-        if as_urls:
+        if urlparse(file_uri).scheme in ['http', 'https']:
             mf.url = file_uri
         else:
             mf.file = file_uri

diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -0,0 +1,21 @@
+import os
+
+from rook.utils.metalink_utils import build_metalink
+
+from .common import TESTS_HOME
+
+
+def test_build_metalink(tmpdir):
+    cmip6_nc = os.path.join(TESTS_HOME, 'mini-esgf-data/test_data/badc/cmip6/data/CMIP6/CMIP/MPI-M/MPI-ESM1-2-HR/historical/r1i1p1f1/SImon/siconc/gn/latest/siconc_SImon_MPI-ESM1-2-HR_historical_r1i1p1f1_gn_185001-185412.nc')  # noqa
+    ml4 = build_metalink(
+        "workflow-result",
+        "Workflow result as NetCDF files.",
+        tmpdir,
+        [
+            'https://data.mips.copernicus-climate.eu/thredds/fileServer/esg_c3s-cmip6/CMIP/NCC/NorESM1-F/piControl/r1i1p1f1/Amon/rsdt/gn/v20190920/rsdt_Amon_NorESM1-F_piControl_r1i1p1f1_gn_150101-151012.nc',  # noqa
+            'http://data.mips.copernicus-climate.eu/thredds/fileServer/esg_c3s-cmip6/CMIP/NCC/NorESM1-F/piControl/r1i1p1f1/Amon/rsdt/gn/v20190920/rsdt_Amon_NorESM1-F_piControl_r1i1p1f1_gn_150101-151012.nc',  # noqa
+            cmip6_nc
+        ])
+    assert 'https://data.mips.copernicus-climate.eu' in ml4.files[0].url
+    assert 'http://data.mips.copernicus-climate.eu' in ml4.files[1].url
+    assert 'file://' in ml4.files[2].url