fix(datasets): strip query string from data filenames

When adding data to a dataset from a URL which includes a query string, this query string used to be present in the filenaeme. This is now fixed.

Author: Andreas Bleuler

conftest.py

@@ -156,6 +156,11 @@ def request_callback(request):
             'https://example.com/file',
             callback=request_callback
         )
+        rsps.add_callback(
+            responses.GET,
+            'http://example.com/file.ext?foo=bar',
+            callback=request_callback
+        )
         yield rsps
 
 

renku/api/datasets.py

@@ -189,11 +189,11 @@ def _add_from_url(self, dataset, path, url, link=False, **kwargs):
         # Respect the directory struture inside the source path.
         relative_to = kwargs.pop('relative_to', None)
         if relative_to:
-            dst_path = Path(url).resolve().absolute().relative_to(
+            dst_path = Path(u.path).resolve().absolute().relative_to(
                 Path(relative_to).resolve().absolute()
             )
         else:
-            dst_path = os.path.basename(url)
+            dst_path = os.path.basename(u.path)
 
         dst = path.joinpath(dst_path).absolute()
 

tests/cli/test_datasets.py

@@ -75,6 +75,15 @@ def test_datasets_import(data_file, data_repository, runner, project, client):
     )
     assert 0 == result.exit_code
 
+    # add data from any URL (not a git repo)
+    result = runner.invoke(
+        cli.cli,
+        ['dataset', 'add', 'dataset', 'http://example.com/file.ext?foo=bar'],
+        catch_exceptions=False,
+    )
+    assert 0 == result.exit_code
+    assert os.stat('data/dataset/file.ext')
+
 
 @pytest.mark.parametrize('output_format', DATASETS_FORMATS.keys())
 def test_datasets_list_empty(output_format, runner, project):