From ba56c5685597d2c8410a348a0b2d8d63a105dded Mon Sep 17 00:00:00 2001 From: aschroed Date: Tue, 27 Sep 2022 15:29:53 -0400 Subject: [PATCH 1/4] change handling of paths and the actual aws call taken from SubmitCGAP --- tests/test_import_data.py | 3 +++ wranglertools/import_data.py | 23 +++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/test_import_data.py b/tests/test_import_data.py index 33963e1a..c93bff25 100644 --- a/tests/test_import_data.py +++ b/tests/test_import_data.py @@ -1556,3 +1556,6 @@ def test_get_collections(mock_profiles): colls = imp.get_collections(mock_profiles) for c in mock_profiles.keys(): assert c.lower() in colls + + + diff --git a/wranglertools/import_data.py b/wranglertools/import_data.py index 43491306..e72eb3a0 100755 --- a/wranglertools/import_data.py +++ b/wranglertools/import_data.py @@ -590,7 +590,8 @@ def build_patch_json(fields, fields2types): def get_just_filename(path): - return path.split('/')[-1] + return pp.Path(path).name + # return path.split('/')[-1] def check_extra_file_meta(ef_info, seen_formats, existing_formats): @@ -1452,15 +1453,25 @@ def upload_file(creds, path): # pragma: no cover print("Uploading file.") start = time.time() try: - subprocess.check_call(['aws', 's3', 'cp', '--only-show-errors', path, creds['upload_url']], env=env) + source = path + target = creds['upload_url'] + print("Going to upload {} to {}.".format(source, target)) + command = ['aws', 's3', 'cp'] + command = command + ['--only-show-errors', source, target] + options = {} + if running_on_windows_native(): + options = {"shell": True} + subprocess.check_call(command, env=env, **options) except subprocess.CalledProcessError as e: - # The aws command returns a non-zero exit code on error. - print("Upload failed with exit code %d" % e.returncode) - sys.exit(e.returncode) + raise RuntimeError("Upload failed with exit code %d" % e.returncode) else: end = time.time() duration = end - start - print("Uploaded in %.2f seconds" % duration) + show("Uploaded in %.2f seconds" % duration) + + +def running_on_windows_native(): + return os.name == 'nt' # the order to try to upload / update the items From 7a4080d9b681e8eb888c5bccd2c85ca3b8f6dee6 Mon Sep 17 00:00:00 2001 From: aschroed Date: Tue, 27 Sep 2022 16:07:15 -0400 Subject: [PATCH 2/4] add simple test --- tests/test_import_data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_import_data.py b/tests/test_import_data.py index c93bff25..3ab349a8 100644 --- a/tests/test_import_data.py +++ b/tests/test_import_data.py @@ -1558,4 +1558,10 @@ def test_get_collections(mock_profiles): assert c.lower() in colls +def test_get_just_filename_unix(): + test_path = '/Files/test/test_file.fastq.gz' + filename = imp.get_just_filename(test_path) + assert filename == 'test_file.fastq.gz' + +# how to test windows paths?? From fb7e00b8f7f79ef68ece4273a0ef2c2d964b12e7 Mon Sep 17 00:00:00 2001 From: andreacosolo Date: Wed, 28 Sep 2022 11:02:00 +0200 Subject: [PATCH 3/4] update posix and windows paths tests --- .gitignore | 1 + tests/test_import_data.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index f23e7a61..1b5be302 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ AllItems.xls Custom_scripts/ Data_Files/MicroscopyCalibration/Files/ .pytest_cache/ +.python-version diff --git a/tests/test_import_data.py b/tests/test_import_data.py index 3ab349a8..ac5fba16 100644 --- a/tests/test_import_data.py +++ b/tests/test_import_data.py @@ -1,5 +1,6 @@ import wranglertools.import_data as imp import pytest +import pathlib as pp # test data is in conftest.py @@ -1558,10 +1559,13 @@ def test_get_collections(mock_profiles): assert c.lower() in colls -def test_get_just_filename_unix(): - test_path = '/Files/test/test_file.fastq.gz' +def test_get_just_filename_posix(): + test_path = pp.PurePosixPath('Users', 'username', 'test_dir', 'test_file.fastq.gz') filename = imp.get_just_filename(test_path) assert filename == 'test_file.fastq.gz' -# how to test windows paths?? +def test_get_just_filename_windows(): + test_path = pp.PureWindowsPath('c:/', 'Users', 'username', 'test_dir', 'test_file.fastq.gz') + filename = imp.get_just_filename(test_path) + assert filename == 'test_file.fastq.gz' From 5d32eef095dcd100b8bdd4d451858e638dbb4e4e Mon Sep 17 00:00:00 2001 From: andreacosolo Date: Wed, 28 Sep 2022 11:16:56 +0200 Subject: [PATCH 4/4] bump patch version --- CHANGELOG.rst | 8 ++++++-- pyproject.toml | 2 +- wranglertools/import_data.py | 1 - 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 76c16365..b7e02776 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Change Log ---------- +3.0.1 +===== + +* Bug fix: Windows paths were not handled properly for File upload. + 3.0.0 ======= @@ -14,7 +19,7 @@ Change Log * Drop support for Python3.6 -* Add this CHANGELOG and test warning if it's not updated +* Add this CHANGELOG and test warning if it's not updated * Update dependency to use dcicutils >=4.0 @@ -212,4 +217,3 @@ Change Log 0.2.2 ===== - diff --git a/pyproject.toml b/pyproject.toml index 7fb6b74e..4e50fd3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "Submit4DN" -version = "3.0.0" +version = "3.0.1" description = "Utility package for submitting data to the 4DN Data Portal" authors = ["4DN-DCIC Team "] license = "MIT" diff --git a/wranglertools/import_data.py b/wranglertools/import_data.py index e72eb3a0..11716b24 100755 --- a/wranglertools/import_data.py +++ b/wranglertools/import_data.py @@ -591,7 +591,6 @@ def build_patch_json(fields, fields2types): def get_just_filename(path): return pp.Path(path).name - # return path.split('/')[-1] def check_extra_file_meta(ef_info, seen_formats, existing_formats):