From d96c77d6ec51ebaa6f788dcf6269ed25fa094b71 Mon Sep 17 00:00:00 2001 From: aschroed Date: Thu, 7 Sep 2023 16:13:24 -0400 Subject: [PATCH 1/4] trying to allow something of a corner case to upload extra_files provided by users where the file is already uploaded --- wranglertools/import_data.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/wranglertools/import_data.py b/wranglertools/import_data.py index eba25d48..61b744db 100755 --- a/wranglertools/import_data.py +++ b/wranglertools/import_data.py @@ -894,7 +894,9 @@ def update_item(verb, file_to_upload, post_json, filename_to_post, extrafiles, c if ftp_download: pp.Path(filename_to_post).unlink() if extrafiles: - extcreds = e['@graph'][0].get('extra_files_creds') + extcreds = e['@graph'][0].get('extra_file_creds') + if not extcreds: + extcreds = get_upload_creds(e['@graph'][0]['accession'], connection, extfilecreds=True) for fformat, filepath in extrafiles.items(): try: file_format = ff_utils.get_metadata(fformat, key=connection.key) @@ -909,8 +911,8 @@ def update_item(verb, file_to_upload, post_json, filename_to_post, extrafiles, c def patch_item(file_to_upload, post_json, filename_to_post, extrafiles, connection, existing_data): - return update_item('PATCH', file_to_upload, post_json, filename_to_post, - extrafiles, connection, existing_data.get('uuid')) + return update_item('PATCH', file_to_upload, post_json, filename_to_post, extrafiles, + connection, existing_data.get('uuid')) def post_item(file_to_upload, post_json, filename_to_post, extrafiles, connection, sheet): @@ -1430,10 +1432,15 @@ def user_workflow_reader(workbook, sheet, connection): error=error, patch="-", not_patched="-")) -def get_upload_creds(file_id, connection): # pragma: no cover +def get_upload_creds(file_id, connection, extfilecreds=False): # pragma: no cover + creds2return = 'upload_credentials' url = f"{file_id}/upload/" - req = ff_utils.post_metadata({}, url, key=connection.key) - return req['@graph'][0]['upload_credentials'] + if extfilecreds: + creds2return = 'extra_files_creds' + req = ff_utils.authorized_request(f"{connection.key.get('server')}/{url}", auth=ff_utils.get_authentication_with_server(connection.key)).json() + else: + req = ff_utils.post_metadata({}, f"{file_id}{stem}", key=connection.key) + return req['@graph'][0][creds2return] def upload_file_item(metadata_post_response, path): From 8aeef1705c69bb77b1ef6c780d5ecbb050c0647c Mon Sep 17 00:00:00 2001 From: aschroed Date: Fri, 8 Sep 2023 14:16:13 -0400 Subject: [PATCH 2/4] fixed the failing test --- tests/test_import_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_import_data.py b/tests/test_import_data.py index 1b2e4d19..8340ebb6 100644 --- a/tests/test_import_data.py +++ b/tests/test_import_data.py @@ -1571,7 +1571,10 @@ def pf_w_extfiles_resp(): def test_update_item_extrafiles(mocker, connection_mock, pf_w_extfiles_resp): extrafiles = {'pairs_px2': '/test/file/test_pairs.gz.px2', 'pairsam_px2': '/test/file/testfile.pairs.sam.gz'} + upld_creds = [{'file_format': 'pairs_px2', 'upload_credentials': 'px2creds'}, + {'file_format': 'pairsam_px2', 'upload_credentials': 'px2creds'}] mocker.patch('wranglertools.import_data.ff_utils.post_metadata', return_value=pf_w_extfiles_resp) + mocker.patch('wranglertools.import_data.get_upload_creds', return_value=upld_creds) mocker.patch('wranglertools.import_data.upload_extra_file', side_effect=[None, None]) mocker.patch('wranglertools.import_data.ff_utils.get_metadata', side_effect=[ {'uuid': 'd13d06cf-218e-4f61-aaf0-91f226348b2c'}, {'uuid': 'd13d06cf-218e-6f61-aaf0-91f226248b2c'} From 5895423f2a17ce1b295c3a15942b1967ca15f289 Mon Sep 17 00:00:00 2001 From: aschroed Date: Tue, 12 Sep 2023 15:12:09 -0400 Subject: [PATCH 3/4] added a wait needed to ensure extracreds are there --- wranglertools/import_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wranglertools/import_data.py b/wranglertools/import_data.py index 61b744db..1ff25cfd 100755 --- a/wranglertools/import_data.py +++ b/wranglertools/import_data.py @@ -896,6 +896,7 @@ def update_item(verb, file_to_upload, post_json, filename_to_post, extrafiles, c if extrafiles: extcreds = e['@graph'][0].get('extra_file_creds') if not extcreds: + time.sleep(5) extcreds = get_upload_creds(e['@graph'][0]['accession'], connection, extfilecreds=True) for fformat, filepath in extrafiles.items(): try: From 2d301b0d3b26957bd7c9aa32af6d080fb970c4dd Mon Sep 17 00:00:00 2001 From: aschroed Date: Tue, 12 Sep 2023 15:18:26 -0400 Subject: [PATCH 4/4] version bump; changelog --- CHANGELOG.rst | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c4a9fb4d..0db5f19d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,13 @@ Submit4DN Change Log ---------- +3.4.1 +===== + +`PR 170: fix bug in extra file upld `_ + +* Update to allow extra_files to be uploaded even if the regular file was already uploaded (which was previously stymied by permission denied for POST to get extracreds) + 3.4.0 ===== diff --git a/pyproject.toml b/pyproject.toml index 33dbb11b..3678a331 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "Submit4DN" -version = "3.4.0" +version = "3.4.1" description = "Utility package for submitting data to the 4DN Data Portal" authors = ["4DN-DCIC Team "] license = "MIT"