Merge pull request #170 from 4dn-dcic/ajs_fix_efile_upld

allow extra_file upload.

CHANGELOG.rst

@@ -6,6 +6,13 @@ Submit4DN
 Change Log
 ----------
 
+3.4.1
+=====
+
+`PR 170: fix bug in extra file upld <https://github.com/4dn-dcic/Submit4DN/pull/170>`_
+
+* Update to allow extra_files to be uploaded even if the regular file was already uploaded (which was previously stymied by permission denied for POST to get extracreds)
+
 3.4.0
 =====
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "Submit4DN"
-version = "3.4.0"
+version = "3.4.1"
 description = "Utility package for submitting data to the 4DN Data Portal"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

tests/test_import_data.py

@@ -1571,7 +1571,10 @@ def pf_w_extfiles_resp():
 
 def test_update_item_extrafiles(mocker, connection_mock, pf_w_extfiles_resp):
     extrafiles = {'pairs_px2': '/test/file/test_pairs.gz.px2', 'pairsam_px2': '/test/file/testfile.pairs.sam.gz'}
+    upld_creds = [{'file_format': 'pairs_px2', 'upload_credentials': 'px2creds'},
+                  {'file_format': 'pairsam_px2', 'upload_credentials': 'px2creds'}]
     mocker.patch('wranglertools.import_data.ff_utils.post_metadata', return_value=pf_w_extfiles_resp)
+    mocker.patch('wranglertools.import_data.get_upload_creds', return_value=upld_creds)
     mocker.patch('wranglertools.import_data.upload_extra_file', side_effect=[None, None])
     mocker.patch('wranglertools.import_data.ff_utils.get_metadata', side_effect=[
         {'uuid': 'd13d06cf-218e-4f61-aaf0-91f226348b2c'}, {'uuid': 'd13d06cf-218e-6f61-aaf0-91f226248b2c'}

wranglertools/import_data.py

@@ -894,7 +894,10 @@ def update_item(verb, file_to_upload, post_json, filename_to_post, extrafiles, c
         if ftp_download:
             pp.Path(filename_to_post).unlink()
     if extrafiles:
-        extcreds = e['@graph'][0].get('extra_files_creds')
+        extcreds = e['@graph'][0].get('extra_file_creds')
+        if not extcreds:
+            time.sleep(5)
+            extcreds = get_upload_creds(e['@graph'][0]['accession'], connection, extfilecreds=True)
         for fformat, filepath in extrafiles.items():
             try:
                 file_format = ff_utils.get_metadata(fformat, key=connection.key)
@@ -909,8 +912,8 @@ def update_item(verb, file_to_upload, post_json, filename_to_post, extrafiles, c
 
 
 def patch_item(file_to_upload, post_json, filename_to_post, extrafiles, connection, existing_data):
-    return update_item('PATCH', file_to_upload, post_json, filename_to_post,
-                       extrafiles, connection, existing_data.get('uuid'))
+    return update_item('PATCH', file_to_upload, post_json, filename_to_post, extrafiles,
+                       connection, existing_data.get('uuid'))
 
 
 def post_item(file_to_upload, post_json, filename_to_post, extrafiles, connection, sheet):
@@ -1430,10 +1433,15 @@ def user_workflow_reader(workbook, sheet, connection):
                   error=error, patch="-", not_patched="-"))
 
 
-def get_upload_creds(file_id, connection):  # pragma: no cover
+def get_upload_creds(file_id, connection, extfilecreds=False):  # pragma: no cover
+    creds2return = 'upload_credentials'
     url = f"{file_id}/upload/"
-    req = ff_utils.post_metadata({}, url, key=connection.key)
-    return req['@graph'][0]['upload_credentials']
+    if extfilecreds:
+        creds2return = 'extra_files_creds'
+        req = ff_utils.authorized_request(f"{connection.key.get('server')}/{url}", auth=ff_utils.get_authentication_with_server(connection.key)).json()
+    else:
+        req = ff_utils.post_metadata({}, f"{file_id}{stem}", key=connection.key)
+    return req['@graph'][0][creds2return]
 
 
 def upload_file_item(metadata_post_response, path):