Skip to content

Commit

Permalink
Expose list/delete multi-part-uploads convenience (#645)
Browse files Browse the repository at this point in the history
* Expose list/delete multi-part-uploads convenience

* format
  • Loading branch information
martindurant committed Aug 30, 2022
1 parent 2195922 commit 353e086
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 2 deletions.
12 changes: 10 additions & 2 deletions s3fs/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -1681,9 +1681,15 @@ async def _cp_file(self, path1, path2, preserve_etag=None, **kwargs):
# serial multipart copy
await self._copy_managed(path1, path2, size, **kwargs)

async def _list_multipart_uploads(self, bucket):
out = await self._call_s3("list_multipart_uploads", Bucket=bucket)
return out.get("Contents", []) or out.get("Uploads", [])

list_multipart_uploads = sync_wrapper(_list_multipart_uploads)

async def _clear_multipart_uploads(self, bucket):
"""Remove any partial uploads in the bucket"""
out = await self._call_s3("list_multipart_uploads", Bucket=bucket)
out = await self._list_multipart_uploads(bucket)
await asyncio.gather(
*[
self._call_s3(
Expand All @@ -1692,10 +1698,12 @@ async def _clear_multipart_uploads(self, bucket):
Key=upload["Key"],
UploadId=upload["UploadId"],
)
for upload in out["Contents"]
for upload in out
]
)

clear_multipart_uploads = sync_wrapper(_clear_multipart_uploads)

async def _bulk_delete(self, pathlist, **kwargs):
"""
Remove multiple keys with one call
Expand Down
18 changes: 18 additions & 0 deletions s3fs/tests/test_s3fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -2374,3 +2374,21 @@ def test_exists_isdir(s3):
bad_path = "s3://nyc-tlc-asdfasdf/trip data/"
assert not s3.exists(bad_path)
assert not s3.isdir(bad_path)


def test_list_del_multipart(s3):
path = test_bucket_name + "/afile"
f = s3.open(path, "wb")
f.write(b"0" * 6 * 2**20)

out = s3.list_multipart_uploads(test_bucket_name)
assert [_ for _ in out if _["Key"] == "afile"]

s3.clear_multipart_uploads(test_bucket_name)
out = s3.list_multipart_uploads(test_bucket_name)
assert not [_ for _ in out if _["Key"] == "afile"]

try:
f.close() # may error
except Exception:
pass

0 comments on commit 353e086

Please sign in to comment.