From babeff4d95f631c34a2b0888794dfe0b7581c8d6 Mon Sep 17 00:00:00 2001 From: Bryan Date: Fri, 15 May 2020 19:08:54 +0800 Subject: [PATCH] raise Exception if delete_objects failed --- awswrangler/s3.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/awswrangler/s3.py b/awswrangler/s3.py index a6de341b4..58247816e 100644 --- a/awswrangler/s3.py +++ b/awswrangler/s3.py @@ -19,6 +19,7 @@ from boto3.s3.transfer import TransferConfig # type: ignore from pandas.io.common import infer_compression # type: ignore + from awswrangler import _data_types, _utils, catalog, exceptions _COMPRESSION_2_EXT: Dict[Optional[str], str] = {None: "", "gzip": ".gz", "snappy": ".snappy"} @@ -289,7 +290,26 @@ def _split_paths_by_bucket(paths: List[str]) -> Dict[str, List[str]]: def _delete_objects(bucket: str, keys: List[str], client_s3: boto3.client) -> None: _logger.debug("len(keys): %s", len(keys)) batch: List[Dict[str, str]] = [{"Key": key} for key in keys] - client_s3.delete_objects(Bucket=bucket, Delete={"Objects": batch}) + res = client_s3.delete_objects(Bucket=bucket, Delete={"Objects": batch}) + deleted = res.get('Deleted') + errors = res.get('Errors') + if errors: + for i in errors: + raise exceptions.ServiceApiError( + 'Path: s3://{}/{}\n' + 'Error Code: {}\n' + 'Message: {}'.format( + bucket, + i.get('Key'), + i.get('Code'), + i.get('Message') + ) + ) + else: + for i in deleted: + _logger.debug( + 's3://{}/{} has been deleted'.format(bucket, i.get('Key')) + ) def describe_objects(