From 5fc308a11284a7d23695a62b6a7106b04f2ec24c Mon Sep 17 00:00:00 2001 From: Austin Morton Date: Wed, 6 Mar 2024 01:12:39 +0000 Subject: [PATCH 1/3] Fix retrying incomplete downloads with urllib3 2.x --- s3transfer/__init__.py | 4 +++- s3transfer/utils.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/s3transfer/__init__.py b/s3transfer/__init__.py index d2a71911..b4137de5 100644 --- a/s3transfer/__init__.py +++ b/s3transfer/__init__.py @@ -135,7 +135,7 @@ def __call__(self, bytes_amount): import threading from botocore.compat import six # noqa: F401 -from botocore.exceptions import IncompleteReadError +from botocore.exceptions import IncompleteReadError, ResponseStreamingError from botocore.vendored.requests.packages.urllib3.exceptions import ( ReadTimeoutError, ) @@ -624,6 +624,7 @@ def _download_range( OSError, ReadTimeoutError, IncompleteReadError, + ResponseStreamingError, ) as e: logger.debug( "Retrying exception caught (%s), " @@ -840,6 +841,7 @@ def _get_object(self, bucket, key, filename, extra_args, callback): OSError, ReadTimeoutError, IncompleteReadError, + ResponseStreamingError, ) as e: # TODO: we need a way to reset the callback if the # download failed. diff --git a/s3transfer/utils.py b/s3transfer/utils.py index 9954dc0a..7370e5a8 100644 --- a/s3transfer/utils.py +++ b/s3transfer/utils.py @@ -21,7 +21,7 @@ import threading from collections import defaultdict -from botocore.exceptions import IncompleteReadError, ReadTimeoutError +from botocore.exceptions import IncompleteReadError, ReadTimeoutError, ResponseStreamingError from botocore.httpchecksum import AwsChunkedWrapper from botocore.utils import is_s3express_bucket @@ -41,6 +41,7 @@ SOCKET_ERROR, ReadTimeoutError, IncompleteReadError, + ResponseStreamingError, ) From c4d548cde3f185295e5bc684d606af167b48a585 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Wed, 13 Mar 2024 22:26:32 -0600 Subject: [PATCH 2/3] Linting fix --- s3transfer/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/s3transfer/utils.py b/s3transfer/utils.py index 7370e5a8..ef171f54 100644 --- a/s3transfer/utils.py +++ b/s3transfer/utils.py @@ -21,7 +21,11 @@ import threading from collections import defaultdict -from botocore.exceptions import IncompleteReadError, ReadTimeoutError, ResponseStreamingError +from botocore.exceptions import ( + IncompleteReadError, + ReadTimeoutError, + ResponseStreamingError, +) from botocore.httpchecksum import AwsChunkedWrapper from botocore.utils import is_s3express_bucket From d57aff533cca02186cc58d5eebafbf81728170ea Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Wed, 13 Mar 2024 22:29:49 -0600 Subject: [PATCH 3/3] Add changelog --- .changes/next-release/bugfix-urllib3-70927.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/next-release/bugfix-urllib3-70927.json diff --git a/.changes/next-release/bugfix-urllib3-70927.json b/.changes/next-release/bugfix-urllib3-70927.json new file mode 100644 index 00000000..96ef2e90 --- /dev/null +++ b/.changes/next-release/bugfix-urllib3-70927.json @@ -0,0 +1,5 @@ +{ + "type": "bugfix", + "category": "``urllib3``", + "description": "Fixed retry handling for IncompleteRead exception raised by urllib3 2.x during data transfer" +}