Skip to content

Commit

Permalink
Enable s3express support in the transfer manager when CRT is enabled
Browse files Browse the repository at this point in the history
  • Loading branch information
nateprewitt committed Dec 21, 2023
1 parent 06cb1a7 commit f0b540d
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 3 deletions.
21 changes: 18 additions & 3 deletions s3transfer/crt.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,12 @@
import awscrt.s3
import botocore.awsrequest
import botocore.session
from awscrt.auth import AwsCredentials, AwsCredentialsProvider
from awscrt.auth import (
AwsCredentials,
AwsCredentialsProvider,
AwsSigningAlgorithm,
AwsSigningConfig,
)
from awscrt.io import (
ClientBootstrap,
ClientTlsContext,
Expand All @@ -35,7 +40,12 @@
from s3transfer.constants import MB
from s3transfer.exceptions import TransferNotDoneError
from s3transfer.futures import BaseTransferFuture, BaseTransferMeta
from s3transfer.utils import CallArgs, OSUtils, get_callbacks
from s3transfer.utils import (
CallArgs,
OSUtils,
get_callbacks,
is_s3express_bucket,
)

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -807,7 +817,7 @@ def _default_get_make_request_args(
on_done_before_calls,
on_done_after_calls,
):
return {
make_request_args = {
'request': self._request_serializer.serialize_http_request(
request_type, future
),
Expand All @@ -819,6 +829,11 @@ def _default_get_make_request_args(
),
'on_progress': self.get_crt_callback(future, 'progress'),
}
if is_s3express_bucket(call_args.bucket):
make_request_args['signing_config'] = AwsSigningConfig(
algorithm=AwsSigningAlgorithm.V4_S3EXPRESS
)
return make_request_args


class RenameTempFileHandler:
Expand Down
3 changes: 3 additions & 0 deletions tests/functional/test_crt.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ class TestCRTTransferManager(unittest.TestCase):
def setUp(self):
self.region = 'us-west-2'
self.bucket = "test_bucket"
self.s3express_bucket = 's3expressbucket--usw2-az5--x-s3'
self.key = "test_key"
self.expected_content = b'my content'
self.expected_download_content = b'new content'
Expand All @@ -77,6 +78,8 @@ def setUp(self):
)
self.expected_path = "/" + self.bucket + "/" + self.key
self.expected_host = "s3.%s.amazonaws.com" % (self.region)
self.expected_s3express_host = f'{self.s3express_bucket}.s3express-usw2-az5.us-west-2.amazonaws.com'
self.expected_s3express_path = f'/{self.key}'
self.s3_request = mock.Mock(awscrt.s3.S3Request)
self.s3_crt_client = mock.Mock(awscrt.s3.S3Client)
self.s3_crt_client.make_request.side_effect = (
Expand Down

0 comments on commit f0b540d

Please sign in to comment.