diff --git a/awscrt/s3.py b/awscrt/s3.py index 73c008ba4..2738f2d4a 100644 --- a/awscrt/s3.py +++ b/awscrt/s3.py @@ -10,7 +10,7 @@ from awscrt import NativeResource from awscrt.http import HttpRequest from awscrt.io import ClientBootstrap, TlsConnectionOptions -from awscrt.auth import AwsCredentialsProvider +from awscrt.auth import AwsCredentialsProvider, AwsSignatureType, AwsSignedBodyHeaderType, AwsSignedBodyValue, AwsSigningAlgorithm, AwsSigningConfig import awscrt.exceptions import threading from enum import IntEnum @@ -68,8 +68,12 @@ class S3Client(NativeResource): If this is :attr:`S3RequestTlsMode.DISABLED`: No TLS options will be used, regardless of `tls_connection_options` value. - credential_provider (Optional[AwsCredentialsProvider]): Credentials providers source the - :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request. + signing_config (Optional[AwsSigningConfig]): + Configuration for signing of the client. Use :func:`create_default_s3_signing_config()` to create the default config. + If None is provided, the request will not be signed. + + credential_provider (Optional[AwsCredentialsProvider]): Deprecated, prefer `signing_config` instead. + Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request. If None is provided, the request will not be signed. tls_connection_options (Optional[TlsConnectionOptions]): Optional TLS Options to be used @@ -91,12 +95,14 @@ def __init__( bootstrap, region, tls_mode=None, + signing_config=None, credential_provider=None, tls_connection_options=None, part_size=None, throughput_target_gbps=None): assert isinstance(bootstrap, ClientBootstrap) or bootstrap is None assert isinstance(region, str) + assert isinstance(signing_config, AwsSigningConfig) or signing_config is None assert isinstance(credential_provider, AwsCredentialsProvider) or credential_provider is None assert isinstance(tls_connection_options, TlsConnectionOptions) or tls_connection_options is None assert isinstance(part_size, int) or part_size is None @@ -106,6 +112,10 @@ def __init__( throughput_target_gbps, float) or throughput_target_gbps is None + if credential_provider and signing_config: + raise ValueError("'credential_provider' has been deprecated in favor of 'signing_config'. " + "Both parameters may not be set.") + super().__init__() shutdown_event = threading.Event() @@ -117,7 +127,8 @@ def on_shutdown(): if not bootstrap: bootstrap = ClientBootstrap.get_or_create_static_default() - s3_client_core = _S3ClientCore(bootstrap, credential_provider, tls_connection_options) + + s3_client_core = _S3ClientCore(bootstrap, credential_provider, signing_config, tls_connection_options) # C layer uses 0 to indicate defaults if tls_mode is None: @@ -129,6 +140,7 @@ def on_shutdown(): self._binding = _awscrt.s3_client_new( bootstrap, + signing_config, credential_provider, tls_connection_options, on_shutdown, @@ -143,6 +155,7 @@ def make_request( *, request, type, + signing_config=None, credential_provider=None, recv_filepath=None, send_filepath=None, @@ -161,9 +174,13 @@ def make_request( type (S3RequestType): The type of S3 request passed in, :attr:`~S3RequestType.GET_OBJECT`/:attr:`~S3RequestType.PUT_OBJECT` can be accelerated - credential_provider (Optional[AwsCredentialsProvider]): Credentials providers source the - :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request, for this request only. - If None is provided, the credential provider in the client will be used. + signing_config (Optional[AwsSigningConfig]): + Configuration for signing of the request to override the configuration from client. Use :func:`create_default_s3_signing_config()` to create the default config. + If None is provided, the client configuration will be used. + + credential_provider (Optional[AwsCredentialsProvider]): Deprecated, prefer `signing_config` instead. + Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request, for this request only. + If None is provided, the client configuration will be used. recv_filepath (Optional[str]): Optional file path. If set, the response body is written directly to a file and the @@ -229,6 +246,7 @@ def make_request( client=self, request=request, type=type, + signing_config=signing_config, credential_provider=credential_provider, recv_filepath=recv_filepath, send_filepath=send_filepath, @@ -261,6 +279,7 @@ def __init__( client, request, type, + signing_config=None, credential_provider=None, recv_filepath=None, send_filepath=None, @@ -284,6 +303,7 @@ def __init__( request, self._finished_future, self.shutdown_event, + signing_config, credential_provider, on_headers, on_body, @@ -295,6 +315,7 @@ def __init__( client, request, type, + signing_config, credential_provider, recv_filepath, send_filepath, @@ -316,9 +337,11 @@ class _S3ClientCore: def __init__(self, bootstrap, credential_provider=None, + signing_config=None, tls_connection_options=None): self._bootstrap = bootstrap self._credential_provider = credential_provider + self._signing_config = signing_config self._tls_connection_options = tls_connection_options @@ -332,6 +355,7 @@ def __init__( request, finish_future, shutdown_event, + signing_config=None, credential_provider=None, on_headers=None, on_body=None, @@ -339,6 +363,7 @@ def __init__( on_progress=None): self._request = request + self._signing_config = signing_config self._credential_provider = credential_provider self._on_headers_cb = on_headers @@ -377,3 +402,30 @@ def _on_finish(self, error_code, error_headers, error_body): def _on_progress(self, progress): if self._on_progress_cb: self._on_progress_cb(progress) + + +def create_default_s3_signing_config(*, region: str, credential_provider: AwsCredentialsProvider, **kwargs): + """Create a default `AwsSigningConfig` for S3 service. + + Attributes: + region (str): The region to sign against. + + credential_provider (AwsCredentialsProvider): Credentials provider + to fetch signing credentials with. + + `**kwargs`: Forward compatibility kwargs. + + Returns: + AwsSigningConfig + """ + return AwsSigningConfig( + algorithm=AwsSigningAlgorithm.V4, + signature_type=AwsSignatureType.HTTP_REQUEST_HEADERS, + service="s3", + signed_body_header_type=AwsSignedBodyHeaderType.X_AMZ_CONTENT_SHA_256, + signed_body_value=AwsSignedBodyValue.UNSIGNED_PAYLOAD, + region=region, + credentials_provider=credential_provider, + use_double_uri_encode=False, + should_normalize_uri_path=False, + ) diff --git a/source/s3_client.c b/source/s3_client.c index 02d7789f6..f7a558f5a 100644 --- a/source/s3_client.c +++ b/source/s3_client.c @@ -72,6 +72,7 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) { struct aws_allocator *allocator = aws_py_get_allocator(); PyObject *bootstrap_py = NULL; + PyObject *signing_config_py = NULL; PyObject *credential_provider_py = NULL; PyObject *tls_options_py = NULL; PyObject *on_shutdown_py = NULL; @@ -83,8 +84,9 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) { int tls_mode; if (!PyArg_ParseTuple( args, - "OOOOs#iKdO", + "OOOOOs#iKdO", &bootstrap_py, + &signing_config_py, &credential_provider_py, &tls_options_py, &on_shutdown_py, @@ -109,14 +111,22 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) { return NULL; } } - - struct aws_signing_config_aws signing_config; - AWS_ZERO_STRUCT(signing_config); + struct aws_signing_config_aws *signing_config = NULL; + if (signing_config_py != Py_None) { + signing_config = aws_py_get_signing_config(signing_config_py); + if (!signing_config) { + return NULL; + } + } + struct aws_signing_config_aws signing_config_from_credentials_provider; + AWS_ZERO_STRUCT(signing_config_from_credentials_provider); struct aws_byte_cursor region_cursor = aws_byte_cursor_from_array((const uint8_t *)region, region_len); if (credential_provider) { - aws_s3_init_default_signing_config(&signing_config, region_cursor, credential_provider); + aws_s3_init_default_signing_config( + &signing_config_from_credentials_provider, region_cursor, credential_provider); + signing_config = &signing_config_from_credentials_provider; } struct aws_tls_connection_options *tls_options = NULL; @@ -150,7 +160,7 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) { .region = aws_byte_cursor_from_array((const uint8_t *)region, region_len), .client_bootstrap = bootstrap, .tls_mode = tls_mode, - .signing_config = credential_provider ? &signing_config : NULL, + .signing_config = signing_config, .part_size = part_size, .tls_connection_options = tls_options, .throughput_target_gbps = throughput_target_gbps, diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c index f7ffa56bd..d164d4460 100644 --- a/source/s3_meta_request.c +++ b/source/s3_meta_request.c @@ -476,6 +476,7 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) { PyObject *s3_client_py = NULL; PyObject *http_request_py = NULL; int type; + PyObject *signing_config_py = NULL; PyObject *credential_provider_py = NULL; const char *recv_filepath; const char *send_filepath; @@ -484,11 +485,12 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) { PyObject *py_core = NULL; if (!PyArg_ParseTuple( args, - "OOOiOzzs#O", + "OOOiOOzzs#O", &py_s3_request, &s3_client_py, &http_request_py, &type, + &signing_config_py, &credential_provider_py, &recv_filepath, &send_filepath, @@ -507,6 +509,14 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) { return NULL; } + struct aws_signing_config_aws *signing_config = NULL; + if (signing_config_py != Py_None) { + signing_config = aws_py_get_signing_config(signing_config_py); + if (!signing_config) { + return NULL; + } + } + struct aws_credentials_provider *credential_provider = NULL; if (credential_provider_py != Py_None) { credential_provider = aws_py_get_credentials_provider(credential_provider_py); @@ -515,11 +525,13 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) { } } - struct aws_signing_config_aws signing_config; - AWS_ZERO_STRUCT(signing_config); + struct aws_signing_config_aws signing_config_from_credentials_provider; + AWS_ZERO_STRUCT(signing_config_from_credentials_provider); if (credential_provider) { struct aws_byte_cursor region_cursor = aws_byte_cursor_from_array((const uint8_t *)region, region_len); - aws_s3_init_default_signing_config(&signing_config, region_cursor, credential_provider); + aws_s3_init_default_signing_config( + &signing_config_from_credentials_provider, region_cursor, credential_provider); + signing_config = &signing_config_from_credentials_provider; } struct s3_meta_request_binding *meta_request = aws_mem_calloc(allocator, 1, sizeof(struct s3_meta_request_binding)); @@ -566,7 +578,7 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) { struct aws_s3_meta_request_options s3_meta_request_opt = { .type = type, .message = meta_request->copied_message ? meta_request->copied_message : http_request, - .signing_config = credential_provider ? &signing_config : NULL, + .signing_config = signing_config, .headers_callback = s_s3_request_on_headers, .body_callback = s_s3_request_on_body, .finish_callback = s_s3_request_on_finish, diff --git a/test/test_s3.py b/test/test_s3.py index 3e5574776..5de4a8cd6 100644 --- a/test/test_s3.py +++ b/test/test_s3.py @@ -10,9 +10,9 @@ from concurrent.futures import Future from awscrt.http import HttpHeaders, HttpRequest -from awscrt.s3 import S3Client, S3RequestType +from awscrt.s3 import S3Client, S3RequestType, create_default_s3_signing_config from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions -from awscrt.auth import AwsCredentialsProvider +from awscrt.auth import AwsCredentialsProvider, AwsSignatureType, AwsSignedBodyHeaderType, AwsSignedBodyValue, AwsSigningAlgorithm, AwsSigningConfig MB = 1024 ** 2 GB = 1024 ** 3 @@ -74,6 +74,7 @@ def s3_client_new(secure, region, part_size=0): host_resolver = DefaultHostResolver(event_loop_group) bootstrap = ClientBootstrap(event_loop_group, host_resolver) credential_provider = AwsCredentialsProvider.new_default_chain(bootstrap) + signing_config = create_default_s3_signing_config(region=region, credential_provider=credential_provider) tls_option = None if secure: opt = TlsContextOptions() @@ -83,7 +84,7 @@ def s3_client_new(secure, region, part_size=0): s3_client = S3Client( bootstrap=bootstrap, region=region, - credential_provider=credential_provider, + signing_config=signing_config, tls_connection_options=tls_option, part_size=part_size) @@ -102,7 +103,6 @@ def read(self, length): return fake_data -@unittest.skipUnless(os.environ.get('AWS_TEST_S3'), 'set env var to run test: AWS_TEST_S3') class S3ClientTest(NativeResourceTest): def setUp(self): @@ -137,6 +137,7 @@ def setUp(self): self.bucket_name = "aws-crt-canary-bucket" self.timeout = 100 # seconds self.num_threads = 0 + self.special_path = "put_object_test_10MB@$%.txt" self.non_ascii_file_name = "ÉxÅmple.txt".encode("utf-8") self.response_headers = None @@ -439,6 +440,50 @@ def test_multipart_upload_with_invalid_request(self): self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT, "AWS_ERROR_S3_INVALID_RESPONSE_STATUS") self.put_body_stream.close() + def test_special_filepath_upload(self): + # remove the input file when request done + with open(self.special_path, 'wb') as file: + file.write(b"a" * 10 * MB) + request = self._put_object_request(self.special_path) + self.put_body_stream.close() + s3_client = s3_client_new(False, self.region, 5 * MB) + request_type = S3RequestType.PUT_OBJECT + + event_loop_group = EventLoopGroup() + host_resolver = DefaultHostResolver(event_loop_group) + bootstrap = ClientBootstrap(event_loop_group, host_resolver) + credential_provider = AwsCredentialsProvider.new_default_chain(bootstrap) + # Let signer to normalize uri path for us. + signing_config = AwsSigningConfig( + algorithm=AwsSigningAlgorithm.V4, + signature_type=AwsSignatureType.HTTP_REQUEST_HEADERS, + service="s3", + signed_body_header_type=AwsSignedBodyHeaderType.X_AMZ_CONTENT_SHA_256, + signed_body_value=AwsSignedBodyValue.UNSIGNED_PAYLOAD, + region=self.region, + credentials_provider=credential_provider, + use_double_uri_encode=False, + should_normalize_uri_path=True, + ) + + s3_request = s3_client.make_request( + request=request, + type=request_type, + send_filepath=self.special_path, + signing_config=signing_config, + on_headers=self._on_request_headers, + on_progress=self._on_progress) + finished_future = s3_request.finished_future + finished_future.result(self.timeout) + + # check result + self.assertEqual( + self.data_len, + self.transferred_len, + "the transferred length reported does not match body we sent") + self._validate_successful_get_response(request_type is S3RequestType.PUT_OBJECT) + os.remove(self.special_path) + def test_non_ascii_filepath_upload(self): # remove the input file when request done with open(self.non_ascii_file_name, 'wb') as file: