Skip to content

Commit

Permalink
Merge branch 'main' into new-submodules
Browse files Browse the repository at this point in the history
  • Loading branch information
@TingDaoK
TingDaoK committed Jul 10, 2023
2 parents fd66bc8 + 91d3dce commit b1f8eee
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 22 deletions.
66 changes: 59 additions & 7 deletions awscrt/s3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from awscrt import NativeResource
from awscrt.http import HttpRequest
from awscrt.io import ClientBootstrap, TlsConnectionOptions
from awscrt.auth import AwsCredentialsProvider
from awscrt.auth import AwsCredentialsProvider, AwsSignatureType, AwsSignedBodyHeaderType, AwsSignedBodyValue, AwsSigningAlgorithm, AwsSigningConfig
import awscrt.exceptions
import threading
from enum import IntEnum
Expand Down Expand Up @@ -68,8 +68,12 @@ class S3Client(NativeResource):
If this is :attr:`S3RequestTlsMode.DISABLED`:
No TLS options will be used, regardless of `tls_connection_options` value.
credential_provider (Optional[AwsCredentialsProvider]): Credentials providers source the
:class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request.
signing_config (Optional[AwsSigningConfig]):
Configuration for signing of the client. Use :func:`create_default_s3_signing_config()` to create the default config.
If None is provided, the request will not be signed.
credential_provider (Optional[AwsCredentialsProvider]): Deprecated, prefer `signing_config` instead.
Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request.
If None is provided, the request will not be signed.
tls_connection_options (Optional[TlsConnectionOptions]): Optional TLS Options to be used
Expand All @@ -91,12 +95,14 @@ def __init__(
bootstrap,
region,
tls_mode=None,
signing_config=None,
credential_provider=None,
tls_connection_options=None,
part_size=None,
throughput_target_gbps=None):
assert isinstance(bootstrap, ClientBootstrap) or bootstrap is None
assert isinstance(region, str)
assert isinstance(signing_config, AwsSigningConfig) or signing_config is None
assert isinstance(credential_provider, AwsCredentialsProvider) or credential_provider is None
assert isinstance(tls_connection_options, TlsConnectionOptions) or tls_connection_options is None
assert isinstance(part_size, int) or part_size is None
Expand All @@ -106,6 +112,10 @@ def __init__(
throughput_target_gbps,
float) or throughput_target_gbps is None

if credential_provider and signing_config:
raise ValueError("'credential_provider' has been deprecated in favor of 'signing_config'. "
"Both parameters may not be set.")

super().__init__()

shutdown_event = threading.Event()
Expand All @@ -117,7 +127,8 @@ def on_shutdown():

if not bootstrap:
bootstrap = ClientBootstrap.get_or_create_static_default()
s3_client_core = _S3ClientCore(bootstrap, credential_provider, tls_connection_options)

s3_client_core = _S3ClientCore(bootstrap, credential_provider, signing_config, tls_connection_options)

# C layer uses 0 to indicate defaults
if tls_mode is None:
Expand All @@ -129,6 +140,7 @@ def on_shutdown():

self._binding = _awscrt.s3_client_new(
bootstrap,
signing_config,
credential_provider,
tls_connection_options,
on_shutdown,
Expand All @@ -143,6 +155,7 @@ def make_request(
*,
request,
type,
signing_config=None,
credential_provider=None,
recv_filepath=None,
send_filepath=None,
Expand All @@ -161,9 +174,13 @@ def make_request(
type (S3RequestType): The type of S3 request passed in,
:attr:`~S3RequestType.GET_OBJECT`/:attr:`~S3RequestType.PUT_OBJECT` can be accelerated
credential_provider (Optional[AwsCredentialsProvider]): Credentials providers source the
:class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request, for this request only.
If None is provided, the credential provider in the client will be used.
signing_config (Optional[AwsSigningConfig]):
Configuration for signing of the request to override the configuration from client. Use :func:`create_default_s3_signing_config()` to create the default config.
If None is provided, the client configuration will be used.
credential_provider (Optional[AwsCredentialsProvider]): Deprecated, prefer `signing_config` instead.
Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request, for this request only.
If None is provided, the client configuration will be used.
recv_filepath (Optional[str]): Optional file path. If set, the
response body is written directly to a file and the
Expand Down Expand Up @@ -229,6 +246,7 @@ def make_request(
client=self,
request=request,
type=type,
signing_config=signing_config,
credential_provider=credential_provider,
recv_filepath=recv_filepath,
send_filepath=send_filepath,
Expand Down Expand Up @@ -261,6 +279,7 @@ def __init__(
client,
request,
type,
signing_config=None,
credential_provider=None,
recv_filepath=None,
send_filepath=None,
Expand All @@ -284,6 +303,7 @@ def __init__(
request,
self._finished_future,
self.shutdown_event,
signing_config,
credential_provider,
on_headers,
on_body,
Expand All @@ -295,6 +315,7 @@ def __init__(
client,
request,
type,
signing_config,
credential_provider,
recv_filepath,
send_filepath,
Expand All @@ -316,9 +337,11 @@ class _S3ClientCore:

def __init__(self, bootstrap,
credential_provider=None,
signing_config=None,
tls_connection_options=None):
self._bootstrap = bootstrap
self._credential_provider = credential_provider
self._signing_config = signing_config
self._tls_connection_options = tls_connection_options


Expand All @@ -332,13 +355,15 @@ def __init__(
request,
finish_future,
shutdown_event,
signing_config=None,
credential_provider=None,
on_headers=None,
on_body=None,
on_done=None,
on_progress=None):

self._request = request
self._signing_config = signing_config
self._credential_provider = credential_provider

self._on_headers_cb = on_headers
Expand Down Expand Up @@ -377,3 +402,30 @@ def _on_finish(self, error_code, error_headers, error_body):
def _on_progress(self, progress):
if self._on_progress_cb:
self._on_progress_cb(progress)


def create_default_s3_signing_config(*, region: str, credential_provider: AwsCredentialsProvider, **kwargs):
"""Create a default `AwsSigningConfig` for S3 service.
Attributes:
region (str): The region to sign against.
credential_provider (AwsCredentialsProvider): Credentials provider
to fetch signing credentials with.
`**kwargs`: Forward compatibility kwargs.
Returns:
AwsSigningConfig
"""
return AwsSigningConfig(
algorithm=AwsSigningAlgorithm.V4,
signature_type=AwsSignatureType.HTTP_REQUEST_HEADERS,
service="s3",
signed_body_header_type=AwsSignedBodyHeaderType.X_AMZ_CONTENT_SHA_256,
signed_body_value=AwsSignedBodyValue.UNSIGNED_PAYLOAD,
region=region,
credentials_provider=credential_provider,
use_double_uri_encode=False,
should_normalize_uri_path=False,
)
22 changes: 16 additions & 6 deletions source/s3_client.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
struct aws_allocator *allocator = aws_py_get_allocator();

PyObject *bootstrap_py = NULL;
PyObject *signing_config_py = NULL;
PyObject *credential_provider_py = NULL;
PyObject *tls_options_py = NULL;
PyObject *on_shutdown_py = NULL;
Expand All @@ -83,8 +84,9 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
int tls_mode;
if (!PyArg_ParseTuple(
args,
"OOOOs#iKdO",
"OOOOOs#iKdO",
&bootstrap_py,
&signing_config_py,
&credential_provider_py,
&tls_options_py,
&on_shutdown_py,
Expand All @@ -109,14 +111,22 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
return NULL;
}
}

struct aws_signing_config_aws signing_config;
AWS_ZERO_STRUCT(signing_config);
struct aws_signing_config_aws *signing_config = NULL;
if (signing_config_py != Py_None) {
signing_config = aws_py_get_signing_config(signing_config_py);
if (!signing_config) {
return NULL;
}
}
struct aws_signing_config_aws signing_config_from_credentials_provider;
AWS_ZERO_STRUCT(signing_config_from_credentials_provider);

struct aws_byte_cursor region_cursor = aws_byte_cursor_from_array((const uint8_t *)region, region_len);

if (credential_provider) {
aws_s3_init_default_signing_config(&signing_config, region_cursor, credential_provider);
aws_s3_init_default_signing_config(
&signing_config_from_credentials_provider, region_cursor, credential_provider);
signing_config = &signing_config_from_credentials_provider;
}

struct aws_tls_connection_options *tls_options = NULL;
Expand Down Expand Up @@ -150,7 +160,7 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
.region = aws_byte_cursor_from_array((const uint8_t *)region, region_len),
.client_bootstrap = bootstrap,
.tls_mode = tls_mode,
.signing_config = credential_provider ? &signing_config : NULL,
.signing_config = signing_config,
.part_size = part_size,
.tls_connection_options = tls_options,
.throughput_target_gbps = throughput_target_gbps,
Expand Down
22 changes: 17 additions & 5 deletions source/s3_meta_request.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,7 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
PyObject *s3_client_py = NULL;
PyObject *http_request_py = NULL;
int type;
PyObject *signing_config_py = NULL;
PyObject *credential_provider_py = NULL;
const char *recv_filepath;
const char *send_filepath;
Expand All @@ -484,11 +485,12 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
PyObject *py_core = NULL;
if (!PyArg_ParseTuple(
args,
"OOOiOzzs#O",
"OOOiOOzzs#O",
&py_s3_request,
&s3_client_py,
&http_request_py,
&type,
&signing_config_py,
&credential_provider_py,
&recv_filepath,
&send_filepath,
Expand All @@ -507,6 +509,14 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
return NULL;
}

struct aws_signing_config_aws *signing_config = NULL;
if (signing_config_py != Py_None) {
signing_config = aws_py_get_signing_config(signing_config_py);
if (!signing_config) {
return NULL;
}
}

struct aws_credentials_provider *credential_provider = NULL;
if (credential_provider_py != Py_None) {
credential_provider = aws_py_get_credentials_provider(credential_provider_py);
Expand All @@ -515,11 +525,13 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
}
}

struct aws_signing_config_aws signing_config;
AWS_ZERO_STRUCT(signing_config);
struct aws_signing_config_aws signing_config_from_credentials_provider;
AWS_ZERO_STRUCT(signing_config_from_credentials_provider);
if (credential_provider) {
struct aws_byte_cursor region_cursor = aws_byte_cursor_from_array((const uint8_t *)region, region_len);
aws_s3_init_default_signing_config(&signing_config, region_cursor, credential_provider);
aws_s3_init_default_signing_config(
&signing_config_from_credentials_provider, region_cursor, credential_provider);
signing_config = &signing_config_from_credentials_provider;
}

struct s3_meta_request_binding *meta_request = aws_mem_calloc(allocator, 1, sizeof(struct s3_meta_request_binding));
Expand Down Expand Up @@ -566,7 +578,7 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
struct aws_s3_meta_request_options s3_meta_request_opt = {
.type = type,
.message = meta_request->copied_message ? meta_request->copied_message : http_request,
.signing_config = credential_provider ? &signing_config : NULL,
.signing_config = signing_config,
.headers_callback = s_s3_request_on_headers,
.body_callback = s_s3_request_on_body,
.finish_callback = s_s3_request_on_finish,
Expand Down
53 changes: 49 additions & 4 deletions test/test_s3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
from concurrent.futures import Future

from awscrt.http import HttpHeaders, HttpRequest
from awscrt.s3 import S3Client, S3RequestType
from awscrt.s3 import S3Client, S3RequestType, create_default_s3_signing_config
from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions
from awscrt.auth import AwsCredentialsProvider
from awscrt.auth import AwsCredentialsProvider, AwsSignatureType, AwsSignedBodyHeaderType, AwsSignedBodyValue, AwsSigningAlgorithm, AwsSigningConfig

MB = 1024 ** 2
GB = 1024 ** 3
Expand Down Expand Up @@ -74,6 +74,7 @@ def s3_client_new(secure, region, part_size=0):
host_resolver = DefaultHostResolver(event_loop_group)
bootstrap = ClientBootstrap(event_loop_group, host_resolver)
credential_provider = AwsCredentialsProvider.new_default_chain(bootstrap)
signing_config = create_default_s3_signing_config(region=region, credential_provider=credential_provider)
tls_option = None
if secure:
opt = TlsContextOptions()
Expand All @@ -83,7 +84,7 @@ def s3_client_new(secure, region, part_size=0):
s3_client = S3Client(
bootstrap=bootstrap,
region=region,
credential_provider=credential_provider,
signing_config=signing_config,
tls_connection_options=tls_option,
part_size=part_size)

Expand All @@ -102,7 +103,6 @@ def read(self, length):
return fake_data


@unittest.skipUnless(os.environ.get('AWS_TEST_S3'), 'set env var to run test: AWS_TEST_S3')
class S3ClientTest(NativeResourceTest):

def setUp(self):
Expand Down Expand Up @@ -137,6 +137,7 @@ def setUp(self):
self.bucket_name = "aws-crt-canary-bucket"
self.timeout = 100 # seconds
self.num_threads = 0
self.special_path = "put_object_test_10MB@$%.txt"
self.non_ascii_file_name = "ÉxÅmple.txt".encode("utf-8")

self.response_headers = None
Expand Down Expand Up @@ -439,6 +440,50 @@ def test_multipart_upload_with_invalid_request(self):
self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT, "AWS_ERROR_S3_INVALID_RESPONSE_STATUS")
self.put_body_stream.close()

def test_special_filepath_upload(self):
# remove the input file when request done
with open(self.special_path, 'wb') as file:
file.write(b"a" * 10 * MB)
request = self._put_object_request(self.special_path)
self.put_body_stream.close()
s3_client = s3_client_new(False, self.region, 5 * MB)
request_type = S3RequestType.PUT_OBJECT

event_loop_group = EventLoopGroup()
host_resolver = DefaultHostResolver(event_loop_group)
bootstrap = ClientBootstrap(event_loop_group, host_resolver)
credential_provider = AwsCredentialsProvider.new_default_chain(bootstrap)
# Let signer to normalize uri path for us.
signing_config = AwsSigningConfig(
algorithm=AwsSigningAlgorithm.V4,
signature_type=AwsSignatureType.HTTP_REQUEST_HEADERS,
service="s3",
signed_body_header_type=AwsSignedBodyHeaderType.X_AMZ_CONTENT_SHA_256,
signed_body_value=AwsSignedBodyValue.UNSIGNED_PAYLOAD,
region=self.region,
credentials_provider=credential_provider,
use_double_uri_encode=False,
should_normalize_uri_path=True,
)

s3_request = s3_client.make_request(
request=request,
type=request_type,
send_filepath=self.special_path,
signing_config=signing_config,
on_headers=self._on_request_headers,
on_progress=self._on_progress)
finished_future = s3_request.finished_future
finished_future.result(self.timeout)

# check result
self.assertEqual(
self.data_len,
self.transferred_len,
"the transferred length reported does not match body we sent")
self._validate_successful_get_response(request_type is S3RequestType.PUT_OBJECT)
os.remove(self.special_path)

def test_non_ascii_filepath_upload(self):
# remove the input file when request done
with open(self.non_ascii_file_name, 'wb') as file:
Expand Down

0 comments on commit b1f8eee

Please sign in to comment.