Add checksum settings for S3 (#512)

awslabs · Oct 18, 2023 · 2ea0145 · 2ea0145
1 parent 965cf27
commit 2ea0145
Show file tree

Hide file tree

Showing 7 changed files with 200 additions and 66 deletions.
diff --git a/awscrt/s3.py b/awscrt/s3.py
@@ -12,7 +12,9 @@
 from awscrt.io import ClientBootstrap, TlsConnectionOptions
 from awscrt.auth import AwsCredentialsProvider, AwsSignatureType, AwsSignedBodyHeaderType, AwsSignedBodyValue, AwsSigningAlgorithm, AwsSigningConfig
 import awscrt.exceptions
+from dataclasses import dataclass
 import threading
+from typing import Optional
 from enum import IntEnum
 
 
@@ -50,6 +52,60 @@ class S3RequestTlsMode(IntEnum):
     """
 
 
+class S3ChecksumAlgorithm(IntEnum):
+    """
+    Checksum algorithm used to verify object integrity.
+    https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    """
+
+    CRC32C = 1
+    """CRC32C"""
+
+    CRC32 = 2
+    """CRC32"""
+
+    SHA1 = 3
+    """SHA-1"""
+
+    SHA256 = 4
+    """SHA-256"""
+
+
+class S3ChecksumLocation(IntEnum):
+    """Where to put the checksum."""
+
+    HEADER = 1
+    """
+    Add checksum as a request header field.
+    The checksum is calculated before any part of the request is sent to the server.
+    """
+
+    TRAILER = 2
+    """
+    Add checksum as a request trailer field.
+    The checksum is calculated as the body is streamed to the server, then
+    added as a trailer field. This may be more efficient than HEADER, but
+    can only be used with "streaming" requests that support it.
+    """
+
+
+@dataclass
+class S3ChecksumConfig:
+    """Configures how the S3Client calculates and verifies checksums."""
+
+    algorithm: Optional[S3ChecksumAlgorithm] = None
+    """
+    If set, the S3Client will calculate a checksum using this algorithm
+    and add it to the request. If you set this, you must also set `location`.
+    """
+
+    location: Optional[S3ChecksumLocation] = None
+    """Where to put the request checksum."""
+
+    validate_response: bool = False
+    """Whether to retrieve and validate response checksums."""
+
+
 class S3Client(NativeResource):
     """S3 client
 
@@ -153,12 +209,13 @@ def on_shutdown():
     def make_request(
             self,
             *,
-            request,
             type,
-            signing_config=None,
-            credential_provider=None,
+            request,
             recv_filepath=None,
             send_filepath=None,
+            signing_config=None,
+            credential_provider=None,
+            checksum_config=None,
             on_headers=None,
             on_body=None,
             on_done=None,
@@ -168,19 +225,11 @@ def make_request(
         requests under the hood for acceleration.
 
         Keyword Args:
-            request (HttpRequest): The overall outgoing API request for S3 operation.
-                If the request body is a file, set send_filepath for better performance.
-
             type (S3RequestType): The type of S3 request passed in,
                 :attr:`~S3RequestType.GET_OBJECT`/:attr:`~S3RequestType.PUT_OBJECT` can be accelerated
 
-            signing_config (Optional[AwsSigningConfig]):
-                Configuration for signing of the request to override the configuration from client. Use :func:`create_default_s3_signing_config()` to create the default config.
-                If None is provided, the client configuration will be used.
-
-            credential_provider (Optional[AwsCredentialsProvider]):  Deprecated, prefer `signing_config` instead.
-                Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request, for this request only.
-                If None is provided, the client configuration will be used.
+            request (HttpRequest): The overall outgoing API request for S3 operation.
+                If the request body is a file, set send_filepath for better performance.
 
             recv_filepath (Optional[str]): Optional file path. If set, the
                 response body is written directly to a file and the
@@ -192,6 +241,16 @@ def make_request(
                 request's `body_stream` is ignored. This should give better
                 performance than reading a file from a stream.
 
+            signing_config (Optional[AwsSigningConfig]):
+                Configuration for signing of the request to override the configuration from client. Use :func:`create_default_s3_signing_config()` to create the default config.
+                If None is provided, the client configuration will be used.
+
+            credential_provider (Optional[AwsCredentialsProvider]):  Deprecated, prefer `signing_config` instead.
+                Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request, for this request only.
+                If None is provided, the client configuration will be used.
+
+            checksum_config (Optional[S3ChecksumConfig]): Optional checksum settings.
+
             on_headers: Optional callback invoked as the response received, and even the API request
                 has been split into multiple parts, this callback will only be invoked once as
                 it's just making one API request to S3.
@@ -244,12 +303,13 @@ def make_request(
         """
         return S3Request(
             client=self,
-            request=request,
             type=type,
-            signing_config=signing_config,
-            credential_provider=credential_provider,
+            request=request,
             recv_filepath=recv_filepath,
             send_filepath=send_filepath,
+            signing_config=signing_config,
+            credential_provider=credential_provider,
+            checksum_config=checksum_config,
             on_headers=on_headers,
             on_body=on_body,
             on_done=on_done,
@@ -277,12 +337,13 @@ def __init__(
             self,
             *,
             client,
-            request,
             type,
-            signing_config=None,
-            credential_provider=None,
+            request,
             recv_filepath=None,
             send_filepath=None,
+            signing_config=None,
+            credential_provider=None,
+            checksum_config=None,
             on_headers=None,
             on_body=None,
             on_done=None,
@@ -299,6 +360,16 @@ def __init__(
         self._finished_future = Future()
         self.shutdown_event = threading.Event()
 
+        checksum_algorithm = 0  # 0 means NONE in C
+        checksum_location = 0  # 0 means NONE in C
+        validate_response_checksum = False
+        if checksum_config is not None:
+            if checksum_config.algorithm is not None:
+                checksum_algorithm = checksum_config.algorithm.value
+            if checksum_config.location is not None:
+                checksum_location = checksum_config.location.value
+            validate_response_checksum = checksum_config.validate_response
+
         s3_request_core = _S3RequestCore(
             request,
             self._finished_future,
@@ -320,6 +391,9 @@ def __init__(
             recv_filepath,
             send_filepath,
             region,
+            checksum_algorithm,
+            checksum_location,
+            validate_response_checksum,
             s3_request_core)
 
     @property

diff --git a/crt/aws-c-cal b/crt/aws-c-cal
diff --git a/crt/aws-c-s3 b/crt/aws-c-s3
diff --git a/crt/s2n b/crt/s2n
diff --git a/source/s3_client.c b/source/s3_client.c
@@ -71,17 +71,16 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
 
     struct aws_allocator *allocator = aws_py_get_allocator();
 
-    PyObject *bootstrap_py = NULL;
-    PyObject *signing_config_py = NULL;
-    PyObject *credential_provider_py = NULL;
-    PyObject *tls_options_py = NULL;
-    PyObject *on_shutdown_py = NULL;
-    PyObject *py_core = NULL;
-    const char *region;
-    Py_ssize_t region_len;
-    uint64_t part_size = 0;
-    double throughput_target_gbps = 0;
-    int tls_mode;
+    PyObject *bootstrap_py;           /* O */
+    PyObject *signing_config_py;      /* O */
+    PyObject *credential_provider_py; /* O */
+    PyObject *tls_options_py;         /* O */
+    PyObject *on_shutdown_py;         /* O */
+    struct aws_byte_cursor region;    /* s# */
+    int tls_mode;                     /* i */
+    uint64_t part_size;               /* K */
+    double throughput_target_gbps;    /* d */
+    PyObject *py_core;                /* O */
     if (!PyArg_ParseTuple(
             args,
             "OOOOOs#iKdO",
@@ -90,8 +89,8 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
             &credential_provider_py,
             &tls_options_py,
             &on_shutdown_py,
-            &region,
-            &region_len,
+            &region.ptr,
+            &region.len,
             &tls_mode,
             &part_size,
             &throughput_target_gbps,
@@ -121,11 +120,8 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
     struct aws_signing_config_aws signing_config_from_credentials_provider;
     AWS_ZERO_STRUCT(signing_config_from_credentials_provider);
 
-    struct aws_byte_cursor region_cursor = aws_byte_cursor_from_array((const uint8_t *)region, region_len);
-
     if (credential_provider) {
-        aws_s3_init_default_signing_config(
-            &signing_config_from_credentials_provider, region_cursor, credential_provider);
+        aws_s3_init_default_signing_config(&signing_config_from_credentials_provider, region, credential_provider);
         signing_config = &signing_config_from_credentials_provider;
     }
 
@@ -157,7 +153,7 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
     Py_INCREF(s3_client->py_core);
 
     struct aws_s3_client_config s3_config = {
-        .region = aws_byte_cursor_from_array((const uint8_t *)region, region_len),
+        .region = region,
         .client_bootstrap = bootstrap,
         .tls_mode = tls_mode,
         .signing_config = signing_config,

diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c
@@ -319,6 +319,7 @@ static void s_s3_request_on_progress(
     const struct aws_s3_meta_request_progress *progress,
     void *user_data) {
 
+    (void)meta_request;
     struct s3_meta_request_binding *request_binding = user_data;
 
     bool report_progress = false;
@@ -346,20 +347,22 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
 
     struct aws_allocator *allocator = aws_py_get_allocator();
 
-    PyObject *py_s3_request = NULL;
-    PyObject *s3_client_py = NULL;
-    PyObject *http_request_py = NULL;
-    int type;
-    PyObject *signing_config_py = NULL;
-    PyObject *credential_provider_py = NULL;
-    const char *recv_filepath;
-    const char *send_filepath;
-    const char *region;
-    Py_ssize_t region_len;
-    PyObject *py_core = NULL;
+    PyObject *py_s3_request;                           /* O */
+    PyObject *s3_client_py;                            /* O */
+    PyObject *http_request_py;                         /* O */
+    int type;                                          /* i */
+    PyObject *signing_config_py;                       /* O */
+    PyObject *credential_provider_py;                  /* O */
+    const char *recv_filepath;                         /* z */
+    const char *send_filepath;                         /* z */
+    struct aws_byte_cursor region;                     /* s# */
+    enum aws_s3_checksum_algorithm checksum_algorithm; /* i */
+    enum aws_s3_checksum_location checksum_location;   /* i */
+    int validate_response_checksum;                    /* p - boolean predicate */
+    PyObject *py_core;                                 /* O */
     if (!PyArg_ParseTuple(
             args,
-            "OOOiOOzzs#O",
+            "OOOiOOzzs#iipO",
             &py_s3_request,
             &s3_client_py,
             &http_request_py,
@@ -368,8 +371,11 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
             &credential_provider_py,
             &recv_filepath,
             &send_filepath,
-            &region,
-            &region_len,
+            &region.ptr,
+            &region.len,
+            &checksum_algorithm,
+            &checksum_location,
+            &validate_response_checksum,
             &py_core)) {
         return NULL;
     }
@@ -402,12 +408,16 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
     struct aws_signing_config_aws signing_config_from_credentials_provider;
     AWS_ZERO_STRUCT(signing_config_from_credentials_provider);
     if (credential_provider) {
-        struct aws_byte_cursor region_cursor = aws_byte_cursor_from_array((const uint8_t *)region, region_len);
-        aws_s3_init_default_signing_config(
-            &signing_config_from_credentials_provider, region_cursor, credential_provider);
+        aws_s3_init_default_signing_config(&signing_config_from_credentials_provider, region, credential_provider);
         signing_config = &signing_config_from_credentials_provider;
     }
 
+    struct aws_s3_checksum_config checksum_config = {
+        .checksum_algorithm = checksum_algorithm,
+        .location = checksum_location,
+        .validate_response_checksum = validate_response_checksum != 0,
+    };
+
     struct s3_meta_request_binding *meta_request = aws_mem_calloc(allocator, 1, sizeof(struct s3_meta_request_binding));
     if (!meta_request) {
         return PyErr_AwsLastError();
@@ -438,6 +448,7 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
         .type = type,
         .message = http_request,
         .signing_config = signing_config,
+        .checksum_config = &checksum_config,
         .send_filepath = aws_byte_cursor_from_c_str(send_filepath),
         .headers_callback = s_s3_request_on_headers,
         .body_callback = s_s3_request_on_body,