awslabs · TingDaoK · Nov 20, 2025 · Nov 5, 2025 · Nov 5, 2025 · Nov 5, 2025
diff --git a/awscrt/s3.py b/awscrt/s3.py
@@ -224,6 +224,7 @@ class S3Client(NativeResource):
             for each connection, unless `tls_mode` is :attr:`S3RequestTlsMode.DISABLED`
 
         part_size (Optional[int]): Size, in bytes, of parts that files will be downloaded or uploaded in.
+            If not set, a dynamic default part size will be used based on the throughput target, memory_limit_in_bytes.
             Note: for :attr:`S3RequestType.PUT_OBJECT` request, client will adjust the part size to meet the service limits.
             (max number of parts per upload is 10,000, minimum upload part size is 5 MiB)
 
@@ -245,10 +246,11 @@ class S3Client(NativeResource):
 
         memory_limit (Optional[int]): Memory limit, in bytes, of how much memory
             client can use for buffering data for requests.
+            If not set, client will try to pick up from environment variable before chose the default.
             Default values scale with target throughput and are currently
-            between 2GiB and 8GiB (may change in future)
+            between 2GiB and 24GiB (may change in future)
 
-        network_interface_names: (Optional[Sequence(str)])
+        network_interface_names (Optional[Sequence(str)]):
             **THIS IS AN EXPERIMENTAL AND UNSTABLE API.**
             A sequence of network interface names. The client will distribute the
             connections across network interfaces. If any interface name is invalid, goes down,
@@ -257,10 +259,14 @@ class S3Client(NativeResource):
             is not supported on Windows. `AWS_ERROR_PLATFORM_NOT_SUPPORTED` will be raised on unsupported platforms. On
             Linux, SO_BINDTODEVICE is used and requires kernel version >= 5.7 or root privileges.
 
-        fio_options: (Optional[S3FileIoOptions])
+        fio_options (Optional[S3FileIoOptions]):
             If set, this controls how the client interact with file I/O.
             If not set, a default options will be created to avoid memory issue based on the size of file.
             Note: Only applies when the request created with `send_filepath`
+
+        max_active_connections_override (Optional[int]):
+            When set, this will cap the number of active connections for the meta request.
+            When not set, the client will determine this value based on `throughput_target_gbps`. (Recommended)
     """
 
     __slots__ = ('shutdown_event', '_region')
@@ -280,7 +286,8 @@ def __init__(
             enable_s3express=False,
             memory_limit=None,
             network_interface_names: Optional[Sequence[str]] = None,
-            fio_options: Optional['S3FileIoOptions'] = None):
+            fio_options: Optional['S3FileIoOptions'] = None,
+            max_active_connections_override: Optional[int] = None):
         assert isinstance(bootstrap, ClientBootstrap) or bootstrap is None
         assert isinstance(region, str)
         assert isinstance(signing_config, AwsSigningConfig) or signing_config is None
@@ -295,6 +302,7 @@ def __init__(
         assert isinstance(enable_s3express, bool) or enable_s3express is None
         assert isinstance(network_interface_names, Sequence) or network_interface_names is None
         assert isinstance(fio_options, S3FileIoOptions) or fio_options is None
+        assert isinstance(max_active_connections_override, int) or max_active_connections_override is None
 
         if credential_provider and signing_config:
             raise ValueError("'credential_provider' has been deprecated in favor of 'signing_config'.  "
@@ -334,6 +342,8 @@ def on_shutdown():
             # ensure this is a list, so it's simpler to process in C
             if not isinstance(network_interface_names, list):
                 network_interface_names = list(network_interface_names)
+        if max_active_connections_override is None:
+            max_active_connections_override = 0
         fio_options_set = False
         should_stream = False
         disk_throughput_gbps = 0.0
@@ -362,6 +372,7 @@ def on_shutdown():
             should_stream,
             disk_throughput_gbps,
             direct_io,
+            max_active_connections_override,
             s3_client_core)
 
     def make_request(
@@ -378,6 +389,7 @@ def make_request(
             part_size=None,
             multipart_upload_threshold=None,
             fio_options=None,
+            max_active_connections_override=None,
             on_headers=None,
             on_body=None,
             on_done=None,
@@ -445,7 +457,8 @@ def make_request(
             checksum_config (Optional[S3ChecksumConfig]): Optional checksum settings.
 
             part_size (Optional[int]): Size, in bytes, of parts that files will be downloaded or uploaded in.
-                If not set, the part size configured for the client will be used.
+                If not set, the part size configured for the client will be used, which defaults to a dynamic value based on
+                the throughput target, memory_limit_in_bytes and the requested object size.
                 Note: for :attr:`S3RequestType.PUT_OBJECT` request, client will adjust the part size to meet the service limits.
                 (max number of parts per upload is 10,000, minimum upload part size is 5 MiB)
 
@@ -458,11 +471,15 @@ def make_request(
                 If both `part_size` and `multipart_upload_threshold` are not set,
                 the values from `aws_s3_client_config` are used.
 
-            fio_options: (Optional[S3FileIoOptions])
+            fio_options (Optional[S3FileIoOptions]):
                 If set, this overrides the client fio_options to control how this request interact with file I/O.
                 If not set, a default options will be created to avoid memory issue based on the size of file.
                 Note: Only applies when the request created with `send_filepath`
 
+            max_active_connections_override (Optional[int]):
+                When set, this will cap the number of active connections for the meta request.
+                When not set, the client will determine it based on client side settings. (Recommended)
+
             on_headers: Optional callback invoked as the response received, and even the API request
                 has been split into multiple parts, this callback will only be invoked once as
                 it's just making one API request to S3.
@@ -549,6 +566,7 @@ def make_request(
             part_size=part_size,
             multipart_upload_threshold=multipart_upload_threshold,
             fio_options=fio_options,
+            max_active_connections_override=max_active_connections_override,
             on_headers=on_headers,
             on_body=on_body,
             on_done=on_done,
@@ -587,6 +605,7 @@ def __init__(
             part_size=None,
             multipart_upload_threshold=None,
             fio_options=None,
+            max_active_connections_override=None,
             on_headers=None,
             on_body=None,
             on_done=None,
@@ -600,6 +619,7 @@ def __init__(
         assert isinstance(part_size, int) or part_size is None
         assert isinstance(multipart_upload_threshold, int) or multipart_upload_threshold is None
         assert isinstance(fio_options, S3FileIoOptions) or fio_options is None
+        assert isinstance(max_active_connections_override, int) or max_active_connections_override is None
 
         if type == S3RequestType.DEFAULT and not operation_name:
             raise ValueError("'operation_name' must be set when using S3RequestType.DEFAULT")
@@ -632,6 +652,8 @@ def __init__(
             should_stream = fio_options.should_stream
             disk_throughput_gbps = fio_options.disk_throughput_gbps
             direct_io = fio_options.direct_io
+        if max_active_connections_override is None:
+            max_active_connections_override = 0
 
         s3_request_core = _S3RequestCore(
             request,
@@ -664,6 +686,7 @@ def __init__(
             should_stream,
             disk_throughput_gbps,
             direct_io,
+            max_active_connections_override,
             s3_request_core)
 
     @property

diff --git a/crt/aws-c-auth b/crt/aws-c-auth
diff --git a/crt/aws-c-cal b/crt/aws-c-cal
diff --git a/crt/aws-c-common b/crt/aws-c-common
diff --git a/crt/aws-c-http b/crt/aws-c-http
diff --git a/crt/aws-c-io b/crt/aws-c-io
diff --git a/crt/aws-c-s3 b/crt/aws-c-s3
diff --git a/crt/aws-lc b/crt/aws-lc
diff --git a/crt/s2n b/crt/s2n
diff --git a/source/s3_client.c b/source/s3_client.c
@@ -245,28 +245,29 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
 
     struct aws_allocator *allocator = aws_py_get_allocator();
 
-    PyObject *bootstrap_py;               /* O */
-    PyObject *signing_config_py;          /* O */
-    PyObject *credential_provider_py;     /* O */
-    PyObject *tls_options_py;             /* O */
-    PyObject *on_shutdown_py;             /* O */
-    struct aws_byte_cursor region;        /* s# */
-    int tls_mode;                         /* i */
-    uint64_t part_size;                   /* K */
-    uint64_t multipart_upload_threshold;  /* K */
-    double throughput_target_gbps;        /* d */
-    int enable_s3express;                 /* p */
-    uint64_t mem_limit;                   /* K */
-    PyObject *network_interface_names_py; /* O */
-    int fio_options_set;                  /* p - boolean predicate */
-    int should_stream;                    /* p - boolean predicate */
-    double disk_throughput_gbps;          /* d */
-    int direct_io;                        /* p - boolean predicate */
-    PyObject *py_core;                    /* O */
+    PyObject *bootstrap_py;                   /* O */
+    PyObject *signing_config_py;              /* O */
+    PyObject *credential_provider_py;         /* O */
+    PyObject *tls_options_py;                 /* O */
+    PyObject *on_shutdown_py;                 /* O */
+    struct aws_byte_cursor region;            /* s# */
+    int tls_mode;                             /* i */
+    uint64_t part_size;                       /* K */
+    uint64_t multipart_upload_threshold;      /* K */
+    double throughput_target_gbps;            /* d */
+    int enable_s3express;                     /* p */
+    uint64_t mem_limit;                       /* K */
+    PyObject *network_interface_names_py;     /* O */
+    int fio_options_set;                      /* p - boolean predicate */
+    int should_stream;                        /* p - boolean predicate */
+    double disk_throughput_gbps;              /* d */
+    int direct_io;                            /* p - boolean predicate */
+    uint64_t max_active_connections_override; /* K */
+    PyObject *py_core;                        /* O */
 
     if (!PyArg_ParseTuple(
             args,
-            "OOOOOs#iKKdpKOppdpO",
+            "OOOOOs#iKKdpKOppdpKO",
             &bootstrap_py,
             &signing_config_py,
             &credential_provider_py,
@@ -285,6 +286,7 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
             &should_stream,
             &disk_throughput_gbps,
             &direct_io,
+            &max_active_connections_override,
             &py_core)) {
         return NULL;
     }
@@ -397,6 +399,7 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
         .num_network_interface_names = num_network_interface_names,
         /* If fio options not set, let native code to decide the default instead */
         .fio_opts = fio_options_set ? &fio_opts : NULL,
+        .max_active_connections_override = max_active_connections_override,
     };
 
     s3_client->native = aws_s3_client_new(allocator, &s3_config);

diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c
@@ -336,10 +336,11 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
     int should_stream;                                 /* p - boolean predicate */
     double disk_throughput_gbps;                       /* d */
     int direct_io;                                     /* p - boolean predicate */
+    uint64_t max_active_connections_override;          /* K */
     PyObject *py_core;                                 /* O */
     if (!PyArg_ParseTuple(
             args,
-            "OOOizOOzzs#iipKKppdpO",
+            "OOOizOOzzs#iipKKppdpKO",
             &py_s3_request,
             &s3_client_py,
             &http_request_py,
@@ -360,6 +361,7 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
             &should_stream,
             &disk_throughput_gbps,
             &direct_io,
+            &max_active_connections_override,
             &py_core)) {
         return NULL;
     }
@@ -441,6 +443,7 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
         .multipart_upload_threshold = multipart_upload_threshold,
         /* If fio options not set, let native code to decide the default instead */
         .fio_opts = fio_options_set ? &fio_opts : NULL,
+        .max_active_connections_override = max_active_connections_override,
         .user_data = meta_request,
     };
 

diff --git a/test/test_aiohttp_client.py b/test/test_aiohttp_client.py
@@ -13,6 +13,7 @@
 from test import NativeResourceTest
 import ssl
 import os
+import json
 from io import BytesIO
 from http.server import HTTPServer, SimpleHTTPRequestHandler
 from awscrt import io
@@ -485,10 +486,10 @@ def setUp(self):
             'crt',
             'aws-c-http',
             'tests',
-            'py_localhost',
-            'server.py')
+            'mock_server',
+            'h2tls_mock_server.py')
         python_path = sys.executable
-        self.mock_server_url = urlparse("https://localhost:3443/upload_test")
+        self.mock_server_url = urlparse("https://localhost:3443/echo")
         self.p_server = subprocess.Popen([python_path, server_path])
         # Wait for server to be ready
         self._wait_for_server_ready()
@@ -572,8 +573,10 @@ async def _test_h2_mock_server_manual_write(self):
         # Create an async generator for the request body
         body_chunks = [b'hello', b'he123123', b'', b'hello']
         total_length = 0
+        content = b''
         for i in body_chunks:
             total_length = total_length + len(i)
+            content += i
 
         async def body_generator():
             for i in body_chunks:
@@ -588,8 +591,14 @@ async def body_generator():
         # Check result
         self.assertEqual(200, status_code)
         self.assertEqual(200, response.status_code)
-        # mock server response the total length received, check if it matches what we sent
-        self.assertEqual(total_length, int(response.body.decode()))
+        # Parse the response from mock server which has format:
+        # {
+        #   "body": <str of data received>,
+        #   "bytes": <byte count of data received>
+        # }
+        parsed_response = json.loads(response.body.decode())
+        self.assertEqual(total_length, int(parsed_response["bytes"]))
+        self.assertEqual(content.decode(), parsed_response["body"])
         await connection.close()
 
     class DelayStream:

diff --git a/test/test_http_client.py b/test/test_http_client.py
@@ -10,6 +10,7 @@
 import threading
 from test import NativeResourceTest
 import ssl
+import json
 import os
 from io import BytesIO
 from http.server import HTTPServer, SimpleHTTPRequestHandler
@@ -417,10 +418,10 @@ def setUp(self):
             'crt',
             'aws-c-http',
             'tests',
-            'py_localhost',
-            'server.py')
+            'mock_server',
+            'h2tls_mock_server.py')
         python_path = sys.executable
-        self.mock_server_url = urlparse("https://localhost:3443/upload_test")
+        self.mock_server_url = urlparse("https://localhost:3443/echo")
         self.p_server = subprocess.Popen([python_path, server_path])
         # Wait for server to be ready
         self._wait_for_server_ready()
@@ -504,21 +505,34 @@ def test_h2_mock_server_manual_write(self):
         stream = connection.request(request, response.on_response, response.on_body, manual_write=True)
         stream.activate()
         exception = None
+        body_chunks = [b'hello', b'he123123', b'hello']
+        total_length = 0
+        content = b''
+        for i in body_chunks:
+            total_length = total_length + len(i)
+            content += i
         try:
             # If the stream is not configured to allow manual writes, this should throw an exception directly
-            f = stream.write_data(BytesIO(b'hello'), False)
+            f = stream.write_data(BytesIO(body_chunks[0]), False)
             f.result(self.timeout)
-            stream.write_data(BytesIO(b'he123123'), False)
+            stream.write_data(BytesIO(body_chunks[1]), False)
             stream.write_data(None, False)
-            stream.write_data(BytesIO(b'hello'), True)
+            stream.write_data(BytesIO(body_chunks[2]), True)
         except RuntimeError as e:
             exception = e
         self.assertIsNone(exception)
         stream_completion_result = stream.completion_future.result(80)
         # check result
         self.assertEqual(200, response.status_code)
         self.assertEqual(200, stream_completion_result)
-        print(response.body)
+        # Parse the response from mock server which has format:
+        # {
+        #   "body": <str of data received>,
+        #   "bytes": <byte count of data received>
+        # }
+        parsed_response = json.loads(response.body.decode())
+        self.assertEqual(total_length, int(parsed_response["bytes"]))
+        self.assertEqual(content.decode(), parsed_response["body"])
 
         self.assertEqual(None, connection.close().exception(self.timeout))
+10 −0		include/aws/auth/auth.h
+78 −2		include/aws/auth/credentials.h
+160 −0		include/aws/auth/private/aws_http_credentials_provider.h
+7 −1		include/aws/auth/private/credentials_utils.h
+172 −0		include/aws/auth/private/login_token_utils.h
+30 −0		source/auth.c
+470 −0		source/aws_http_credentials_provider.c
+472 −0		source/credentials_provider_login.c
+218 −617		source/credentials_provider_sso.c
+3 −1		source/credentials_provider_sts.c
+3 −1		source/credentials_provider_sts_web_identity.c
+22 −7		source/credentials_utils.c
+1,134 −0		source/login_token_utils.c
+13 −0		tests/CMakeLists.txt
+770 −0		tests/credentials_provider_login_tests.c
+28 −10		tests/credentials_utils_tests.c
+6 −0		CMakeLists.txt
+0 −7		include/aws/cal/ecc.h
+55 −0		include/aws/cal/hkdf.h
+26 −54		source/darwin/securityframework_ecc.c
+2 −2		source/der.c
+47 −0		source/ecc.c
+61 −0		source/hkdf.c
+149 −0		source/shared/ref_hkdf.c
+2 −2		source/unix/opensslcrypto_ecc.c
+2 −2		source/windows/bcrypt_ecc.c
+4 −0		tests/CMakeLists.txt
+128 −0		tests/hkdf_test.c
+0 −4		CMakeLists.txt
+0 −7		cmake/AwsThreadName.cmake
+18 −0		include/aws/common/encoding.h
+4 −0		include/aws/common/platform.h
+17 −0		include/aws/common/system_info.h
+13 −4		source/arch/intel/encoding_avx2.c
+1 −0		source/byte_buf.c
+127 −47		source/encoding.c
+7 −4		source/posix/system_info.c
+36 −0		source/system_info.c
+10 −0		tests/CMakeLists.txt
+143 −28		tests/encoding_test.c
+30 −1		tests/system_info_tests.c
+5 −4		.builder/action/local-server-setup.py
+2 −2		.github/workflows/handle-stale-discussions.yml
+1 −1		.github/workflows/stale_issue.yml
+1 −1		README.md
+1 −1		bin/h2benchmark/README.md
+2 −0		include/aws/http/private/proxy_impl.h
+6 −0		include/aws/http/proxy.h
+41 −15		integration-testing/http_client_test.py
+2 −1		source/connection_manager.c
+4 −3		source/http2_stream_manager.c
+33 −2		source/proxy_connection.c
+28 −14		source/proxy_strategy.c
+3 −1		tests/CMakeLists.txt
+126 −0		tests/mock_server/README.md
+259 −0		tests/mock_server/h11mock_server.py
+0 −0		tests/mock_server/h2non_tls_server.py
+87 −44		tests/mock_server/h2tls_mock_server.py
+18 −0		tests/proxy_test_helper.c
+6 −0		tests/proxy_test_helper.h
+0 −40		tests/py_localhost/README.md
+20 −5		tests/test_localhost_integ.c
+78 −0		tests/test_proxy.c
+97 −28		tests/test_stream_manager.c
+6 −0		include/aws/io/event_loop.h
+1 −0		include/aws/io/private/event_loop_impl.h
+9 −3		source/event_loop.c
+1 −1		source/s2n/s2n_tls_channel_handler.c
+2 −0		.github/workflows/ci.yml
+1 −0		.gitignore
+1 −0		CMakeLists.txt
+45 −8		README.md
+13 −0		include/aws/s3/private/s3_auto_ranged_get.h
+11 −1		include/aws/s3/private/s3_client_impl.h
+70 −0		include/aws/s3/private/s3_default_buffer_pool.h
+23 −0		include/aws/s3/private/s3_meta_request_impl.h
+67 −4		include/aws/s3/private/s3_request.h
+73 −1		include/aws/s3/private/s3_util.h
+53 −0		include/aws/s3/s3_buffer_pool.h
+118 −5		include/aws/s3/s3_client.h
+131 −25		source/s3_auto_ranged_get.c
+47 −32		source/s3_auto_ranged_put.c
+27 −0		source/s3_buffer_pool.c
+180 −42		source/s3_client.c
+379 −129		source/s3_default_buffer_pool.c
+7 −0		source/s3_default_meta_request.c
+63 −61		source/s3_endpoint_resolver/aws_s3_endpoint_resolver_partition.c
+5,218 −4,057		source/s3_endpoint_resolver/aws_s3_endpoint_rule_set.c
+54 −4		source/s3_meta_request.c
+240 −22		source/s3_request.c
+287 −30		source/s3_util.c
+54 −1		tests/CMakeLists.txt
+373 −0		tests/fuzz/fuzz_buffer_pool_special_size.c
+282 −0		tests/fuzz/fuzz_buffer_pool_special_size_keep_pending.c
+70 −0		tests/fuzz/fuzz_extract_parts_from_etag.c
+129 −0		tests/fuzz/fuzz_optimal_range_size.c
+2 −1		tests/mock_s3_server/mock_s3_server.py
+533 −0		tests/s3_buffer_pool_special_size_tests.c
+46 −15		tests/s3_cancel_tests.c
+181 −0		tests/s3_client_memory_limit_env_var_test.c
+75 −0		tests/s3_client_test.c
+537 −36		tests/s3_data_plane_tests.c
+253 −0		tests/s3_max_active_connections_override_test.c
+1 −1		tests/s3_meta_request_test.c
+401 −64		tests/s3_mock_server_tests.c
+14 −7		tests/s3_tester.c
+10 −0		tests/s3_tester.h
+269 −6		tests/s3_util_tests.c