From bb352a7efb42c8cd184f201384b171638d3fb1da Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Nov 2025 10:36:52 -0500 Subject: [PATCH 1/4] pass null part size to CRT when not configured --- pom.xml | 2 +- .../s3/internal/crt/S3CrtAsyncHttpClient.java | 2 +- .../crt/S3NativeClientConfiguration.java | 28 +++++++++++++------ .../crt/S3CrtAsyncHttpClientTest.java | 19 +++++++++++++ 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 0068f081fbba..bec0c82929a4 100644 --- a/pom.xml +++ b/pom.xml @@ -129,7 +129,7 @@ 3.1.5 1.17.1 1.37 - 0.39.4 + 0.40.1 5.10.0 diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java index 1fed55813d33..3e69d4d29abc 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java @@ -104,7 +104,6 @@ private S3ClientOptions createS3ClientOption() { .withCredentialsProvider(s3NativeClientConfiguration.credentialsProvider()) .withClientBootstrap(s3NativeClientConfiguration.clientBootstrap()) .withTlsContext(s3NativeClientConfiguration.tlsContext()) - .withPartSize(s3NativeClientConfiguration.partSizeBytes()) .withMultipartUploadThreshold(s3NativeClientConfiguration.thresholdInBytes()) .withComputeContentMd5(false) .withEnableS3Express(true) @@ -120,6 +119,7 @@ private S3ClientOptions createS3ClientOption() { if (Boolean.FALSE.equals(s3NativeClientConfiguration.isUseEnvironmentVariableValues())) { options.withProxyEnvironmentVariableSetting(disabledHttpProxyEnvironmentVariableSetting()); } + Optional.ofNullable(s3NativeClientConfiguration.partSizeBytes()).ifPresent(options::withPartSize); Optional.ofNullable(s3NativeClientConfiguration.proxyOptions()).ifPresent(options::withProxyOptions); Optional.ofNullable(s3NativeClientConfiguration.connectionTimeout()) .map(Duration::toMillis) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java index f43cbf84d46e..370dea6918fc 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java @@ -51,7 +51,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable { private final ClientBootstrap clientBootstrap; private final CrtCredentialsProviderAdapter credentialProviderAdapter; private final CredentialsProvider credentialsProvider; - private final long partSizeInBytes; + private final Long partSizeInBytes; private final long thresholdInBytes; private final double targetThroughputInGbps; private final int maxConcurrency; @@ -85,10 +85,8 @@ public S3NativeClientConfiguration(Builder builder) { this.credentialsProvider = credentialProviderAdapter.crtCredentials(); - this.partSizeInBytes = builder.partSizeInBytes == null ? DEFAULT_PART_SIZE_IN_BYTES : - builder.partSizeInBytes; - this.thresholdInBytes = builder.thresholdInBytes == null ? this.partSizeInBytes : - builder.thresholdInBytes; + this.partSizeInBytes = builder.partSizeInBytes; + this.thresholdInBytes = resolveThresholdInBytes(builder); this.targetThroughputInGbps = builder.targetThroughputInGbps == null ? DEFAULT_TARGET_THROUGHPUT_IN_GBPS : builder.targetThroughputInGbps; @@ -98,8 +96,7 @@ public S3NativeClientConfiguration(Builder builder) { this.endpointOverride = builder.endpointOverride; - this.readBufferSizeInBytes = builder.readBufferSizeInBytes == null ? - partSizeInBytes * 10 : builder.readBufferSizeInBytes; + this.readBufferSizeInBytes = resolveReadBufferSizeInBytes(builder); if (builder.httpConfiguration != null) { this.proxyOptions = resolveProxy(builder.httpConfiguration.proxyConfiguration(), tlsContext).orElse(null); @@ -115,6 +112,21 @@ public S3NativeClientConfiguration(Builder builder) { this.useEnvironmentVariableProxyOptionsValues = resolveUseEnvironmentVariableValues(builder); } + private Long resolveReadBufferSizeInBytes(Builder builder) { + if (builder.readBufferSizeInBytes != null) { + return builder.readBufferSizeInBytes; + } + long partSize = this.partSizeInBytes == null ? DEFAULT_PART_SIZE_IN_BYTES : this.partSizeInBytes; + return partSize * 10; + } + + private long resolveThresholdInBytes(Builder builder) { + if (builder.thresholdInBytes != null) { + return builder.thresholdInBytes; + } + return this.partSizeInBytes == null ? DEFAULT_PART_SIZE_IN_BYTES : this.partSizeInBytes; + } + private static Boolean resolveUseEnvironmentVariableValues(Builder builder) { if (builder != null && builder.httpConfiguration != null && builder.httpConfiguration.proxyConfiguration() != null) { return builder.httpConfiguration.proxyConfiguration().isUseEnvironmentVariableValues(); @@ -159,7 +171,7 @@ public TlsContext tlsContext() { return tlsContext; } - public long partSizeBytes() { + public Long partSizeBytes() { return partSizeInBytes; } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index 776b69c4a10a..1a952849b60b 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -486,6 +486,25 @@ void build_partSizeConfigured_shouldApplyToThreshold() { } } + @Test + public void build_noPartSize_shouldUseDefaultsForThresholdAndReadWindowSize() { + S3NativeClientConfiguration configuration = + S3NativeClientConfiguration.builder() + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", + "test"))) + .signingRegion("us-west-2") + .build(); + try (S3CrtAsyncHttpClient client = + (S3CrtAsyncHttpClient) S3CrtAsyncHttpClient.builder() + .s3ClientConfiguration(configuration).build()) { + long defaultPartSizeInBytes = 1024 * 1024L * 8L; + S3ClientOptions clientOptions = client.s3ClientOptions(); + assertThat(clientOptions.getPartSize()).isEqualTo(0); + assertThat(clientOptions.getMultiPartUploadThreshold()).isEqualTo(defaultPartSizeInBytes); + assertThat(clientOptions.getInitialReadWindowSize()).isEqualTo(defaultPartSizeInBytes * 10); + } + } + @Test void build_nullHttpConfiguration() { S3NativeClientConfiguration configuration = From 49f86fe550ce1e276eeece711a09d54ad428b67c Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Nov 2025 11:00:20 -0500 Subject: [PATCH 2/4] add changelog --- .changes/next-release/feature-s3-3d1dcf9.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/next-release/feature-s3-3d1dcf9.json diff --git a/.changes/next-release/feature-s3-3d1dcf9.json b/.changes/next-release/feature-s3-3d1dcf9.json new file mode 100644 index 000000000000..8351169e98a2 --- /dev/null +++ b/.changes/next-release/feature-s3-3d1dcf9.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "s3", + "contributor": "", + "description": "Pass null part-size to CRT when not configured" +} From 795e2631b5b10c5df739c5bd6067e08905bc22f5 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Nov 2025 14:04:26 -0500 Subject: [PATCH 3/4] fix crt resource management test - wrong final part content-range --- .../crt/S3CrtClientGetObjectResourceManagementTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtClientGetObjectResourceManagementTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtClientGetObjectResourceManagementTest.java index 6de8ae6f7380..d18d93318f1f 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtClientGetObjectResourceManagementTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtClientGetObjectResourceManagementTest.java @@ -144,7 +144,7 @@ private static void stubGetObjectCalls() { } // final part - String contentRange = "bytes " + PART_SIZE * numOfParts + "-" + (totalContentSize - 1) + "/" + totalContentSize; + String contentRange = "bytes " + PART_SIZE * (numOfParts - 1) + "-" + (totalContentSize - 1) + "/" + totalContentSize; String range = "bytes=" + PART_SIZE * (numOfParts - 1) + "-" + (totalContentSize - 1); stubFor(get(anyUrl()).withHeader("Range", equalTo(range)).willReturn(aResponse().withStatus(200) .withHeader("content-length", String.valueOf(finalPartSize)) From a22df8f509948ab0d6c415d584d0727f1933ff05 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Nov 2025 17:48:56 -0500 Subject: [PATCH 4/4] regression test logs --- .../awssdk/services/s3/regression/upload/UploadConfig.java | 2 +- .../s3/regression/upload/UploadStreamingRegressionTesting.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadConfig.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadConfig.java index cc197a225317..e79dcb960825 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadConfig.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadConfig.java @@ -106,7 +106,7 @@ public void setPayloadSigning(boolean payloadSigning) { @Override public String toString() { - return ToString.builder("FlattenUploadConfig") + return ToString.builder("UploadConfig") .add("bucketType", bucketType) .add("forcePathStyle", forcePathStyle) .add("requestChecksumValidation", requestChecksumValidation) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadStreamingRegressionTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadStreamingRegressionTesting.java index aa33a24a6c9d..0c06d3474c4d 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadStreamingRegressionTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadStreamingRegressionTesting.java @@ -176,6 +176,7 @@ protected TestCallable callTmUpload(PutObjectRequest request, CompletedUpload completedUpload = CompletableFutureUtils.joinLikeSync(upload.completionFuture()); return completedUpload.response(); } catch (Exception e) { + LOG.error(() -> "Upload failed for " + config.toString()); throw new RuntimeException(e); } finally { transferManager.close();