From c99559b7c92422aca8a5e718ee57e46fb6433c2c Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 18 Dec 2015 12:14:50 -0800 Subject: [PATCH] Add an option to disable chunked encoding. This is done to allow sha256 calculation of proper payload in PutObjectRequest and UploadPartRequest. Fixes #580 --- .../amazonaws/services/s3/AmazonS3Client.java | 12 +++-- .../services/s3/S3ClientOptions.java | 54 ++++++++++++++++++- .../services/s3/internal/AWSS3V4Signer.java | 9 +++- .../internal/S3V4AuthErrorRetryStrategy.java | 13 +++-- 4 files changed, 79 insertions(+), 9 deletions(-) diff --git a/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/AmazonS3Client.java b/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/AmazonS3Client.java index 7ad8a7a5a49f..4fd234864642 100644 --- a/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/AmazonS3Client.java +++ b/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/AmazonS3Client.java @@ -2993,7 +2993,8 @@ protected Signer createSigner(final Request request, final AmazonWebServiceRequest req = request.getOriginalRequest(); if (!(signer instanceof AWSS3V4Signer) && ((upgradeToSigV4(req)))) { - final AWSS3V4Signer v4Signer = new AWSS3V4Signer(); + boolean isChunkedEncodingDisabled = this.clientOptions.isChunkedEncodingDisabled(); + final AWSS3V4Signer v4Signer = new AWSS3V4Signer(isChunkedEncodingDisabled); // Always set the service name; if the user has overridden it via // setEndpoint(String, String, String), this will return the right // value. Otherwise it will return "s3", which is an appropriate @@ -3609,8 +3610,13 @@ private X invoke(Request request, checkHttps(originalRequest); ExecutionContext executionContext = createExecutionContext(originalRequest); // Retry V4 auth errors - executionContext.setAuthErrorRetryStrategy(new S3V4AuthErrorRetryStrategy(buildDefaultEndpointResolver( - getProtocol(request), bucket, key))); + S3RequestEndpointResolver endpointResolver = buildDefaultEndpointResolver(getProtocol(request), + bucket, key); + boolean isChunkedEncodingDisabled = this.clientOptions.isChunkedEncodingDisabled(); + S3V4AuthErrorRetryStrategy authErrRetryStrategy = new S3V4AuthErrorRetryStrategy(endpointResolver, + isChunkedEncodingDisabled); + executionContext.setAuthErrorRetryStrategy(authErrRetryStrategy); + // -- ends retry strategy. AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics(); // Binds the request metrics to the current request. request.setAWSRequestMetrics(awsRequestMetrics); diff --git a/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/S3ClientOptions.java b/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/S3ClientOptions.java index c588c5b6b59f..87dbda6fd2e3 100644 --- a/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/S3ClientOptions.java +++ b/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/S3ClientOptions.java @@ -22,13 +22,20 @@ public class S3ClientOptions { /** The default setting for use of path-style access */ public static final boolean DEFAULT_PATH_STYLE_ACCESS = false; + /** The default setting for use of chunked encoding */ + public static final boolean DEFAULT_CHUNKED_ENCODING_DISABLED = false; + /** Flag for use of path-style access */ private boolean pathStyleAccess = DEFAULT_PATH_STYLE_ACCESS; + /** Flag for user of chunked encoding */ + private boolean chunkedEncodingDisabled = DEFAULT_CHUNKED_ENCODING_DISABLED; + public S3ClientOptions() {} public S3ClientOptions( S3ClientOptions other ) { this.pathStyleAccess = other.pathStyleAccess; + this.chunkedEncodingDisabled = other.chunkedEncodingDisabled; } /** @@ -53,6 +60,28 @@ public boolean isPathStyleAccess() { return pathStyleAccess; } + /** + *

+ * Returns whether the client has chunked encoding disabled for all + * requests. + *

+ *

+ * The default behaviour is to detect API request which enables chunked + * encoding automatically for PutObjectRequest and UploadPartRequest. + * Setting this flag will result in disabling chunked encoding for all + * requests. + * Note: Enabling this option has performance implications since each + * payload will have to be checksum calculated. If your payload is large + * this will affect the overall time required to upload an object. + * Using this option is recommended only if your endpoint does not + * implement chunked uploading. + *

+ * @return True is the client should always enable chunked encoding. + */ + public boolean isChunkedEncodingDisabled() { + return chunkedEncodingDisabled; + } + /** *

* Configures the client to use path-style access for all requests. @@ -103,4 +132,27 @@ public S3ClientOptions withPathStyleAccess(boolean pathStyleAccess) { return this; } -} \ No newline at end of file + /** + *

+ * Configures the client to disable chunked encoding for all requests. + *

+ *

+ * The default behaviour is to detect API request which enables chunked + * encoding automatically for PutObjectRequest and UploadPartRequest. + * Setting this flag will result in disabling chunked encoding for all + * requests. + * Note: Enabling this option has performance implications since each + * payload will have to be checksum calculated. If your payload is large + * this will affect the overall time required to upload an object. + * Using this option is recommended only if your endpoint does not + * implement chunked uploading. + *

+ * @return The updated S3ClientOptions object with chunked encoding + * disabled setting. + */ + public S3ClientOptions disableChunkedEncoding() { + this.chunkedEncodingDisabled = true; + return this; + } + +} diff --git a/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/AWSS3V4Signer.java b/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/AWSS3V4Signer.java index 82f58b672f3d..1b26994c48ad 100644 --- a/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/AWSS3V4Signer.java +++ b/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/AWSS3V4Signer.java @@ -36,13 +36,15 @@ */ public class AWSS3V4Signer extends AWS4Signer { private static final String CONTENT_SHA_256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + private static boolean isChunkedEncodingDisabled; /** * Don't double-url-encode path elements; S3 expects path elements to be * encoded only once in the canonical URI. */ - public AWSS3V4Signer() { + public AWSS3V4Signer(boolean isChunkedEncodingDisabled) { super(false); + this.isChunkedEncodingDisabled = isChunkedEncodingDisabled; } /** @@ -120,6 +122,11 @@ protected String calculateContentHash(SignableRequest request) { private static boolean useChunkEncoding(SignableRequest request) { // Whether to use chunked encoding for signing the request boolean chunkedEncodingEnabled = false; + // if chunked encoding is explicitly disabled through client options + // return right here. + if (isChunkedEncodingDisabled) { + return chunkedEncodingEnabled; + } if (request.getOriginalRequestObject() instanceof PutObjectRequest || request.getOriginalRequestObject() instanceof UploadPartRequest) { chunkedEncodingEnabled = true; diff --git a/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/S3V4AuthErrorRetryStrategy.java b/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/S3V4AuthErrorRetryStrategy.java index 8143199e3be2..27d039b5f5e4 100644 --- a/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/S3V4AuthErrorRetryStrategy.java +++ b/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/S3V4AuthErrorRetryStrategy.java @@ -46,10 +46,13 @@ public class S3V4AuthErrorRetryStrategy implements AuthErrorRetryStrategy { + "buckets located in regions that require V4 signing."; private final S3RequestEndpointResolver endpointResolver; + private final boolean isChunkedEncodingDisabled; private final SdkPredicate sigV4RetryPredicate; - public S3V4AuthErrorRetryStrategy(S3RequestEndpointResolver endpointResolver) { + public S3V4AuthErrorRetryStrategy(S3RequestEndpointResolver endpointResolver, + boolean isChunkedEncodingDisabled) { this.endpointResolver = endpointResolver; + this.isChunkedEncodingDisabled = isChunkedEncodingDisabled; this.sigV4RetryPredicate = new IsSigV4RetryablePredicate(); } @@ -57,9 +60,11 @@ public S3V4AuthErrorRetryStrategy(S3RequestEndpointResolver endpointResolver) { * Currently only used for testing */ S3V4AuthErrorRetryStrategy(S3RequestEndpointResolver endpointResolver, - SdkPredicate isSigV4Retryable) { + SdkPredicate isSigV4Retryable, + boolean isChunkedEncodingDisabled) { this.endpointResolver = endpointResolver; this.sigV4RetryPredicate = isSigV4Retryable; + this.isChunkedEncodingDisabled = isChunkedEncodingDisabled; } @Override @@ -92,7 +97,7 @@ private AuthRetryParameters redirectToRegionInHeader(Request request, HttpRes /** * If the response doesn't have the x-amz-region header we have to resort to sending a request * to s3-external-1 - * + * * @return */ private AuthRetryParameters redirectToS3External() { @@ -107,7 +112,7 @@ private AuthRetryParameters redirectToS3External() { } private AWSS3V4Signer buildSigV4Signer(final String region) { - AWSS3V4Signer v4Signer = new AWSS3V4Signer(); + AWSS3V4Signer v4Signer = new AWSS3V4Signer(isChunkedEncodingDisabled); v4Signer.setRegionName(region); v4Signer.setServiceName(AmazonS3Client.S3_SERVICE_NAME); return v4Signer;