From 588b102c326a40c07fe66961d70b2288fe30f79b Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 4 Dec 2023 02:07:04 -0800 Subject: [PATCH 01/50] Adding interfaces for Progress Listener and state capturing Progress Snapshots --- .../ExecutionSuccessObjectRequest.java | 27 ++ .../progress/listener/ProgressListener.java | 395 ++++++++++++++++++ .../listener/snapshot/ListenerProgress.java | 27 ++ .../listener/snapshot/ProgressSnapshot.java | 66 +++ 4 files changed, 515 insertions(+) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java new file mode 100644 index 000000000000..7ae61df403ed --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java @@ -0,0 +1,27 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress.listener; + +import software.amazon.awssdk.core.SdkResponse; + +public interface ExecutionSuccessObjectRequest extends ProgressListener.Context.ExecutionSuccess { + /** + * Return the {@link SdkResponse} associated with this request + */ + default SdkResponse response() { + throw new UnsupportedOperationException(); + } +} \ No newline at end of file diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java new file mode 100644 index 000000000000..5384f52d47cd --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -0,0 +1,395 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress.listener; + +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.progress.listener.snapshot.ProgressSnapshot; +import software.amazon.awssdk.http.SdkHttpRequest; + +/** + * The {@link ProgressListener} interface may be implemented by your application in order to receive event-driven updates on + * the progress of a service call. When you construct an {@link PutObjectRequest} or {@link + * UploadPartRequest} request submitted to the Sdk, you may provide a variable number of {@link + * ProgressListener}s to be associated with that request. Then, throughout the lifecycle of the request, + * the Sdk + * will invoke the provided {@link ProgressListener}s when important events occur, like additional bytes being transferred, + * allowing you to monitor the ongoing progress of the transaction. + *

+ * Each {@link ProgressListener} callback is invoked with an immutable {@link Context} object. Depending on the current + * lifecycle + * of the request, different {@link Context} objects have different attributes available (indicated by the provided context + * interface). Most notably, every callback is given access to the current {@link ProgressSnapshot}, which contains + * helpful progress-related methods like {@link ProgressSnapshot#transferredBytes()} and {@link + * ProgressSnapshot#ratioTransferred()}. + *

+ * A successful transfer callback lifecycle is sequenced as follows: + *

    + *
  1. {@link #requestPrepared(Context.RequestPrepared)} - A new Request has been initiated. This method is called + * exactly once per transfer.
    2. + * + *
  2. {@link #requestBytesSent(Context.RequestBytesSent)} - Additional bytes have been sent. This + * method may be called many times per request, depending on the request payload size and I/O buffer sizes. + *
  3. {@link #responseBytesReceived(Context.ResponseBytesReceived)} - Additional bytes have been received. This + * method may be called many times per request, depending on the response payload size and I/O buffer sizes. + *
  4. {@link #executionSuccess(Context.ExecutionSuccess)} - The transfer has completed successfully. This method is called + * exactly once for a successful transfer.
    5. + *
+ * For every failed attempt {@link #attemptFailure(Context.AttemptFailure)} will be called exactly once. + * + *

+ * There are a few important rules and best practices that govern the usage of {@link ProgressListener}s: + *

    + *
  1. {@link ProgressListener} implementations should not block, sleep, or otherwise delay the calling thread. If you need + * to perform blocking operations, you should schedule them in a separate thread or executor that you control.
    2. + *
  2. Be mindful that {@link #requestBytesSent(Context.RequestBytesSent)} or + * {@link #responseBytesReceived(Context.ResponseBytesReceived)} + * may be called extremely often for large payloads + * (subject to I/O buffer sizes). Be careful in implementing expensive operations as a side effect. Consider rate-limiting + * your side effect operations, if needed.
    3. + *
  3. {@link ProgressListener}s may be invoked by different threads. If your {@link ProgressListener} is stateful, + * ensure that it is also thread-safe.
    4. + *
  4. {@link ProgressListener}s are not intended to be used for control flow, and therefore your implementation + * should not throw. Any thrown exceptions will be suppressed and logged as an error.
    5. + *
+ *

+ * A classical use case of {@link ProgressListener} is to create a progress bar to monitor an ongoing transfer's progress. + * Refer to the implementation of {@link LoggingProgressListener} for a basic example, or test it in your application by providing + * the listener as part of your {@link SdkHttpRequest}. E.g., + * 

{@code
+ * AmazonS3Client s3Client = new AmazonS3Client(awsCredentials);
+ * PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+ *                                        .bucket("bucket")
+ *                                        .key("key")
+ *                                        .overrideConfiguration(o -> o.addProgressListener(LoggingTransferListener.create())
+ *                                        .build();
+ * s3Client.putObject(putObjectRequest);
+ * }
+ * And then a successful transfer may output something similar to: + * 
+ * Request initiated...
+ * |                    | 0.0%
+ * |==                  | 12.5%
+ * |=====               | 25.0%
+ * |=======             | 37.5%
+ * |==========          | 50.0%
+ * |============        | 62.5%
+ * |===============     | 75.0%
+ * |=================   | 87.5%
+ * |====================| 100.0%
+ * Request execution successful!
+ *
+ */ +@SdkPublicApi +public interface ProgressListener { + + /** + * This method is called right after a request object is marshalled and ready to be sent to the service + *

+ * Available context attributes: + *

    + *
  1. {@link Context.RequestPrepared#request()}
    2. + *
  2. {@link Context.RequestPrepared#progressSnapshot()}
    3. + *
+ */ + default void requestPrepared(Context.RequestPrepared context) { + } + + /** + * This method is called after the request transaction is initiated, i.e. request header is sent to the service + *

+ * Available context attributes: + *

    + *
  1. {@link Context.RequestHeaderSent#request()}
    2. + *
  2. {@link Context.RequestHeaderSent#progressSnapshot()}
    3. + *
+ */ + default void requestHeaderSent(Context.RequestHeaderSent context) { + } + + /** + * This method is called with any additional payload bytes sent; it may be called many times per request, depending on the + * payload size. + *

+ * Available context attributes: + *

    + *
  1. {@link Context.RequestBytesSent#request()}
    2. + *
  2. {@link Context.RequestBytesSent#progressSnapshot()}
    3. + *
+ */ + default void requestBytesSent(Context.RequestBytesSent context) { + } + + /** + * The service returns the response headers + *

+ * Available context attributes: + *

    + *
  1. {@link Context.ResponseHeaderReceived#request()}
    2. + *
  2. {@link Context.ResponseHeaderReceived#progressSnapshot()}
    3. + *
+ */ + default void responseHeaderReceived(Context.ResponseHeaderReceived context) { + } + + /** + * Additional bytes received + *

+ * Available context attributes: + *

    + *
  1. {@link Context.ResponseBytesReceived#request()}
    2. + *
  2. {@link Context.ResponseBytesReceived#progressSnapshot()}
    3. + *
+ */ + default void responseBytesReceived(Context.ResponseBytesReceived context) { + } + + /** + * Additional bytes received + *

+ * Available context attributes: + *

    + *
  1. {@link Context.ResponseBytesReceived#request()}
    2. + *
  2. {@link Context.ResponseBytesReceived#progressSnapshot()}
    3. + *
+ */ + default void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseBytesReceived context) { + } + + /** + * Successful request execution + *

+ * Available context attributes: + *

    + *
  1. {@link Context.ExecutionSuccess#request()}
    2. + *
  2. {@link Context.ExecutionSuccess#progressSnapshot()}
    3. + *
  3. {@link Context.ExecutionSuccess#executionSuccess()} ()}
    4. + *
+ */ + default void executionSuccess(Context.ExecutionSuccess context) { + } + + /** + * This method is called for every failure of a request attempt. + * An ideal implementation would invoke executionFailure for a number of attemptFailures greater than a threshold + *

+ * Available context attributes: + *

    + *
  1. {@link Context.AttemptFailure#request()}
    2. + *
  2. {@link Context.AttemptFailure#progressSnapshot()}
    3. + *
+ */ + default void attemptFailure(Context.AttemptFailure context) { + } + + /** + * A wrapper class that groups together the different context interfaces that are exposed to {@link ProgressListener}s. + *

+ * Successful transfer interface hierarchy: + *

    + *
  1. {@link RequestPrepared}
    2. + *
  2. {@link RequestHeaderSent}
    3. + *
  3. {@link RequestBytesSent}
    4. + *
  4. {@link ResponseHeaderReceived}
    5. + *
  5. {@link ResponseBytesReceived}
    6. + *
  6. {@link ExecutionSuccess}
    7. + *
+ * Failed transfer interface hierarchy: + *
    + *
  1. {@link RequestPrepared}
    2. + *
  2. {@link AttemptFailure}
    3. + *
  3. {@link ExecutionFailure}
    4. + *
+ * + * @see ProgressListener + */ + @SdkProtectedApi + final class Context { + private Context() { + } + + /** + * A new transfer has been initiated. + *

+ * Available context attributes: + *

    + *
  1. {@link RequestPrepared#request()}
    2. + *
  2. {@link RequestPrepared#progressSnapshot()}
    3. + *
+ */ + @Immutable + @ThreadSafe + @SdkPublicApi + @SdkPreviewApi + public interface RequestPrepared { + /** + * The {@link SdkHttppRequest} that was submitted to SDK, i.e., the {@link PutObjectRequest} or + * {@link GetObjectRequest}. + */ + SdkHttpRequest request(); + + /** + * The immutable {@link ProgressSnapahot} for this specific update. + */ + ProgressSnapshot progressSnapshot(); + } + + /** + * The submitted {@link SdkHttppRequest} request header was successfully sent to the service + *

+ * Available context attributes: + *

    + *
  1. {@link RequestHeaderSent#request()}
    2. + *
  2. {@link RequestHeaderSent#progressSnapshot()}
    3. + *
+ */ + @Immutable + @ThreadSafe + @SdkPublicApi + @SdkPreviewApi + public interface RequestHeaderSent extends RequestPrepared { + } + + /** + * Additional bytes sent + *

+ * Available context attributes: + *

    + *
  1. {@link RequestBytesSent#request()}
    2. + *
  2. {@link RequestBytesSent#progressSnapshot()}
    3. + *
+ */ + @Immutable + @ThreadSafe + @SdkPublicApi + @SdkPreviewApi + public interface RequestBytesSent extends RequestHeaderSent { + } + + /** + * Service has sent back a response header, denoting the start of response reception + *

+ * Available context attributes: + *

    + *
  1. {@link ResponseHeaderReceived#request()}
    2. + *
  2. {@link ResponseHeaderReceived#progressSnapshot()}
    3. + *
+ */ + @Immutable + @ThreadSafe + @SdkPublicApi + @SdkPreviewApi + public interface ResponseHeaderReceived extends RequestBytesSent { + } + + /** + * Additional bytes received + *

+ * Available context attributes: + *

    + *
  1. {@link ResponseBytesReceived#request()}
    2. + *
  2. {@link ResponseBytesReceived#progressSnapshot()}
    3. + *
+ */ + @Immutable + @ThreadSafe + @SdkPublicApi + @SdkPreviewApi + public interface ResponseBytesReceived extends ResponseHeaderReceived { + } + + /** + * The request execution is successful. + *

+ * Available context attributes: + *

    + *
  1. {@link ExecutionSuccess#request()}
    2. + *
  2. {@link ExecutionSuccess#progressSnapshot()}
    3. + *
  3. {@link ExecutionSuccess#executionSuccess()} ()}
    4. + *
+ */ + @Immutable + @ThreadSafe + @SdkPublicApi + @SdkPreviewApi + public interface ExecutionSuccess extends ResponseBytesReceived { + /** + * The successful completion of a request submitted to the Sdk + */ + ExecutionSuccessObjectRequest executionSuccess(); + } + + /** + * For Expect: 100-continue embedded requests, the service returning anything other than 100 continue + * indicates a service error. The progress state captured indicates that no bytes are received. + *

+ * Available context attributes: + *

    + *
  1. {@link AttemptFailureResponseBytesReceived#request()}
    2. + *
  2. {@link AttemptFailureResponseBytesReceived#progressSnapshot()}
    3. + *
+ */ + @Immutable + @ThreadSafe + @SdkPublicApi + @SdkPreviewApi + public interface AttemptFailureResponseBytesReceived extends ResponseHeaderReceived { + } + + /** + * The request execution attempt failed. + *

+ * Available context attributes: + *

    + *
  1. {@link AttemptFailure#request()}
    2. + *
  2. {@link AttemptFailure#progressSnapshot()}
    3. + *
+ */ + @Immutable + @ThreadSafe + @SdkPublicApi + @SdkPreviewApi + public interface AttemptFailure extends RequestPrepared { + } + + /** + * The request execution failed. + *

+ * Available context attributes: + *

    + *
  1. {@link ExecutionFailure#request()}
    2. + *
  2. {@link ExecutionFailure#progressSnapshot()}
    3. + *
  3. {@link ExecutionFailure#exception()}
    4. + *
+ */ + @Immutable + @ThreadSafe + @SdkPublicApi + @SdkPreviewApi + public interface ExecutionFailure extends RequestPrepared { + /** + * The exception associated with the failed request. + */ + Throwable exception(); + } + } +} \ No newline at end of file diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java new file mode 100644 index 000000000000..62223f2d51e0 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java @@ -0,0 +1,27 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress.listener.snapshot; + +public interface ListenerProgress { + + /** + * Takes a snapshot of the request execution progress + * represented by an immutable {@link ProgressSnapshot}. + */ + + ProgressSnapshot progressSnapshot(); + +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java new file mode 100644 index 000000000000..148df78c84a9 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress.listener.snapshot; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; + +public interface ProgressSnapshot { + /** + * The total number of bytes that have been sent or received so far. + */ + long transferredBytes(); + + /** + * Time at which transaction started + */ + Instant startTime(); + + /** + * Elapsed time since the start of the transaction + */ + Duration elapsedTime(); + + /** + * If transaction size is known, estimate time remaining for transaction completion + */ + Optional estimatedTimeRemaining(); + + /** + * Rate of transfer + */ + double averageBytesPer(TimeUnit timeUnit); + + /** + * The total size of the transfer, in bytes, or {@link Optional#empty()} if unknown. + */ + OptionalLong totalTransferSize(); + /** + * The ratio of the {@link #totalBytes()} that has been transferred so far, or {@link Optional#empty()} if unknown. + * This method depends on the {@link #totalBytes()} being known in order to return non-empty. + */ + OptionalDouble ratioTransferred(); + + /** + * The total number of bytes that are remaining to be transferred, or {@link Optional#empty()} if unknown. This method depends + * on the {@link #totalBytes()} being known in order to return non-empty. + */ + OptionalLong remainingBytes(); +} From da26d3e9a5633c63469dccd23b4751749efd75b0 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 4 Dec 2023 16:27:42 -0800 Subject: [PATCH 02/50] Modified Indentation --- .../ExecutionSuccessObjectRequest.java | 6 ++++++ .../progress/listener/ProgressListener.java | 18 +++++++++--------- .../listener/snapshot/ListenerProgress.java | 18 ++++++++++++------ .../listener/snapshot/ProgressSnapshot.java | 7 +++++++ 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java index 7ae61df403ed..05a5be107861 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java @@ -15,9 +15,15 @@ package software.amazon.awssdk.core.progress.listener; +import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.core.SdkResponse; +/** + * A successfully completed single object request. + */ +@SdkPublicApi public interface ExecutionSuccessObjectRequest extends ProgressListener.Context.ExecutionSuccess { + /** * Return the {@link SdkResponse} associated with this request */ diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index 5384f52d47cd..de3818e4a62d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -163,15 +163,15 @@ default void responseHeaderReceived(Context.ResponseHeaderReceived context) { default void responseBytesReceived(Context.ResponseBytesReceived context) { } - /** - * Additional bytes received - *

- * Available context attributes: - *

    - *
  1. {@link Context.ResponseBytesReceived#request()}
    2. - *
  2. {@link Context.ResponseBytesReceived#progressSnapshot()}
    3. - *
- */ + /** + * Additional bytes received + *

+ * Available context attributes: + *

    + *
  1. {@link Context.ResponseBytesReceived#request()}
    2. + *
  2. {@link Context.ResponseBytesReceived#progressSnapshot()}
    3. + *
+ */ default void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseBytesReceived context) { } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java index 62223f2d51e0..ba4433b627a9 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java @@ -15,13 +15,19 @@ package software.amazon.awssdk.core.progress.listener.snapshot; -public interface ListenerProgress { +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; - /** - * Takes a snapshot of the request execution progress - * represented by an immutable {@link ProgressSnapshot}. - */ +@Immutable +@ThreadSafe +@SdkPublicApi +public interface ListenerProgress { - ProgressSnapshot progressSnapshot(); + /** + * Takes a snapshot of the request execution progress + * represented by an immutable {@link ProgressSnapshot}. + */ + ProgressSnapshot progressSnapshot(); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java index 148df78c84a9..70721f814c38 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java @@ -21,7 +21,13 @@ import java.util.OptionalDouble; import java.util.OptionalLong; import java.util.concurrent.TimeUnit; +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; +@Immutable +@ThreadSafe +@SdkPublicApi public interface ProgressSnapshot { /** * The total number of bytes that have been sent or received so far. @@ -52,6 +58,7 @@ public interface ProgressSnapshot { * The total size of the transfer, in bytes, or {@link Optional#empty()} if unknown. */ OptionalLong totalTransferSize(); + /** * The ratio of the {@link #totalBytes()} that has been transferred so far, or {@link Optional#empty()} if unknown. * This method depends on the {@link #totalBytes()} being known in order to return non-empty. From 99febb6355d349bc6aa393493cbabd55833f5533 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 4 Dec 2023 16:30:36 -0800 Subject: [PATCH 03/50] Added new change script --- .changes/next-release/feature-AWSSDKforJavav2-b190cbb.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/next-release/feature-AWSSDKforJavav2-b190cbb.json diff --git a/.changes/next-release/feature-AWSSDKforJavav2-b190cbb.json b/.changes/next-release/feature-AWSSDKforJavav2-b190cbb.json new file mode 100644 index 000000000000..9580df40a297 --- /dev/null +++ b/.changes/next-release/feature-AWSSDKforJavav2-b190cbb.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "AWS SDK for Java v2", + "contributor": "anirudh9391", + "description": "Adding interface methods for Progress Listener and state capturing Progress Snapshots" +} From 7e70224545e0cad83e9f68d5272ce4f3ad7dada1 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 4 Dec 2023 16:32:22 -0800 Subject: [PATCH 04/50] Revert "Added new change script" This reverts commit 99febb6355d349bc6aa393493cbabd55833f5533. --- .changes/next-release/feature-AWSSDKforJavav2-b190cbb.json | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .changes/next-release/feature-AWSSDKforJavav2-b190cbb.json diff --git a/.changes/next-release/feature-AWSSDKforJavav2-b190cbb.json b/.changes/next-release/feature-AWSSDKforJavav2-b190cbb.json deleted file mode 100644 index 9580df40a297..000000000000 --- a/.changes/next-release/feature-AWSSDKforJavav2-b190cbb.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": "feature", - "category": "AWS SDK for Java v2", - "contributor": "anirudh9391", - "description": "Adding interface methods for Progress Listener and state capturing Progress Snapshots" -} From 5e9057e1d6bac8dc563b478ea2db38add1a05107 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 4 Dec 2023 16:32:31 -0800 Subject: [PATCH 05/50] Added new change script --- .changes/next-release/feature-AWSSDKforJavav2-83dc9e4.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/next-release/feature-AWSSDKforJavav2-83dc9e4.json diff --git a/.changes/next-release/feature-AWSSDKforJavav2-83dc9e4.json b/.changes/next-release/feature-AWSSDKforJavav2-83dc9e4.json new file mode 100644 index 000000000000..5e66a5c18528 --- /dev/null +++ b/.changes/next-release/feature-AWSSDKforJavav2-83dc9e4.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "AWS SDK for Java v2", + "contributor": "anirudh9391", + "description": "General Progress Listeners" +} From e607d6ca39b7640681b88bc49b8df5b8e1f6ed13 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 4 Dec 2023 17:11:43 -0800 Subject: [PATCH 06/50] Moved ProgressSnapshot to its own folder --- .../amazon/awssdk/core/progress/listener/ProgressListener.java | 2 +- .../core/progress/{listener => }/snapshot/ListenerProgress.java | 2 +- .../core/progress/{listener => }/snapshot/ProgressSnapshot.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/{listener => }/snapshot/ListenerProgress.java (93%) rename core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/{listener => }/snapshot/ProgressSnapshot.java (97%) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index de3818e4a62d..40f7c036f15f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -20,7 +20,7 @@ import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; -import software.amazon.awssdk.core.progress.listener.snapshot.ProgressSnapshot; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; import software.amazon.awssdk.http.SdkHttpRequest; /** diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java similarity index 93% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java index ba4433b627a9..e1692fa27752 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ListenerProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.progress.listener.snapshot; +package software.amazon.awssdk.core.progress.snapshot; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java similarity index 97% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java index 70721f814c38..89f5fb138819 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/snapshot/ProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.progress.listener.snapshot; +package software.amazon.awssdk.core.progress.snapshot; import java.time.Duration; import java.time.Instant; From 3d26526821ff824d5874837c970d9d6ab75553a9 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Tue, 5 Dec 2023 15:35:34 -0800 Subject: [PATCH 07/50] Added sdkResponse to Progress Snapshot --- .../core/progress/snapshot/DefaultProgressSnapshot.java | 2 ++ .../awssdk/core/progress/snapshot/ProgressSnapshot.java | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java new file mode 100644 index 000000000000..074fbea14010 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java @@ -0,0 +1,2 @@ +package software.amazon.awssdk.core.progress.snapshot;public class DefaultProgressSnapshot { +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java index 89f5fb138819..e17c3bf48ddf 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java @@ -24,6 +24,7 @@ import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.SdkResponse; @Immutable @ThreadSafe @@ -49,6 +50,11 @@ public interface ProgressSnapshot { */ Optional estimatedTimeRemaining(); + /** + * The SDK response, or {@link Optional#empty()} if unknown. + */ + Optional sdkResponse(); + /** * Rate of transfer */ From 48a66175bf7f5b0d717a4ec80475a2fab150f31d Mon Sep 17 00:00:00 2001 From: Krishnan Date: Tue, 5 Dec 2023 15:53:40 -0800 Subject: [PATCH 08/50] Change totalTransferSize to totalBytes --- .../amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java index e17c3bf48ddf..038ea2777956 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java @@ -63,7 +63,7 @@ public interface ProgressSnapshot { /** * The total size of the transfer, in bytes, or {@link Optional#empty()} if unknown. */ - OptionalLong totalTransferSize(); + OptionalLong totalBytes(); /** * The ratio of the {@link #totalBytes()} that has been transferred so far, or {@link Optional#empty()} if unknown. From b0967658dac900533a959f551bf377a07fda389e Mon Sep 17 00:00:00 2001 From: Krishnan Date: Tue, 5 Dec 2023 16:28:35 -0800 Subject: [PATCH 09/50] Modify return types of start and elapsedTime to Optional --- .../awssdk/core/progress/snapshot/ProgressSnapshot.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java index 038ea2777956..2a037519a8be 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java @@ -38,12 +38,12 @@ public interface ProgressSnapshot { /** * Time at which transaction started */ - Instant startTime(); + Optional startTime(); /** * Elapsed time since the start of the transaction */ - Duration elapsedTime(); + Optional elapsedTime(); /** * If transaction size is known, estimate time remaining for transaction completion From b558e6bc12d54179ebc7f84e0e7e9d019e07c9dc Mon Sep 17 00:00:00 2001 From: Krishnan Date: Thu, 7 Dec 2023 20:15:39 -0800 Subject: [PATCH 10/50] Progress Listener Interface definition --- .../ExecutionSuccessObjectRequest.java | 33 ---- .../progress/listener/ProgressListener.java | 175 ++++++++++++++---- .../snapshot/DefaultProgressSnapshot.java | 2 - .../progress/snapshot/ListenerProgress.java | 1 - .../progress/snapshot/ProgressSnapshot.java | 19 +- 5 files changed, 144 insertions(+), 86 deletions(-) delete mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java delete mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java deleted file mode 100644 index 05a5be107861..000000000000 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ExecutionSuccessObjectRequest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.progress.listener; - -import software.amazon.awssdk.annotations.SdkPublicApi; -import software.amazon.awssdk.core.SdkResponse; - -/** - * A successfully completed single object request. - */ -@SdkPublicApi -public interface ExecutionSuccessObjectRequest extends ProgressListener.Context.ExecutionSuccess { - - /** - * Return the {@link SdkResponse} associated with this request - */ - default SdkResponse response() { - throw new UnsupportedOperationException(); - } -} \ No newline at end of file diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index 40f7c036f15f..a78fd8f01672 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -20,29 +20,34 @@ import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.SdkHttpResponse; /** * The {@link ProgressListener} interface may be implemented by your application in order to receive event-driven updates on - * the progress of a service call. When you construct an {@link PutObjectRequest} or {@link - * UploadPartRequest} request submitted to the Sdk, you may provide a variable number of {@link - * ProgressListener}s to be associated with that request. Then, throughout the lifecycle of the request, - * the Sdk - * will invoke the provided {@link ProgressListener}s when important events occur, like additional bytes being transferred, - * allowing you to monitor the ongoing progress of the transaction. + * the progress of a service call. When an {@link SdkRequest} like {@link PutObjectRequest} or {@link + * UploadPartRequest} is submitted to the Sdk, you may provide a variable number of {@link + * ProgressListener}s to be associated with that request. + *

+ * While ExecutionInterceptors are focused on the lifecycle of the + * request within the SDK, ProgressListeners are focused on the lifecycle of the request within the HTTP client. Throughout + * the lifecycle of the client request to which it is attached, the Sdk will invoke the provided {@link ProgressListener}s when + * important events occur, like additional bytes being transferred, allowing you to monitor the ongoing progress of the + * transaction. *

* Each {@link ProgressListener} callback is invoked with an immutable {@link Context} object. Depending on the current - * lifecycle - * of the request, different {@link Context} objects have different attributes available (indicated by the provided context - * interface). Most notably, every callback is given access to the current {@link ProgressSnapshot}, which contains + * lifecycle of the request, different {@link Context} objects have different attributes available (indicated by the provided + * context interface). Most notably, every callback is given access to the current {@link ProgressSnapshot}, which contains * helpful progress-related methods like {@link ProgressSnapshot#transferredBytes()} and {@link * ProgressSnapshot#ratioTransferred()}. *

* A successful transfer callback lifecycle is sequenced as follows: *

    - *
  1. {@link #requestPrepared(Context.RequestPrepared)} - A new Request has been initiated. This method is called - * exactly once per transfer.
    2. + *
  2. {@link #requestPrepared(Context.RequestPrepared)} - This method is called for every newly initiated SdkRequest, + * after it is marshalled and signed, before it is sent to the service
    3. *
      Available context attributes: *
    • {@link Context.RequestPrepared#request()}
      • *
    • {@link Context.RequestPrepared#progressSnapshot()}
      • @@ -52,9 +57,9 @@ *
    • {@link #responseBytesReceived(Context.ResponseBytesReceived)} - Additional bytes have been received. This * method may be called many times per request, depending on the response payload size and I/O buffer sizes. *
    • {@link #executionSuccess(Context.ExecutionSuccess)} - The transfer has completed successfully. This method is called - * exactly once for a successful transfer.
      • + * for every successful transfer. *
- * For every failed attempt {@link #attemptFailure(Context.AttemptFailure)} will be called exactly once. + * For every failed attempt {@link #attemptFailure(Context.AttemptFailure)}. * *

* There are a few important rules and best practices that govern the usage of {@link ProgressListener}s: @@ -103,11 +108,14 @@ public interface ProgressListener { /** - * This method is called right after a request object is marshalled and ready to be sent to the service + * This method is called right after a {@link SdkRequest} is marshalled, signed, transformed into an {@link SdkHttpRequest} and ready to + * be sent to the service + * After this method has returned, either requestHeaderSent or executionFailure will always be invoked *

* Available context attributes: *

    *
  1. {@link Context.RequestPrepared#request()}
    2. + *
  2. {@link Context.RequestPrepared#httpRequest()}
    3. *
  3. {@link Context.RequestPrepared#progressSnapshot()}
    4. *
*/ @@ -116,10 +124,13 @@ default void requestPrepared(Context.RequestPrepared context) { /** * This method is called after the request transaction is initiated, i.e. request header is sent to the service + * After this method, one among requestBytesSent, responseHeaderReceived, and attemptFailure will be always be + * invoked *

* Available context attributes: *

    *
  1. {@link Context.RequestHeaderSent#request()}
    2. + *
  2. {@link Context.RequestHeaderSent#httpRequest()}
    3. *
  3. {@link Context.RequestHeaderSent#progressSnapshot()}
    4. *
*/ @@ -129,10 +140,12 @@ default void requestHeaderSent(Context.RequestHeaderSent context) { /** * This method is called with any additional payload bytes sent; it may be called many times per request, depending on the * payload size. + * After this method, either responseHeaderReceived or attemptFailure will always be invoked *

* Available context attributes: *

    *
  1. {@link Context.RequestBytesSent#request()}
    2. + *
  2. {@link Context.RequestBytesSent#httpRequest()}
    3. *
  3. {@link Context.RequestBytesSent#progressSnapshot()}
    4. *
*/ @@ -141,11 +154,14 @@ default void requestBytesSent(Context.RequestBytesSent context) { /** * The service returns the response headers + * After this, one among responseBytesReceived, attemptFailureResponseBytesReceived and attemptFailure will always be invoked *

* Available context attributes: *

    *
  1. {@link Context.ResponseHeaderReceived#request()}
    2. + *
  2. {@link Context.ResponseHeaderReceived#httpRequest()}
    3. *
  3. {@link Context.ResponseHeaderReceived#progressSnapshot()}
    4. + *
  4. {@link Context.ResponseHeaderReceived#httpResponse()} ()}
    5. *
*/ default void responseHeaderReceived(Context.ResponseHeaderReceived context) { @@ -153,23 +169,31 @@ default void responseHeaderReceived(Context.ResponseHeaderReceived context) { /** * Additional bytes received + * After this, either executionSuccess or attemptFailure will always be invoked *

* Available context attributes: *

    *
  1. {@link Context.ResponseBytesReceived#request()}
    2. + *
  2. {@link Context.ResponseBytesReceived#httpRequest()}
    3. *
  3. {@link Context.ResponseBytesReceived#progressSnapshot()}
    4. + *
  4. {@link Context.ResponseBytesReceived#httpResponse()}
    5. *
*/ default void responseBytesReceived(Context.ResponseBytesReceived context) { } /** - * Additional bytes received + * For Expect: 100-continue embedded requests, the service returning anything other than 100 continue + * indicates a request failure. This method captures the error in the payload + * After this, either executionFailure or requestHeaderSent will always be invoked depending on + * whether the error type is retryable or not *

* Available context attributes: *

    - *
  1. {@link Context.ResponseBytesReceived#request()}
    2. - *
  2. {@link Context.ResponseBytesReceived#progressSnapshot()}
    3. + *
  3. {@link Context.AttemptFailureResponseBytesReceived#request()}
    4. + *
  4. {@link Context.AttemptFailureResponseBytesReceived#httpRequest()}
    5. + *
  5. {@link Context.AttemptFailureResponseBytesReceived#progressSnapshot()}
    6. + *
  6. {@link Context.AttemptFailureResponseBytesReceived#httpResponse()} ()}
    7. *
*/ default void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseBytesReceived context) { @@ -177,30 +201,51 @@ default void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseB /** * Successful request execution + * This marks the end of the request path. *

* Available context attributes: *

    *
  1. {@link Context.ExecutionSuccess#request()}
    2. + *
  2. {@link Context.ExecutionSuccess#httpRequest()}
    3. *
  3. {@link Context.ExecutionSuccess#progressSnapshot()}
    4. - *
  4. {@link Context.ExecutionSuccess#executionSuccess()} ()}
    5. + *
  5. {@link Context.ExecutionSuccess#httpResponse()}
    6. + *
  6. {@link Context.ExecutionSuccess#response()}
    7. *
*/ default void executionSuccess(Context.ExecutionSuccess context) { } /** - * This method is called for every failure of a request attempt. - * An ideal implementation would invoke executionFailure for a number of attemptFailures greater than a threshold + * This method is called for every failure of a request attempt + * This method is followed by either a retry attempt which would be requestHeaderSent, + * or an executionFailure if it has exceeded the maximum number of retries configured *

* Available context attributes: *

    *
  1. {@link Context.AttemptFailure#request()}
    2. + *
  2. {@link Context.AttemptFailure#httpRequest()}
    3. *
  3. {@link Context.AttemptFailure#progressSnapshot()}
    4. + *
  4. {@link Context.AttemptFailure#exception()} ()}
    5. *
*/ default void attemptFailure(Context.AttemptFailure context) { } + /** + * This method is called for every failed request execution + * This marks end of the request path with an exception being throw with the appropriate message + *

+ * Available context attributes: + *

    + *
  1. {@link Context.ExecutionFailure#request()}
    2. + *
  2. {@link Context.ExecutionFailure#httpRequest()}
    3. + *
  3. {@link Context.ExecutionFailure#progressSnapshot()}
    4. + *
  4. {@link Context.ExecutionFailure#exception()} ()}
    5. + *
+ */ + default void executionFailure(Context.ExecutionFailure context) { + } + /** * A wrapper class that groups together the different context interfaces that are exposed to {@link ProgressListener}s. *

@@ -213,12 +258,22 @@ default void attemptFailure(Context.AttemptFailure context) { *

  • {@link ResponseBytesReceived}
    • *
  • {@link ExecutionSuccess}
    • * - * Failed transfer interface hierarchy: + * Failed transfer method hierarchy: *
      *
    1. {@link RequestPrepared}
      2. *
    2. {@link AttemptFailure}
      3. *
    3. {@link ExecutionFailure}
      4. *
    + * If the request header includes an Expect: 100-Continue and the service returns a different value, the method invokation + * hierarchy is as follows : + *
      + *
    1. {@link RequestPrepared}
      2. + *
    2. {@link RequestHeaderSent}
      3. + *
    3. {@link RequestBytesSent}
      4. + *
    4. {@link ResponseHeaderReceived}
      5. + *
    5. {@link AttemptFailureResponseBytesReceived}
      6. + *
    6. {@link ExecutionFailure}
      7. + *
    * * @see ProgressListener */ @@ -242,24 +297,32 @@ private Context() { @SdkPreviewApi public interface RequestPrepared { /** - * The {@link SdkHttppRequest} that was submitted to SDK, i.e., the {@link PutObjectRequest} or - * {@link GetObjectRequest}. + * The {@link SdkRequest} that was submitted to SDK, i.e., the {@link PutObjectRequest} or + * {@link GetObjectRequest} */ - SdkHttpRequest request(); + SdkRequest request(); /** - * The immutable {@link ProgressSnapahot} for this specific update. + * The {@link SdkRequest} that was submitted to SDK, i.e., the {@link PutObjectRequest} or + * {@link GetObjectRequest} is marshalled, signed and transformed into an {@link SdkHttpRequest} + * */ - ProgressSnapshot progressSnapshot(); + SdkHttpRequest httpRequest(); + + /** + * The immutable {@link ProgressSnapshot} to track upload progress state + */ + ProgressSnapshot uploadProgressSnapshot(); } /** - * The submitted {@link SdkHttppRequest} request header was successfully sent to the service + * The submitted {@link SdkHttpRequest} request header was successfully sent to the service *

    * Available context attributes: *

      *
    1. {@link RequestHeaderSent#request()}
      2. - *
    2. {@link RequestHeaderSent#progressSnapshot()}
      3. + *
    3. {@link RequestHeaderSent#httpRequest()}
      4. + *
    4. {@link RequestHeaderSent#uploadProgressSnapshot()}
      5. *
    */ @Immutable @@ -275,7 +338,8 @@ public interface RequestHeaderSent extends RequestPrepared { * Available context attributes: *
      *
    1. {@link RequestBytesSent#request()}
      2. - *
    2. {@link RequestBytesSent#progressSnapshot()}
      3. + *
    3. {@link RequestBytesSent#httpRequest()}
      4. + *
    4. {@link RequestBytesSent#uploadProgressSnapshot()}
      5. *
    */ @Immutable @@ -291,7 +355,9 @@ public interface RequestBytesSent extends RequestHeaderSent { * Available context attributes: *
      *
    1. {@link ResponseHeaderReceived#request()}
      2. - *
    2. {@link ResponseHeaderReceived#progressSnapshot()}
      3. + *
    3. {@link ResponseHeaderReceived#httpRequest()}
      4. + *
    4. {@link ResponseHeaderReceived#uploadProgressSnapshot()}
      5. + *
    5. {@link ResponseHeaderReceived#httpResponse()}
      6. *
    */ @Immutable @@ -299,6 +365,12 @@ public interface RequestBytesSent extends RequestHeaderSent { @SdkPublicApi @SdkPreviewApi public interface ResponseHeaderReceived extends RequestBytesSent { + SdkHttpResponse httpResponse(); + + /** + * The immutable {@link ProgressSnapshot} to track download progress state + */ + ProgressSnapshot downloadProgressSnapshot(); } /** @@ -307,7 +379,10 @@ public interface ResponseHeaderReceived extends RequestBytesSent { * Available context attributes: *
      *
    1. {@link ResponseBytesReceived#request()}
      2. - *
    2. {@link ResponseBytesReceived#progressSnapshot()}
      3. + *
    3. {@link ResponseBytesReceived#httpRequest()} ()}
      4. + *
    4. {@link ResponseBytesReceived#uploadProgressSnapshot()}
      5. + *
    5. {@link ResponseBytesReceived#httpResponse()}
      6. + *
    6. {@link ResponseBytesReceived#downloadProgressSnapshot()}
      7. *
    */ @Immutable @@ -323,8 +398,11 @@ public interface ResponseBytesReceived extends ResponseHeaderReceived { * Available context attributes: *
      *
    1. {@link ExecutionSuccess#request()}
      2. - *
    2. {@link ExecutionSuccess#progressSnapshot()}
      3. - *
    3. {@link ExecutionSuccess#executionSuccess()} ()}
      4. + *
    4. {@link ExecutionSuccess#httpRequest()}
      5. + *
    5. {@link ExecutionSuccess#uploadProgressSnapshot()}
      6. + *
    6. {@link ExecutionSuccess#httpResponse()}
      7. + *
    7. {@link ExecutionSuccess#downloadProgressSnapshot()}
      8. + *
    8. {@link ExecutionSuccess#response()}
      9. *
    */ @Immutable @@ -335,17 +413,20 @@ public interface ExecutionSuccess extends ResponseBytesReceived { /** * The successful completion of a request submitted to the Sdk */ - ExecutionSuccessObjectRequest executionSuccess(); + SdkResponse response(); } /** - * For Expect: 100-continue embedded requests, the service returning anything other than 100 continue - * indicates a service error. The progress state captured indicates that no bytes are received. + * This facilitates capturing and handling an error response returned by service *

    * Available context attributes: *

      *
    1. {@link AttemptFailureResponseBytesReceived#request()}
      2. - *
    2. {@link AttemptFailureResponseBytesReceived#progressSnapshot()}
      3. + *
    3. {@link AttemptFailureResponseBytesReceived#httpRequest()}
      4. + *
    4. {@link AttemptFailureResponseBytesReceived#uploadProgressSnapshot()}
      5. + *
    5. {@link AttemptFailureResponseBytesReceived#httpResponse()} ()}
      6. + *
    6. {@link AttemptFailureResponseBytesReceived#downloadProgressSnapshot()}
      7. + *
    7. {@link AttemptFailureResponseBytesReceived#exception()}
      8. *
    */ @Immutable @@ -353,6 +434,7 @@ public interface ExecutionSuccess extends ResponseBytesReceived { @SdkPublicApi @SdkPreviewApi public interface AttemptFailureResponseBytesReceived extends ResponseHeaderReceived { + Throwable exception(); } /** @@ -361,14 +443,22 @@ public interface AttemptFailureResponseBytesReceived extends ResponseHeaderRecei * Available context attributes: *
      *
    1. {@link AttemptFailure#request()}
      2. - *
    2. {@link AttemptFailure#progressSnapshot()}
      3. + *
    3. {@link AttemptFailure#httpRequest()}
      4. + *
    4. {@link AttemptFailure#uploadProgressSnapshot()}
      5. + *
    5. {@link AttemptFailure#httpResponse()}
      6. + *
    6. {@link AttemptFailure#downloadProgressSnapshot()}
      7. + *
    7. {@link AttemptFailure#exception()}
      8. *
    */ @Immutable @ThreadSafe @SdkPublicApi @SdkPreviewApi - public interface AttemptFailure extends RequestPrepared { + public interface AttemptFailure extends ResponseBytesReceived { + /** + * The exception associated with the failed request. + */ + Throwable exception(); } /** @@ -377,7 +467,10 @@ public interface AttemptFailure extends RequestPrepared { * Available context attributes: *
      *
    1. {@link ExecutionFailure#request()}
      2. - *
    2. {@link ExecutionFailure#progressSnapshot()}
      3. + *
    3. {@link ExecutionFailure#httpRequest()}
      4. + *
    4. {@link ExecutionFailure#uploadProgressSnapshot()}
      5. + *
    5. {@link AttemptFailure#httpResponse()}
      6. + *
    6. {@link ExecutionFailure#downloadProgressSnapshot()}
      7. *
    7. {@link ExecutionFailure#exception()}
      8. *
    */ @@ -385,7 +478,7 @@ public interface AttemptFailure extends RequestPrepared { @ThreadSafe @SdkPublicApi @SdkPreviewApi - public interface ExecutionFailure extends RequestPrepared { + public interface ExecutionFailure extends ResponseBytesReceived { /** * The exception associated with the failed request. */ diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java deleted file mode 100644 index 074fbea14010..000000000000 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java +++ /dev/null @@ -1,2 +0,0 @@ -package software.amazon.awssdk.core.progress.snapshot;public class DefaultProgressSnapshot { -} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java index e1692fa27752..71439266246f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java @@ -28,6 +28,5 @@ public interface ListenerProgress { * Takes a snapshot of the request execution progress * represented by an immutable {@link ProgressSnapshot}. */ - ProgressSnapshot progressSnapshot(); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java index 2a037519a8be..af314248b8b0 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java @@ -36,32 +36,33 @@ public interface ProgressSnapshot { long transferredBytes(); /** - * Time at which transaction started + * Time at which the HTTP Request header is sent */ Optional startTime(); /** - * Elapsed time since the start of the transaction + * Elapsed time since the HTTP request header was sent to the service */ Optional elapsedTime(); /** * If transaction size is known, estimate time remaining for transaction completion + * This is a predictive calculation based on the rate of transfer + *

    + * Double rateOfTimeUnitsPerByte = elapsedTime() / transferredBytes(); + * Double estimatedTimeRemaining = rateOfTimeUnitsPerByte * (totalBytes() - transferredBytes()); + *

    */ Optional estimatedTimeRemaining(); - /** - * The SDK response, or {@link Optional#empty()} if unknown. - */ - Optional sdkResponse(); - /** * Rate of transfer */ - double averageBytesPer(TimeUnit timeUnit); + OptionalDouble averageBytesPer(TimeUnit timeUnit); /** - * The total size of the transfer, in bytes, or {@link Optional#empty()} if unknown. + * The total size of the transfer, in bytes, or {@link Optional#empty()} if total payload being transacted is unknown + * and not set. This could happen for streaming operations. */ OptionalLong totalBytes(); From 07f2bfb07b089bc48349070a9422dffdee503f00 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Thu, 7 Dec 2023 20:38:38 -0800 Subject: [PATCH 11/50] Address PR comments on Progress Listener --- .../progress/listener/ProgressListener.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index a78fd8f01672..98f6159b6901 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -50,7 +50,7 @@ * after it is marshalled and signed, before it is sent to the service *
      Available context attributes: *
    • {@link Context.RequestPrepared#request()}
      • - *
    • {@link Context.RequestPrepared#progressSnapshot()}
      • + *
    • {@link Context.RequestPrepared#uploadProgressSnapshot()}
      • *
    *
  • {@link #requestBytesSent(Context.RequestBytesSent)} - Additional bytes have been sent. This * method may be called many times per request, depending on the request payload size and I/O buffer sizes. @@ -116,7 +116,7 @@ public interface ProgressListener { *
      *
    1. {@link Context.RequestPrepared#request()}
      2. *
    2. {@link Context.RequestPrepared#httpRequest()}
      3. - *
    3. {@link Context.RequestPrepared#progressSnapshot()}
      4. + *
    4. {@link Context.RequestPrepared#uploadProgressSnapshot()}
      5. *
    */ default void requestPrepared(Context.RequestPrepared context) { @@ -131,7 +131,7 @@ default void requestPrepared(Context.RequestPrepared context) { *
      *
    1. {@link Context.RequestHeaderSent#request()}
      2. *
    2. {@link Context.RequestHeaderSent#httpRequest()}
      3. - *
    3. {@link Context.RequestHeaderSent#progressSnapshot()}
      4. + *
    4. {@link Context.RequestHeaderSent#uploadProgressSnapshot()}
      5. *
    */ default void requestHeaderSent(Context.RequestHeaderSent context) { @@ -146,7 +146,7 @@ default void requestHeaderSent(Context.RequestHeaderSent context) { *
      *
    1. {@link Context.RequestBytesSent#request()}
      2. *
    2. {@link Context.RequestBytesSent#httpRequest()}
      3. - *
    3. {@link Context.RequestBytesSent#progressSnapshot()}
      4. + *
    4. {@link Context.RequestBytesSent#uploadProgressSnapshot()}
      5. *
    */ default void requestBytesSent(Context.RequestBytesSent context) { @@ -160,8 +160,9 @@ default void requestBytesSent(Context.RequestBytesSent context) { *
      *
    1. {@link Context.ResponseHeaderReceived#request()}
      2. *
    2. {@link Context.ResponseHeaderReceived#httpRequest()}
      3. - *
    3. {@link Context.ResponseHeaderReceived#progressSnapshot()}
      4. + *
    4. {@link Context.ResponseHeaderReceived#uploadProgressSnapshot()}
      5. *
    5. {@link Context.ResponseHeaderReceived#httpResponse()} ()}
      6. + *
    6. {@link Context.ResponseHeaderReceived#downloadProgressSnapshot()}
      7. *
    */ default void responseHeaderReceived(Context.ResponseHeaderReceived context) { @@ -175,8 +176,9 @@ default void responseHeaderReceived(Context.ResponseHeaderReceived context) { *
      *
    1. {@link Context.ResponseBytesReceived#request()}
      2. *
    2. {@link Context.ResponseBytesReceived#httpRequest()}
      3. - *
    3. {@link Context.ResponseBytesReceived#progressSnapshot()}
      4. + *
    4. {@link Context.ResponseBytesReceived#uploadProgressSnapshot()}
      5. *
    5. {@link Context.ResponseBytesReceived#httpResponse()}
      6. + *
    6. {@link Context.ResponseBytesReceived#downloadProgressSnapshot()}
      7. *
    */ default void responseBytesReceived(Context.ResponseBytesReceived context) { @@ -192,8 +194,10 @@ default void responseBytesReceived(Context.ResponseBytesReceived context) { *
      *
    1. {@link Context.AttemptFailureResponseBytesReceived#request()}
      2. *
    2. {@link Context.AttemptFailureResponseBytesReceived#httpRequest()}
      3. - *
    3. {@link Context.AttemptFailureResponseBytesReceived#progressSnapshot()}
      4. + *
    4. {@link Context.AttemptFailureResponseBytesReceived#uploadProgressSnapshot()}
      5. *
    5. {@link Context.AttemptFailureResponseBytesReceived#httpResponse()} ()}
      6. + *
    6. {@link Context.AttemptFailureResponseBytesReceived#downloadProgressSnapshot()}
      7. + *
    7. {@link Context.AttemptFailureResponseBytesReceived#exception()}
      8. *
    */ default void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseBytesReceived context) { @@ -207,8 +211,9 @@ default void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseB *
      *
    1. {@link Context.ExecutionSuccess#request()}
      2. *
    2. {@link Context.ExecutionSuccess#httpRequest()}
      3. - *
    3. {@link Context.ExecutionSuccess#progressSnapshot()}
      4. + *
    4. {@link Context.ExecutionSuccess#uploadProgressSnapshot()}
      5. *
    5. {@link Context.ExecutionSuccess#httpResponse()}
      6. + *
    6. {@link Context.ExecutionSuccess#downloadProgressSnapshot()}
      7. *
    7. {@link Context.ExecutionSuccess#response()}
      8. *
    */ @@ -224,7 +229,7 @@ default void executionSuccess(Context.ExecutionSuccess context) { *
      *
    1. {@link Context.AttemptFailure#request()}
      2. *
    2. {@link Context.AttemptFailure#httpRequest()}
      3. - *
    3. {@link Context.AttemptFailure#progressSnapshot()}
      4. + *
    4. {@link Context.AttemptFailure#uploadProgressSnapshot()}
      5. *
    5. {@link Context.AttemptFailure#exception()} ()}
      6. *
    */ @@ -239,7 +244,7 @@ default void attemptFailure(Context.AttemptFailure context) { *
      *
    1. {@link Context.ExecutionFailure#request()}
      2. *
    2. {@link Context.ExecutionFailure#httpRequest()}
      3. - *
    3. {@link Context.ExecutionFailure#progressSnapshot()}
      4. + *
    4. {@link Context.ExecutionFailure#uploadProgressSnapshot()}
      5. *
    5. {@link Context.ExecutionFailure#exception()} ()}
      6. *
    */ @@ -445,8 +450,6 @@ public interface AttemptFailureResponseBytesReceived extends ResponseHeaderRecei *
  • {@link AttemptFailure#request()}
    • *
  • {@link AttemptFailure#httpRequest()}
    • *
  • {@link AttemptFailure#uploadProgressSnapshot()}
    • - *
  • {@link AttemptFailure#httpResponse()}
    • - *
  • {@link AttemptFailure#downloadProgressSnapshot()}
    • *
  • {@link AttemptFailure#exception()}
    • * */ @@ -454,7 +457,7 @@ public interface AttemptFailureResponseBytesReceived extends ResponseHeaderRecei @ThreadSafe @SdkPublicApi @SdkPreviewApi - public interface AttemptFailure extends ResponseBytesReceived { + public interface AttemptFailure extends RequestPrepared { /** * The exception associated with the failed request. */ @@ -469,8 +472,6 @@ public interface AttemptFailure extends ResponseBytesReceived { *
  • {@link ExecutionFailure#request()}
    • *
  • {@link ExecutionFailure#httpRequest()}
    • *
  • {@link ExecutionFailure#uploadProgressSnapshot()}
    • - *
  • {@link AttemptFailure#httpResponse()}
    • - *
  • {@link ExecutionFailure#downloadProgressSnapshot()}
    • *
  • {@link ExecutionFailure#exception()}
    • * */ @@ -478,7 +479,7 @@ public interface AttemptFailure extends ResponseBytesReceived { @ThreadSafe @SdkPublicApi @SdkPreviewApi - public interface ExecutionFailure extends ResponseBytesReceived { + public interface ExecutionFailure extends RequestPrepared { /** * The exception associated with the failed request. */ From d2a325322a4e8685aee1a2610871354c3caf38d4 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 11 Dec 2023 10:08:39 -0800 Subject: [PATCH 12/50] Fix checkstyle issues --- .../amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java index af314248b8b0..829ce35b0dc0 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java @@ -24,7 +24,6 @@ import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; -import software.amazon.awssdk.core.SdkResponse; @Immutable @ThreadSafe From eaf94708e401de421c0915f92aaa70cae1e13407 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 11 Dec 2023 10:47:40 -0800 Subject: [PATCH 13/50] Fix checkstyle issues --- .../awssdk/core/progress/listener/ProgressListener.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index 98f6159b6901..ee0cff4d249e 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -108,8 +108,8 @@ public interface ProgressListener { /** - * This method is called right after a {@link SdkRequest} is marshalled, signed, transformed into an {@link SdkHttpRequest} and ready to - * be sent to the service + * This method is called right after a {@link SdkRequest} is marshalled, signed, transformed into an + * {@link SdkHttpRequest} and ready to be sent to the service * After this method has returned, either requestHeaderSent or executionFailure will always be invoked *

    * Available context attributes: From 276fb70834a56be09e5d0805b55545c85843f369 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Thu, 14 Dec 2023 02:41:39 -0800 Subject: [PATCH 14/50] Implement Default Progress Snapshot --- core/sdk-core/pom.xml | 8 + .../listener/DefaultSdkRequestProgress.java | 56 +++++ .../SdkRequestProgress.java} | 5 +- .../snapshot/DefaultProgressSnapshot.java | 174 +++++++++++++++ .../progress/DefaultProgressSnapshotTest.java | 202 ++++++++++++++++++ 5 files changed, 443 insertions(+), 2 deletions(-) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java rename core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/{snapshot/ListenerProgress.java => listener/SdkRequestProgress.java} (85%) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java diff --git a/core/sdk-core/pom.xml b/core/sdk-core/pom.xml index f19120fde62d..d15be86a3608 100644 --- a/core/sdk-core/pom.xml +++ b/core/sdk-core/pom.xml @@ -284,6 +284,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 11 + 11 + + diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java new file mode 100644 index 000000000000..6c8a602afcc6 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java @@ -0,0 +1,56 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress.listener; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.Mutable; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.progress.snapshot.DefaultProgressSnapshot; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; + +/** + * An SDK-internal implementation of {@link SdkRequestProgress}. This implementation acts as a thin wrapper around {@link + * AtomicReference}, where calls to get the latest {@link #progressSnapshot()} simply return the latest reference, while {@link + * SdkRequestProgressUpdater} is responsible for continuously updating the latest reference. + * + * @see SdkRequestProgress + */ +@Mutable +@ThreadSafe +@SdkInternalApi +public class DefaultSdkRequestProgress implements SdkRequestProgress { + + private final AtomicReference snapshot; + + public DefaultSdkRequestProgress(ProgressSnapshot snapshot) { + this.snapshot = new AtomicReference<>(snapshot); + } + + /** + * Atomically convert the current snapshot reference to its {@link Builder}, perform updates using the provided {@link + * Consumer}, and save the result as the latest snapshot. + */ + public ProgressSnapshot updateAndGet(Consumer updater) { + return this.snapshot.updateAndGet(s -> ((DefaultProgressSnapshot) s).copy(updater)); + } + + @Override + public ProgressSnapshot progressSnapshot() { + return this.snapshot.get(); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java similarity index 85% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java index 71439266246f..837b30535868 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java @@ -13,16 +13,17 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.progress.snapshot; +package software.amazon.awssdk.core.progress.listener; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; @Immutable @ThreadSafe @SdkPublicApi -public interface ListenerProgress { +public interface SdkRequestProgress { /** * Takes a snapshot of the request execution progress diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java new file mode 100644 index 000000000000..03de4239f204 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java @@ -0,0 +1,174 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress.snapshot; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * An SDK-internal implementation of {@link ProgressSnapshot}. + */ +@SdkInternalApi +public class DefaultProgressSnapshot + implements ToCopyableBuilder, + ProgressSnapshot { + + private final long transferredBytes; + private final Long totalBytes; + private final Optional startTime; + + public DefaultProgressSnapshot(Builder builder) { + if (builder.totalBytes != null) { + Validate.isNotNegative(builder.totalBytes, "totalBytes"); + Validate.isTrue(builder.transferredBytes <= builder.totalBytes, + "transferredBytes (%s) must not be greater than totalBytes (%s)", + builder.transferredBytes, builder.totalBytes); + } + Validate.paramNotNull(builder.transferredBytes, "byteTransferred"); + this.transferredBytes = Validate.isNotNegative(builder.transferredBytes, "transferredBytes"); + this.totalBytes = builder.totalBytes; + + if (builder.startTime.isPresent()) { + Instant currentTime = Instant.now(); + Validate.isTrue(currentTime.isAfter(builder.startTime.get()), + "currentTime (%s) must not be before startTime (%s)", + currentTime, builder.startTime.get()); + } + + this.startTime = builder.startTime; + } + + @Override + public long transferredBytes() { + return this.transferredBytes; + } + + @Override + public Optional startTime() { + return this.startTime; + } + + @Override + public Optional elapsedTime() { + return this.startTime.isPresent() ? Optional.of(Duration.between(startTime.get(), Instant.now())) : Optional.empty(); + } + + @Override + public Optional estimatedTimeRemaining() { + if (!elapsedTime().isPresent() || !remainingBytes().isPresent()) { + return Optional.empty(); + } + + long remainingTime = remainingBytes().getAsLong() * elapsedTime().get().toMillis() / transferredBytes; + return Optional.of(Duration.ofMillis(remainingTime)); + + } + + @Override + public OptionalDouble averageBytesPer(TimeUnit timeUnit) { + if (!this.elapsedTime().isPresent()) { + return OptionalDouble.empty(); + } + + return this.elapsedTime().get().equals(Duration.ZERO) ? OptionalDouble.of(1.0) : + OptionalDouble.of((double) this.transferredBytes / timeUnit.convert(elapsedTime().get().toMillis(), timeUnit)); + } + + @Override + public OptionalLong totalBytes() { + return totalBytes == null ? OptionalLong.empty() : OptionalLong.of(totalBytes); + } + + @Override + public OptionalDouble ratioTransferred() { + if (totalBytes == null) { + return OptionalDouble.empty(); + } + return totalBytes == 0 ? OptionalDouble.of(1.0) : OptionalDouble.of(transferredBytes / totalBytes.doubleValue()); + } + + @Override + public OptionalLong remainingBytes() { + if (totalBytes == null) { + return OptionalLong.empty(); + } + return totalBytes == 0 ? OptionalLong.of(0) : OptionalLong.of(totalBytes - transferredBytes); + } + + @Override + public DefaultProgressSnapshot.Builder toBuilder() { + return new Builder(this); + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder implements CopyableBuilder { + private long transferredBytes; + private Long totalBytes; + private Optional startTime = Optional.empty(); + + private Builder() { + } + + private Builder(DefaultProgressSnapshot progressSnapshot) { + this.transferredBytes = progressSnapshot.transferredBytes; + this.totalBytes = progressSnapshot.totalBytes; + this.startTime = progressSnapshot.startTime; + } + + public Builder transferredBytes(Long transferredBytes) { + this.transferredBytes = transferredBytes; + return this; + } + + public Long getTransferredBytes() { + return this.transferredBytes; + } + + public Builder totalBytes(Long totalBytes) { + this.totalBytes = totalBytes; + return this; + } + + public Long getTotalBytes() { + return this.totalBytes; + } + + public Builder startTime(Instant startTime) { + this.startTime = Optional.of(startTime); + return this; + } + + public Optional startTime() { + return this.startTime; + } + + @Override + public DefaultProgressSnapshot build() { + return new DefaultProgressSnapshot(this); + } + } +} \ No newline at end of file diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java new file mode 100644 index 000000000000..2cf09d75cec5 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java @@ -0,0 +1,202 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.junit.Assert.assertEquals; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.progress.snapshot.DefaultProgressSnapshot; + +public class DefaultProgressSnapshotTest { + @Test + public void bytesTransferred_greaterThan_totalBytes_shouldThrow() { + DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(1L); + assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("transferredBytes (2) must not be greater than totalBytes (1)"); + } + + @Test + public void transferredBytes_negative_shouldThrow() { + DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() + .transferredBytes(-2L); + + assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("transferredBytes must not be negative"); + } + + @Test + public void transferredBytes_null_isZero() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .build(); + + Assertions.assertEquals(0, snapshot.transferredBytes()); + } + + @Test + public void totalBytes_negative_shouldThrow() { + DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(-2L); + + assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("totalBytes must not be negative"); + } + + @Test + public void totalBytes_empty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .build(); + + assertThat(snapshot.totalBytes()).isNotPresent(); + } + + @Test + public void totalBytes() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(5L) + .build(); + + Assertions.assertEquals(5, snapshot.totalBytes().getAsLong()); + } + + @Test + public void startTime_after_currentTime_shouldThrow() { + Instant timeAfterFiveSeconds = Instant.now().plus(5, ChronoUnit.SECONDS); + DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() + .transferredBytes(0L) + .startTime(timeAfterFiveSeconds); + + assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("currentTime") + .hasMessageEndingWith(" must not be before startTime (" + timeAfterFiveSeconds + ")"); + } + + @Test + public void ratioTransferred_withoutTotalBytes_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .build(); + assertThat(snapshot.ratioTransferred()).isNotPresent(); + } + + @Test + public void ratioTransferred() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .totalBytes(5L) + .build(); + assertEquals(0.2, snapshot.ratioTransferred().getAsDouble(), 0.0); + } + + @Test + public void remainingBytes_withoutTotalBytes_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .build(); + assertThat(snapshot.remainingBytes()).isNotPresent(); + } + + @Test + public void remainingBytes() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .totalBytes(5L) + .build(); + Assertions.assertEquals(4.0, snapshot.remainingBytes().getAsLong(), 0.0); + } + + @Test + public void elapsedTime_withoutStartTime_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .build(); + assertThat(snapshot.elapsedTime()).isNotPresent(); + } + + @Test + public void elapsedTime() { + + Instant startTime = Instant.now().minusMillis(100); + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .startTime(startTime) + .build(); + Duration expectedDuration = Duration.between(startTime, Instant.now()); + + Assertions.assertEquals(snapshot.elapsedTime().get().toMillis(), expectedDuration.toMillis(), 0.1); + } + + @Test + public void averageBytesPer_withoutStartTime_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .totalBytes(5L) + .build(); + assertThat(snapshot.averageBytesPer(TimeUnit.MILLISECONDS)).isNotPresent(); + } + + @Test + public void averageBytesPer() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(100L) + .startTime(Instant.now().minusMillis(100)) + .build(); + Assertions.assertEquals(1.0, snapshot.averageBytesPer(TimeUnit.MILLISECONDS).getAsDouble(), 0.2); + } + + @Test + public void estimatedTimeRemaining_withoutStartTime_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .totalBytes(5L) + .build(); + assertThat(snapshot.estimatedTimeRemaining()).isNotPresent(); + } + + @Test + public void estimatedTimeRemaining_withoutTotalBytes_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .startTime(Instant.now().minusMillis(5)) + .build(); + assertThat(snapshot.estimatedTimeRemaining()).isNotPresent(); + } + + @Test + public void estimatedTimeRemaining() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(100L) + .startTime(Instant.now().minusSeconds(1)) + .totalBytes(500L) + .build(); + Assertions.assertEquals(4000.0, snapshot.estimatedTimeRemaining().get().toMillis(), 10.0); + } +} From 7b24093c0bd828730f94e04ec1c073784ddd53ec Mon Sep 17 00:00:00 2001 From: Krishnan Date: Thu, 14 Dec 2023 02:45:18 -0800 Subject: [PATCH 15/50] Revert "Implement Default Progress Snapshot" This reverts commit 276fb70834a56be09e5d0805b55545c85843f369. --- core/sdk-core/pom.xml | 8 - .../listener/DefaultSdkRequestProgress.java | 56 ----- .../snapshot/DefaultProgressSnapshot.java | 174 --------------- .../ListenerProgress.java} | 5 +- .../progress/DefaultProgressSnapshotTest.java | 202 ------------------ 5 files changed, 2 insertions(+), 443 deletions(-) delete mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java delete mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java rename core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/{listener/SdkRequestProgress.java => snapshot/ListenerProgress.java} (85%) delete mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java diff --git a/core/sdk-core/pom.xml b/core/sdk-core/pom.xml index d15be86a3608..f19120fde62d 100644 --- a/core/sdk-core/pom.xml +++ b/core/sdk-core/pom.xml @@ -284,14 +284,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - - 11 - 11 - - diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java deleted file mode 100644 index 6c8a602afcc6..000000000000 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.progress.listener; - -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import software.amazon.awssdk.annotations.Mutable; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.annotations.ThreadSafe; -import software.amazon.awssdk.core.progress.snapshot.DefaultProgressSnapshot; -import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; - -/** - * An SDK-internal implementation of {@link SdkRequestProgress}. This implementation acts as a thin wrapper around {@link - * AtomicReference}, where calls to get the latest {@link #progressSnapshot()} simply return the latest reference, while {@link - * SdkRequestProgressUpdater} is responsible for continuously updating the latest reference. - * - * @see SdkRequestProgress - */ -@Mutable -@ThreadSafe -@SdkInternalApi -public class DefaultSdkRequestProgress implements SdkRequestProgress { - - private final AtomicReference snapshot; - - public DefaultSdkRequestProgress(ProgressSnapshot snapshot) { - this.snapshot = new AtomicReference<>(snapshot); - } - - /** - * Atomically convert the current snapshot reference to its {@link Builder}, perform updates using the provided {@link - * Consumer}, and save the result as the latest snapshot. - */ - public ProgressSnapshot updateAndGet(Consumer updater) { - return this.snapshot.updateAndGet(s -> ((DefaultProgressSnapshot) s).copy(updater)); - } - - @Override - public ProgressSnapshot progressSnapshot() { - return this.snapshot.get(); - } -} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java deleted file mode 100644 index 03de4239f204..000000000000 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.progress.snapshot; - -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.OptionalLong; -import java.util.concurrent.TimeUnit; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.utils.Validate; -import software.amazon.awssdk.utils.builder.CopyableBuilder; -import software.amazon.awssdk.utils.builder.ToCopyableBuilder; - -/** - * An SDK-internal implementation of {@link ProgressSnapshot}. - */ -@SdkInternalApi -public class DefaultProgressSnapshot - implements ToCopyableBuilder, - ProgressSnapshot { - - private final long transferredBytes; - private final Long totalBytes; - private final Optional startTime; - - public DefaultProgressSnapshot(Builder builder) { - if (builder.totalBytes != null) { - Validate.isNotNegative(builder.totalBytes, "totalBytes"); - Validate.isTrue(builder.transferredBytes <= builder.totalBytes, - "transferredBytes (%s) must not be greater than totalBytes (%s)", - builder.transferredBytes, builder.totalBytes); - } - Validate.paramNotNull(builder.transferredBytes, "byteTransferred"); - this.transferredBytes = Validate.isNotNegative(builder.transferredBytes, "transferredBytes"); - this.totalBytes = builder.totalBytes; - - if (builder.startTime.isPresent()) { - Instant currentTime = Instant.now(); - Validate.isTrue(currentTime.isAfter(builder.startTime.get()), - "currentTime (%s) must not be before startTime (%s)", - currentTime, builder.startTime.get()); - } - - this.startTime = builder.startTime; - } - - @Override - public long transferredBytes() { - return this.transferredBytes; - } - - @Override - public Optional startTime() { - return this.startTime; - } - - @Override - public Optional elapsedTime() { - return this.startTime.isPresent() ? Optional.of(Duration.between(startTime.get(), Instant.now())) : Optional.empty(); - } - - @Override - public Optional estimatedTimeRemaining() { - if (!elapsedTime().isPresent() || !remainingBytes().isPresent()) { - return Optional.empty(); - } - - long remainingTime = remainingBytes().getAsLong() * elapsedTime().get().toMillis() / transferredBytes; - return Optional.of(Duration.ofMillis(remainingTime)); - - } - - @Override - public OptionalDouble averageBytesPer(TimeUnit timeUnit) { - if (!this.elapsedTime().isPresent()) { - return OptionalDouble.empty(); - } - - return this.elapsedTime().get().equals(Duration.ZERO) ? OptionalDouble.of(1.0) : - OptionalDouble.of((double) this.transferredBytes / timeUnit.convert(elapsedTime().get().toMillis(), timeUnit)); - } - - @Override - public OptionalLong totalBytes() { - return totalBytes == null ? OptionalLong.empty() : OptionalLong.of(totalBytes); - } - - @Override - public OptionalDouble ratioTransferred() { - if (totalBytes == null) { - return OptionalDouble.empty(); - } - return totalBytes == 0 ? OptionalDouble.of(1.0) : OptionalDouble.of(transferredBytes / totalBytes.doubleValue()); - } - - @Override - public OptionalLong remainingBytes() { - if (totalBytes == null) { - return OptionalLong.empty(); - } - return totalBytes == 0 ? OptionalLong.of(0) : OptionalLong.of(totalBytes - transferredBytes); - } - - @Override - public DefaultProgressSnapshot.Builder toBuilder() { - return new Builder(this); - } - - public static Builder builder() { - return new Builder(); - } - - public static final class Builder implements CopyableBuilder { - private long transferredBytes; - private Long totalBytes; - private Optional startTime = Optional.empty(); - - private Builder() { - } - - private Builder(DefaultProgressSnapshot progressSnapshot) { - this.transferredBytes = progressSnapshot.transferredBytes; - this.totalBytes = progressSnapshot.totalBytes; - this.startTime = progressSnapshot.startTime; - } - - public Builder transferredBytes(Long transferredBytes) { - this.transferredBytes = transferredBytes; - return this; - } - - public Long getTransferredBytes() { - return this.transferredBytes; - } - - public Builder totalBytes(Long totalBytes) { - this.totalBytes = totalBytes; - return this; - } - - public Long getTotalBytes() { - return this.totalBytes; - } - - public Builder startTime(Instant startTime) { - this.startTime = Optional.of(startTime); - return this; - } - - public Optional startTime() { - return this.startTime; - } - - @Override - public DefaultProgressSnapshot build() { - return new DefaultProgressSnapshot(this); - } - } -} \ No newline at end of file diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java similarity index 85% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java index 837b30535868..71439266246f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java @@ -13,17 +13,16 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.progress.listener; +package software.amazon.awssdk.core.progress.snapshot; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; -import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; @Immutable @ThreadSafe @SdkPublicApi -public interface SdkRequestProgress { +public interface ListenerProgress { /** * Takes a snapshot of the request execution progress diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java deleted file mode 100644 index 2cf09d75cec5..000000000000 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.progress; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; -import static org.junit.Assert.assertEquals; - -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.concurrent.TimeUnit; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.core.progress.snapshot.DefaultProgressSnapshot; - -public class DefaultProgressSnapshotTest { - @Test - public void bytesTransferred_greaterThan_totalBytes_shouldThrow() { - DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() - .transferredBytes(2L) - .totalBytes(1L); - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("transferredBytes (2) must not be greater than totalBytes (1)"); - } - - @Test - public void transferredBytes_negative_shouldThrow() { - DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() - .transferredBytes(-2L); - - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("transferredBytes must not be negative"); - } - - @Test - public void transferredBytes_null_isZero() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .build(); - - Assertions.assertEquals(0, snapshot.transferredBytes()); - } - - @Test - public void totalBytes_negative_shouldThrow() { - DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() - .transferredBytes(2L) - .totalBytes(-2L); - - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("totalBytes must not be negative"); - } - - @Test - public void totalBytes_empty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(2L) - .build(); - - assertThat(snapshot.totalBytes()).isNotPresent(); - } - - @Test - public void totalBytes() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(2L) - .totalBytes(5L) - .build(); - - Assertions.assertEquals(5, snapshot.totalBytes().getAsLong()); - } - - @Test - public void startTime_after_currentTime_shouldThrow() { - Instant timeAfterFiveSeconds = Instant.now().plus(5, ChronoUnit.SECONDS); - DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() - .transferredBytes(0L) - .startTime(timeAfterFiveSeconds); - - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageStartingWith("currentTime") - .hasMessageEndingWith(" must not be before startTime (" + timeAfterFiveSeconds + ")"); - } - - @Test - public void ratioTransferred_withoutTotalBytes_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .build(); - assertThat(snapshot.ratioTransferred()).isNotPresent(); - } - - @Test - public void ratioTransferred() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .totalBytes(5L) - .build(); - assertEquals(0.2, snapshot.ratioTransferred().getAsDouble(), 0.0); - } - - @Test - public void remainingBytes_withoutTotalBytes_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .build(); - assertThat(snapshot.remainingBytes()).isNotPresent(); - } - - @Test - public void remainingBytes() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .totalBytes(5L) - .build(); - Assertions.assertEquals(4.0, snapshot.remainingBytes().getAsLong(), 0.0); - } - - @Test - public void elapsedTime_withoutStartTime_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .build(); - assertThat(snapshot.elapsedTime()).isNotPresent(); - } - - @Test - public void elapsedTime() { - - Instant startTime = Instant.now().minusMillis(100); - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .startTime(startTime) - .build(); - Duration expectedDuration = Duration.between(startTime, Instant.now()); - - Assertions.assertEquals(snapshot.elapsedTime().get().toMillis(), expectedDuration.toMillis(), 0.1); - } - - @Test - public void averageBytesPer_withoutStartTime_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .totalBytes(5L) - .build(); - assertThat(snapshot.averageBytesPer(TimeUnit.MILLISECONDS)).isNotPresent(); - } - - @Test - public void averageBytesPer() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(100L) - .startTime(Instant.now().minusMillis(100)) - .build(); - Assertions.assertEquals(1.0, snapshot.averageBytesPer(TimeUnit.MILLISECONDS).getAsDouble(), 0.2); - } - - @Test - public void estimatedTimeRemaining_withoutStartTime_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .totalBytes(5L) - .build(); - assertThat(snapshot.estimatedTimeRemaining()).isNotPresent(); - } - - @Test - public void estimatedTimeRemaining_withoutTotalBytes_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .startTime(Instant.now().minusMillis(5)) - .build(); - assertThat(snapshot.estimatedTimeRemaining()).isNotPresent(); - } - - @Test - public void estimatedTimeRemaining() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(100L) - .startTime(Instant.now().minusSeconds(1)) - .totalBytes(500L) - .build(); - Assertions.assertEquals(4000.0, snapshot.estimatedTimeRemaining().get().toMillis(), 10.0); - } -} From a943002fde07c59fdc227030168c77deae9d7e98 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Thu, 14 Dec 2023 02:46:36 -0800 Subject: [PATCH 16/50] Revert "Revert "Implement Default Progress Snapshot"" This reverts commit 7b24093c0bd828730f94e04ec1c073784ddd53ec. --- core/sdk-core/pom.xml | 8 + .../listener/DefaultSdkRequestProgress.java | 56 +++++ .../SdkRequestProgress.java} | 5 +- .../snapshot/DefaultProgressSnapshot.java | 174 +++++++++++++++ .../progress/DefaultProgressSnapshotTest.java | 202 ++++++++++++++++++ 5 files changed, 443 insertions(+), 2 deletions(-) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java rename core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/{snapshot/ListenerProgress.java => listener/SdkRequestProgress.java} (85%) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java diff --git a/core/sdk-core/pom.xml b/core/sdk-core/pom.xml index f19120fde62d..d15be86a3608 100644 --- a/core/sdk-core/pom.xml +++ b/core/sdk-core/pom.xml @@ -284,6 +284,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 11 + 11 + + diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java new file mode 100644 index 000000000000..6c8a602afcc6 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java @@ -0,0 +1,56 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress.listener; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.Mutable; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.progress.snapshot.DefaultProgressSnapshot; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; + +/** + * An SDK-internal implementation of {@link SdkRequestProgress}. This implementation acts as a thin wrapper around {@link + * AtomicReference}, where calls to get the latest {@link #progressSnapshot()} simply return the latest reference, while {@link + * SdkRequestProgressUpdater} is responsible for continuously updating the latest reference. + * + * @see SdkRequestProgress + */ +@Mutable +@ThreadSafe +@SdkInternalApi +public class DefaultSdkRequestProgress implements SdkRequestProgress { + + private final AtomicReference snapshot; + + public DefaultSdkRequestProgress(ProgressSnapshot snapshot) { + this.snapshot = new AtomicReference<>(snapshot); + } + + /** + * Atomically convert the current snapshot reference to its {@link Builder}, perform updates using the provided {@link + * Consumer}, and save the result as the latest snapshot. + */ + public ProgressSnapshot updateAndGet(Consumer updater) { + return this.snapshot.updateAndGet(s -> ((DefaultProgressSnapshot) s).copy(updater)); + } + + @Override + public ProgressSnapshot progressSnapshot() { + return this.snapshot.get(); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java similarity index 85% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java index 71439266246f..837b30535868 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ListenerProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java @@ -13,16 +13,17 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.progress.snapshot; +package software.amazon.awssdk.core.progress.listener; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; @Immutable @ThreadSafe @SdkPublicApi -public interface ListenerProgress { +public interface SdkRequestProgress { /** * Takes a snapshot of the request execution progress diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java new file mode 100644 index 000000000000..03de4239f204 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java @@ -0,0 +1,174 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress.snapshot; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * An SDK-internal implementation of {@link ProgressSnapshot}. + */ +@SdkInternalApi +public class DefaultProgressSnapshot + implements ToCopyableBuilder, + ProgressSnapshot { + + private final long transferredBytes; + private final Long totalBytes; + private final Optional startTime; + + public DefaultProgressSnapshot(Builder builder) { + if (builder.totalBytes != null) { + Validate.isNotNegative(builder.totalBytes, "totalBytes"); + Validate.isTrue(builder.transferredBytes <= builder.totalBytes, + "transferredBytes (%s) must not be greater than totalBytes (%s)", + builder.transferredBytes, builder.totalBytes); + } + Validate.paramNotNull(builder.transferredBytes, "byteTransferred"); + this.transferredBytes = Validate.isNotNegative(builder.transferredBytes, "transferredBytes"); + this.totalBytes = builder.totalBytes; + + if (builder.startTime.isPresent()) { + Instant currentTime = Instant.now(); + Validate.isTrue(currentTime.isAfter(builder.startTime.get()), + "currentTime (%s) must not be before startTime (%s)", + currentTime, builder.startTime.get()); + } + + this.startTime = builder.startTime; + } + + @Override + public long transferredBytes() { + return this.transferredBytes; + } + + @Override + public Optional startTime() { + return this.startTime; + } + + @Override + public Optional elapsedTime() { + return this.startTime.isPresent() ? Optional.of(Duration.between(startTime.get(), Instant.now())) : Optional.empty(); + } + + @Override + public Optional estimatedTimeRemaining() { + if (!elapsedTime().isPresent() || !remainingBytes().isPresent()) { + return Optional.empty(); + } + + long remainingTime = remainingBytes().getAsLong() * elapsedTime().get().toMillis() / transferredBytes; + return Optional.of(Duration.ofMillis(remainingTime)); + + } + + @Override + public OptionalDouble averageBytesPer(TimeUnit timeUnit) { + if (!this.elapsedTime().isPresent()) { + return OptionalDouble.empty(); + } + + return this.elapsedTime().get().equals(Duration.ZERO) ? OptionalDouble.of(1.0) : + OptionalDouble.of((double) this.transferredBytes / timeUnit.convert(elapsedTime().get().toMillis(), timeUnit)); + } + + @Override + public OptionalLong totalBytes() { + return totalBytes == null ? OptionalLong.empty() : OptionalLong.of(totalBytes); + } + + @Override + public OptionalDouble ratioTransferred() { + if (totalBytes == null) { + return OptionalDouble.empty(); + } + return totalBytes == 0 ? OptionalDouble.of(1.0) : OptionalDouble.of(transferredBytes / totalBytes.doubleValue()); + } + + @Override + public OptionalLong remainingBytes() { + if (totalBytes == null) { + return OptionalLong.empty(); + } + return totalBytes == 0 ? OptionalLong.of(0) : OptionalLong.of(totalBytes - transferredBytes); + } + + @Override + public DefaultProgressSnapshot.Builder toBuilder() { + return new Builder(this); + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder implements CopyableBuilder { + private long transferredBytes; + private Long totalBytes; + private Optional startTime = Optional.empty(); + + private Builder() { + } + + private Builder(DefaultProgressSnapshot progressSnapshot) { + this.transferredBytes = progressSnapshot.transferredBytes; + this.totalBytes = progressSnapshot.totalBytes; + this.startTime = progressSnapshot.startTime; + } + + public Builder transferredBytes(Long transferredBytes) { + this.transferredBytes = transferredBytes; + return this; + } + + public Long getTransferredBytes() { + return this.transferredBytes; + } + + public Builder totalBytes(Long totalBytes) { + this.totalBytes = totalBytes; + return this; + } + + public Long getTotalBytes() { + return this.totalBytes; + } + + public Builder startTime(Instant startTime) { + this.startTime = Optional.of(startTime); + return this; + } + + public Optional startTime() { + return this.startTime; + } + + @Override + public DefaultProgressSnapshot build() { + return new DefaultProgressSnapshot(this); + } + } +} \ No newline at end of file diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java new file mode 100644 index 000000000000..2cf09d75cec5 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java @@ -0,0 +1,202 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.progress; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.junit.Assert.assertEquals; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.progress.snapshot.DefaultProgressSnapshot; + +public class DefaultProgressSnapshotTest { + @Test + public void bytesTransferred_greaterThan_totalBytes_shouldThrow() { + DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(1L); + assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("transferredBytes (2) must not be greater than totalBytes (1)"); + } + + @Test + public void transferredBytes_negative_shouldThrow() { + DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() + .transferredBytes(-2L); + + assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("transferredBytes must not be negative"); + } + + @Test + public void transferredBytes_null_isZero() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .build(); + + Assertions.assertEquals(0, snapshot.transferredBytes()); + } + + @Test + public void totalBytes_negative_shouldThrow() { + DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(-2L); + + assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("totalBytes must not be negative"); + } + + @Test + public void totalBytes_empty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .build(); + + assertThat(snapshot.totalBytes()).isNotPresent(); + } + + @Test + public void totalBytes() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(5L) + .build(); + + Assertions.assertEquals(5, snapshot.totalBytes().getAsLong()); + } + + @Test + public void startTime_after_currentTime_shouldThrow() { + Instant timeAfterFiveSeconds = Instant.now().plus(5, ChronoUnit.SECONDS); + DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() + .transferredBytes(0L) + .startTime(timeAfterFiveSeconds); + + assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("currentTime") + .hasMessageEndingWith(" must not be before startTime (" + timeAfterFiveSeconds + ")"); + } + + @Test + public void ratioTransferred_withoutTotalBytes_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .build(); + assertThat(snapshot.ratioTransferred()).isNotPresent(); + } + + @Test + public void ratioTransferred() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .totalBytes(5L) + .build(); + assertEquals(0.2, snapshot.ratioTransferred().getAsDouble(), 0.0); + } + + @Test + public void remainingBytes_withoutTotalBytes_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .build(); + assertThat(snapshot.remainingBytes()).isNotPresent(); + } + + @Test + public void remainingBytes() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .totalBytes(5L) + .build(); + Assertions.assertEquals(4.0, snapshot.remainingBytes().getAsLong(), 0.0); + } + + @Test + public void elapsedTime_withoutStartTime_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .build(); + assertThat(snapshot.elapsedTime()).isNotPresent(); + } + + @Test + public void elapsedTime() { + + Instant startTime = Instant.now().minusMillis(100); + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .startTime(startTime) + .build(); + Duration expectedDuration = Duration.between(startTime, Instant.now()); + + Assertions.assertEquals(snapshot.elapsedTime().get().toMillis(), expectedDuration.toMillis(), 0.1); + } + + @Test + public void averageBytesPer_withoutStartTime_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .totalBytes(5L) + .build(); + assertThat(snapshot.averageBytesPer(TimeUnit.MILLISECONDS)).isNotPresent(); + } + + @Test + public void averageBytesPer() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(100L) + .startTime(Instant.now().minusMillis(100)) + .build(); + Assertions.assertEquals(1.0, snapshot.averageBytesPer(TimeUnit.MILLISECONDS).getAsDouble(), 0.2); + } + + @Test + public void estimatedTimeRemaining_withoutStartTime_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .totalBytes(5L) + .build(); + assertThat(snapshot.estimatedTimeRemaining()).isNotPresent(); + } + + @Test + public void estimatedTimeRemaining_withoutTotalBytes_isEmpty() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .startTime(Instant.now().minusMillis(5)) + .build(); + assertThat(snapshot.estimatedTimeRemaining()).isNotPresent(); + } + + @Test + public void estimatedTimeRemaining() { + DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() + .transferredBytes(100L) + .startTime(Instant.now().minusSeconds(1)) + .totalBytes(500L) + .build(); + Assertions.assertEquals(4000.0, snapshot.estimatedTimeRemaining().get().toMillis(), 10.0); + } +} From 1d56ce252d24dc397b8f03c718dbaf4f055b2525 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Thu, 14 Dec 2023 02:50:20 -0800 Subject: [PATCH 17/50] Remove changes to pom --- core/sdk-core/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/sdk-core/pom.xml b/core/sdk-core/pom.xml index d15be86a3608..f19120fde62d 100644 --- a/core/sdk-core/pom.xml +++ b/core/sdk-core/pom.xml @@ -284,14 +284,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - - 11 - 11 - - From 2e66a9a9438e8313af2b3160578f5f5bf28190d6 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Wed, 20 Dec 2023 11:49:14 -0800 Subject: [PATCH 18/50] Add parameterized unit tests --- .../listener/DefaultSdkRequestProgress.java | 5 +- .../snapshot/DefaultProgressSnapshot.java | 89 ++++--- .../progress/snapshot/ProgressSnapshot.java | 6 +- .../progress/DefaultProgressSnapshotTest.java | 238 ++++++++---------- .../DefaultTransferProgressSnapshot.java | 8 - 5 files changed, 159 insertions(+), 187 deletions(-) rename core/sdk-core/src/main/java/software/amazon/awssdk/core/{progress => internal}/listener/DefaultSdkRequestProgress.java (91%) rename core/sdk-core/src/main/java/software/amazon/awssdk/core/{progress => internal}/snapshot/DefaultProgressSnapshot.java (68%) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/listener/DefaultSdkRequestProgress.java similarity index 91% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/listener/DefaultSdkRequestProgress.java index 6c8a602afcc6..20a4b7b3e2fb 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/DefaultSdkRequestProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/listener/DefaultSdkRequestProgress.java @@ -13,14 +13,15 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.progress.listener; +package software.amazon.awssdk.core.internal.listener; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import software.amazon.awssdk.annotations.Mutable; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.annotations.ThreadSafe; -import software.amazon.awssdk.core.progress.snapshot.DefaultProgressSnapshot; +import software.amazon.awssdk.core.internal.snapshot.DefaultProgressSnapshot; +import software.amazon.awssdk.core.progress.listener.SdkRequestProgress; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; /** diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/snapshot/DefaultProgressSnapshot.java similarity index 68% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/snapshot/DefaultProgressSnapshot.java index 03de4239f204..2f20b450cf25 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/DefaultProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/snapshot/DefaultProgressSnapshot.java @@ -13,15 +13,18 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.progress.snapshot; +package software.amazon.awssdk.core.internal.snapshot; import java.time.Duration; import java.time.Instant; +import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalLong; import java.util.concurrent.TimeUnit; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; +import software.amazon.awssdk.utils.ToString; import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; @@ -36,7 +39,7 @@ public class DefaultProgressSnapshot private final long transferredBytes; private final Long totalBytes; - private final Optional startTime; + private final Instant startTime; public DefaultProgressSnapshot(Builder builder) { if (builder.totalBytes != null) { @@ -45,17 +48,11 @@ public DefaultProgressSnapshot(Builder builder) { "transferredBytes (%s) must not be greater than totalBytes (%s)", builder.transferredBytes, builder.totalBytes); } - Validate.paramNotNull(builder.transferredBytes, "byteTransferred"); + Validate.paramNotNull(builder.transferredBytes, "transferredBytes"); this.transferredBytes = Validate.isNotNegative(builder.transferredBytes, "transferredBytes"); this.totalBytes = builder.totalBytes; - if (builder.startTime.isPresent()) { - Instant currentTime = Instant.now(); - Validate.isTrue(currentTime.isAfter(builder.startTime.get()), - "currentTime (%s) must not be before startTime (%s)", - currentTime, builder.startTime.get()); - } - + Validate.paramNotNull(builder.startTime, "startTime"); this.startTime = builder.startTime; } @@ -65,34 +62,30 @@ public long transferredBytes() { } @Override - public Optional startTime() { + public Instant startTime() { return this.startTime; } @Override - public Optional elapsedTime() { - return this.startTime.isPresent() ? Optional.of(Duration.between(startTime.get(), Instant.now())) : Optional.empty(); + public Duration elapsedTime() { + return Duration.between(this.startTime, Instant.now()); } @Override public Optional estimatedTimeRemaining() { - if (!elapsedTime().isPresent() || !remainingBytes().isPresent()) { + if (!remainingBytes().isPresent()) { return Optional.empty(); } - long remainingTime = remainingBytes().getAsLong() * elapsedTime().get().toMillis() / transferredBytes; + long remainingTime = remainingBytes().getAsLong() * elapsedTime().toMillis() / transferredBytes; return Optional.of(Duration.ofMillis(remainingTime)); } @Override - public OptionalDouble averageBytesPer(TimeUnit timeUnit) { - if (!this.elapsedTime().isPresent()) { - return OptionalDouble.empty(); - } - - return this.elapsedTime().get().equals(Duration.ZERO) ? OptionalDouble.of(1.0) : - OptionalDouble.of((double) this.transferredBytes / timeUnit.convert(elapsedTime().get().toMillis(), timeUnit)); + public double averageBytesPer(TimeUnit timeUnit) { + return this.elapsedTime().equals(Duration.ZERO) ? 1.0 : + (double) this.transferredBytes / timeUnit.convert(elapsedTime().toMillis(), timeUnit); } @Override @@ -116,6 +109,42 @@ public OptionalLong remainingBytes() { return totalBytes == 0 ? OptionalLong.of(0) : OptionalLong.of(totalBytes - transferredBytes); } + @Override + public String toString() { + return ToString.builder("ProgressSnapshot") + .add("transferredBytes", transferredBytes) + .add("totalBytes", totalBytes) + .add("elapsedTime", this.elapsedTime()) + .build(); + } + + @Override + public int hashCode() { + int result = (int) (transferredBytes ^ (transferredBytes >>> 32)); + result = 31 * result + (totalBytes != null ? totalBytes.hashCode() : 0); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DefaultProgressSnapshot that = (DefaultProgressSnapshot) o; + + if (this.transferredBytes != that.transferredBytes) { + return false; + } + if (!Objects.equals(this.totalBytes, that.totalBytes)) { + return false; + } + return Objects.equals(this.startTime, that.startTime); + } + @Override public DefaultProgressSnapshot.Builder toBuilder() { return new Builder(this); @@ -128,7 +157,7 @@ public static Builder builder() { public static final class Builder implements CopyableBuilder { private long transferredBytes; private Long totalBytes; - private Optional startTime = Optional.empty(); + private Instant startTime; private Builder() { } @@ -144,28 +173,16 @@ public Builder transferredBytes(Long transferredBytes) { return this; } - public Long getTransferredBytes() { - return this.transferredBytes; - } - public Builder totalBytes(Long totalBytes) { this.totalBytes = totalBytes; return this; } - public Long getTotalBytes() { - return this.totalBytes; - } - public Builder startTime(Instant startTime) { - this.startTime = Optional.of(startTime); + this.startTime = startTime; return this; } - public Optional startTime() { - return this.startTime; - } - @Override public DefaultProgressSnapshot build() { return new DefaultProgressSnapshot(this); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java index 829ce35b0dc0..300657e79399 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/snapshot/ProgressSnapshot.java @@ -37,12 +37,12 @@ public interface ProgressSnapshot { /** * Time at which the HTTP Request header is sent */ - Optional startTime(); + Instant startTime(); /** * Elapsed time since the HTTP request header was sent to the service */ - Optional elapsedTime(); + Duration elapsedTime(); /** * If transaction size is known, estimate time remaining for transaction completion @@ -57,7 +57,7 @@ public interface ProgressSnapshot { /** * Rate of transfer */ - OptionalDouble averageBytesPer(TimeUnit timeUnit); + double averageBytesPer(TimeUnit timeUnit); /** * The total size of the transfer, in bytes, or {@link Optional#empty()} if total payload being transacted is unknown diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java index 2cf09d75cec5..fdf159727eaf 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java @@ -18,93 +18,117 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import software.amazon.awssdk.core.progress.snapshot.DefaultProgressSnapshot; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.core.internal.snapshot.DefaultProgressSnapshot; public class DefaultProgressSnapshotTest { - @Test - public void bytesTransferred_greaterThan_totalBytes_shouldThrow() { - DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() - .transferredBytes(2L) - .totalBytes(1L); - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("transferredBytes (2) must not be greater than totalBytes (1)"); - } - @Test - public void transferredBytes_negative_shouldThrow() { - DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() - .transferredBytes(-2L); - - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("transferredBytes must not be negative"); - } - - @Test - public void transferredBytes_null_isZero() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .build(); + private static Stream getArgumentsForInvalidParameterValidationTests() { + return Stream.of(Arguments.of("transferredBytes (2) must not be greater than totalBytes (1)", + DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(1L), + new IllegalArgumentException()), + Arguments.of("startTime must not be null.", + DefaultProgressSnapshot.builder() + .transferredBytes(2L), + new NullPointerException()), + Arguments.of("transferredBytes must not be negative", + DefaultProgressSnapshot.builder() + .transferredBytes(-2L), + new IllegalArgumentException()), + Arguments.of("totalBytes must not be negative", + DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(-2L), + new IllegalArgumentException())); + } + + private static Stream getArgumentsForMissingParameterValidationTests() { + + DefaultProgressSnapshot snapshotNoTotalBytes = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .startTime(Instant.now()) + .build(); + + DefaultProgressSnapshot snapshotRatioTransferredWithoutTotalBytesIsEmpty = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .startTime(Instant.now()) + .build(); + + DefaultProgressSnapshot snapshotRemainingBytesWithoutTotalBytesIsEmpty = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .startTime(Instant.now()) + .build(); + + DefaultProgressSnapshot snapshotEstimatedTimeRemainingWithoutTotalBytesIsEmpty = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .startTime(Instant.now()) + .build(); + + return Stream.of(Arguments.of(snapshotNoTotalBytes.totalBytes().isPresent()), + Arguments.of(snapshotRatioTransferredWithoutTotalBytesIsEmpty.ratioTransferred().isPresent()), + Arguments.of(snapshotRemainingBytesWithoutTotalBytesIsEmpty.remainingBytes().isPresent()), + Arguments.of(snapshotEstimatedTimeRemainingWithoutTotalBytesIsEmpty.estimatedTimeRemaining().isPresent())); + } + + private static Stream getArgumentsForTimeTest() { + + DefaultProgressSnapshot snapshotEstimatedTimeReamining = DefaultProgressSnapshot.builder() + .transferredBytes(100L) + .startTime(Instant.now().minusSeconds(1)) + .totalBytes(500L) + .build(); - Assertions.assertEquals(0, snapshot.transferredBytes()); - } - - @Test - public void totalBytes_negative_shouldThrow() { - DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() - .transferredBytes(2L) - .totalBytes(-2L); + Instant startTime = Instant.now().minusMillis(100); + DefaultProgressSnapshot snapshotTimeElapsed = DefaultProgressSnapshot.builder() + .transferredBytes(1L) + .startTime(startTime) + .build(); + Duration expectedDuration = Duration.between(startTime, Instant.now()); - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("totalBytes must not be negative"); + return Stream.of(Arguments.of(4000L, snapshotEstimatedTimeReamining.estimatedTimeRemaining().get().toMillis() + , 10L), + Arguments.of(snapshotTimeElapsed.elapsedTime().toMillis(), expectedDuration.toMillis(), 1L)); } - @Test - public void totalBytes_empty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(2L) - .build(); - - assertThat(snapshot.totalBytes()).isNotPresent(); - } + private static Stream getArgumentsForBytesTest() { - @Test - public void totalBytes() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(2L) - .totalBytes(5L) - .build(); + DefaultProgressSnapshot snapshotBytes = DefaultProgressSnapshot.builder() + .transferredBytes(2L) + .totalBytes(5L) + .startTime(Instant.now()) + .build(); - Assertions.assertEquals(5, snapshot.totalBytes().getAsLong()); + return Stream.of(Arguments.of(5L, snapshotBytes.totalBytes().getAsLong()), + Arguments.of(3L, snapshotBytes.remainingBytes().getAsLong(), 3L)); } - @Test - public void startTime_after_currentTime_shouldThrow() { - Instant timeAfterFiveSeconds = Instant.now().plus(5, ChronoUnit.SECONDS); - DefaultProgressSnapshot.Builder builder = DefaultProgressSnapshot.builder() - .transferredBytes(0L) - .startTime(timeAfterFiveSeconds); - + @ParameterizedTest + @MethodSource("getArgumentsForInvalidParameterValidationTests") + public void test_invalid_arguments_shouldThrow(String expectedErrorMsg, DefaultProgressSnapshot.Builder builder, + Exception e) { assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageStartingWith("currentTime") - .hasMessageEndingWith(" must not be before startTime (" + timeAfterFiveSeconds + ")"); + .isInstanceOf(e.getClass()) + .hasMessage(expectedErrorMsg); } - @Test - public void ratioTransferred_withoutTotalBytes_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .build(); - assertThat(snapshot.ratioTransferred()).isNotPresent(); + @ParameterizedTest + @MethodSource("getArgumentsForMissingParameterValidationTests") + public void test_missing_params_shouldReturnEmpty(boolean condition) { + Assertions.assertFalse(condition); } @Test @@ -112,55 +136,21 @@ public void ratioTransferred() { DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() .transferredBytes(1L) .totalBytes(5L) + .startTime(Instant.now()) .build(); assertEquals(0.2, snapshot.ratioTransferred().getAsDouble(), 0.0); } - @Test - public void remainingBytes_withoutTotalBytes_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .build(); - assertThat(snapshot.remainingBytes()).isNotPresent(); + @ParameterizedTest + @MethodSource("getArgumentsForBytesTest") + public void test_estimatedBytesRemaining_and_totalBytes(long expectedBytes, long actualBytes) { + Assertions.assertEquals(expectedBytes, actualBytes); } - @Test - public void remainingBytes() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .totalBytes(5L) - .build(); - Assertions.assertEquals(4.0, snapshot.remainingBytes().getAsLong(), 0.0); - } - - @Test - public void elapsedTime_withoutStartTime_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .build(); - assertThat(snapshot.elapsedTime()).isNotPresent(); - } - - @Test - public void elapsedTime() { - - Instant startTime = Instant.now().minusMillis(100); - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .startTime(startTime) - .build(); - Duration expectedDuration = Duration.between(startTime, Instant.now()); - - Assertions.assertEquals(snapshot.elapsedTime().get().toMillis(), expectedDuration.toMillis(), 0.1); - } - - @Test - public void averageBytesPer_withoutStartTime_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .totalBytes(5L) - .build(); - assertThat(snapshot.averageBytesPer(TimeUnit.MILLISECONDS)).isNotPresent(); + @ParameterizedTest + @MethodSource("getArgumentsForTimeTest") + public void test_elapsedTime_and_estimatedTimeRemaining(long expected, long timeInMillis, long delta) { + Assertions.assertEquals(expected, timeInMillis, delta); } @Test @@ -169,34 +159,6 @@ public void averageBytesPer() { .transferredBytes(100L) .startTime(Instant.now().minusMillis(100)) .build(); - Assertions.assertEquals(1.0, snapshot.averageBytesPer(TimeUnit.MILLISECONDS).getAsDouble(), 0.2); - } - - @Test - public void estimatedTimeRemaining_withoutStartTime_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .totalBytes(5L) - .build(); - assertThat(snapshot.estimatedTimeRemaining()).isNotPresent(); - } - - @Test - public void estimatedTimeRemaining_withoutTotalBytes_isEmpty() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(1L) - .startTime(Instant.now().minusMillis(5)) - .build(); - assertThat(snapshot.estimatedTimeRemaining()).isNotPresent(); - } - - @Test - public void estimatedTimeRemaining() { - DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() - .transferredBytes(100L) - .startTime(Instant.now().minusSeconds(1)) - .totalBytes(500L) - .build(); - Assertions.assertEquals(4000.0, snapshot.estimatedTimeRemaining().get().toMillis(), 10.0); + Assertions.assertEquals(1.0, snapshot.averageBytesPer(TimeUnit.MILLISECONDS), 0.2); } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java index b0bf72e91404..64b67b081ee9 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java @@ -150,19 +150,11 @@ public Builder transferredBytes(Long transferredBytes) { return this; } - public long getTransferredBytes() { - return transferredBytes; - } - public Builder totalBytes(Long totalBytes) { this.totalBytes = totalBytes; return this; } - public Long getTotalBytes() { - return totalBytes; - } - public Builder sdkResponse(SdkResponse sdkResponse) { this.sdkResponse = sdkResponse; return this; From 3ffbca2c98def6d09cdde06df9b1d15cdc0ce343 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Wed, 20 Dec 2023 13:58:46 -0800 Subject: [PATCH 19/50] fix failing tests --- .../listener/DefaultSdkRequestProgress.java | 4 ++-- .../{ => progress}/snapshot/DefaultProgressSnapshot.java | 2 +- .../awssdk/core/progress/DefaultProgressSnapshotTest.java | 4 +--- .../progress/DefaultTransferProgressSnapshot.java | 8 ++++++++ 4 files changed, 12 insertions(+), 6 deletions(-) rename core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/{ => progress}/listener/DefaultSdkRequestProgress.java (93%) rename core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/{ => progress}/snapshot/DefaultProgressSnapshot.java (98%) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/listener/DefaultSdkRequestProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java similarity index 93% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/listener/DefaultSdkRequestProgress.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java index 20a4b7b3e2fb..019c869ae060 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/listener/DefaultSdkRequestProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.internal.listener; +package software.amazon.awssdk.core.internal.progress.listener; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import software.amazon.awssdk.annotations.Mutable; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.annotations.ThreadSafe; -import software.amazon.awssdk.core.internal.snapshot.DefaultProgressSnapshot; +import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; import software.amazon.awssdk.core.progress.listener.SdkRequestProgress; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/snapshot/DefaultProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshot.java similarity index 98% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/snapshot/DefaultProgressSnapshot.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshot.java index 2f20b450cf25..86e9a64004d2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/snapshot/DefaultProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshot.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.internal.snapshot; +package software.amazon.awssdk.core.internal.progress.snapshot; import java.time.Duration; import java.time.Instant; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java index fdf159727eaf..760e7a0c3c90 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java @@ -22,7 +22,6 @@ import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; @@ -30,8 +29,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; -import software.amazon.awssdk.core.internal.snapshot.DefaultProgressSnapshot; +import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; public class DefaultProgressSnapshotTest { diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java index 64b67b081ee9..b0bf72e91404 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/DefaultTransferProgressSnapshot.java @@ -150,11 +150,19 @@ public Builder transferredBytes(Long transferredBytes) { return this; } + public long getTransferredBytes() { + return transferredBytes; + } + public Builder totalBytes(Long totalBytes) { this.totalBytes = totalBytes; return this; } + public Long getTotalBytes() { + return totalBytes; + } + public Builder sdkResponse(SdkResponse sdkResponse) { this.sdkResponse = sdkResponse; return this; From 2dfdef78ec8275044127eb5dafb813a6f70372d8 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Wed, 27 Dec 2023 02:12:17 -0800 Subject: [PATCH 20/50] Defined ProgressListener invoker methods, along with success and failure context objects. Added progress listener method to RequestOverrideConfiguration --- .../core/RequestOverrideConfiguration.java | 149 ++++++++++------ .../progress/ProgressListenerContext.java | 162 ++++++++++++++++++ .../ProgressListenerFailedContext.java | 121 +++++++++++++ .../listener/ProgressListenerInvoker.java | 104 +++++++++++ .../progress/listener/ProgressListener.java | 2 +- 5 files changed, 484 insertions(+), 54 deletions(-) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java index ae6bd6fd85a5..f437c0dde2ea 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java @@ -31,6 +31,7 @@ import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.interceptor.ExecutionAttribute; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.core.signer.Signer; import software.amazon.awssdk.endpoints.EndpointProvider; import software.amazon.awssdk.metrics.MetricPublisher; @@ -55,6 +56,7 @@ public abstract class RequestOverrideConfiguration { private final EndpointProvider endpointProvider; private final CompressionConfiguration compressionConfiguration; private final List plugins; + private final List progressListeners; protected RequestOverrideConfiguration(Builder builder) { this.headers = CollectionUtils.deepUnmodifiableMap(builder.headers(), () -> new TreeMap<>(String.CASE_INSENSITIVE_ORDER)); @@ -68,6 +70,7 @@ protected RequestOverrideConfiguration(Builder builder) { this.endpointProvider = builder.endpointProvider(); this.compressionConfiguration = builder.compressionConfiguration(); this.plugins = Collections.unmodifiableList(new ArrayList<>(builder.plugins())); + this.progressListeners = Collections.unmodifiableList(new ArrayList<>()); } /** @@ -103,9 +106,9 @@ public List apiNames() { * unmarshalling, etc. This value should always be positive, if present. * *

    The api call timeout feature doesn't have strict guarantees on how quickly a request is aborted when the - * timeout is breached. The typical case aborts the request within a few milliseconds but there may occasionally be - * requests that don't get aborted until several seconds after the timer has been breached. Because of this, the client - * execution timeout feature should not be used when absolute precision is needed. + * timeout is breached. The typical case aborts the request within a few milliseconds but there may occasionally be requests + * that don't get aborted until several seconds after the timer has been breached. Because of this, the client execution + * timeout feature should not be used when absolute precision is needed. * *

    This may be used together with {@link #apiCallAttemptTimeout()} to enforce both a timeout on each individual HTTP * request (i.e. each retry) and the total time spent on all requests across retries (i.e. the 'api call' time). @@ -121,13 +124,12 @@ public Optional apiCallTimeout() { * positive, if present. * *

    The request timeout feature doesn't have strict guarantees on how quickly a request is aborted when the timeout is - * breached. The typical case aborts the request within a few milliseconds but there may occasionally be requests that - * don't get aborted until several seconds after the timer has been breached. Because of this, the request timeout - * feature should not be used when absolute precision is needed. + * breached. The typical case aborts the request within a few milliseconds but there may occasionally be requests that don't + * get aborted until several seconds after the timer has been breached. Because of this, the request timeout feature should + * not be used when absolute precision is needed. * *

    This may be used together with {@link #apiCallTimeout()} to enforce both a timeout on each individual HTTP - * request - * (i.e. each retry) and the total time spent on all requests across retries (i.e. the 'api call' time). + * request (i.e. each retry) and the total time spent on all requests across retries (i.e. the 'api call' time). * * @see Builder#apiCallAttemptTimeout(Duration) */ @@ -136,16 +138,16 @@ public Optional apiCallAttemptTimeout() { } /** - * @return the signer for signing the request. This signer get priority over the signer set on the client while - * signing the requests. If this value is not set, then the client level signer is used for signing the request. + * @return the signer for signing the request. This signer get priority over the signer set on the client while signing the + * requests. If this value is not set, then the client level signer is used for signing the request. */ public Optional signer() { return Optional.ofNullable(signer); } /** - * Return the metric publishers for publishing the metrics collected for this request. This list supersedes the - * metric publishers set on the client. + * Return the metric publishers for publishing the metrics collected for this request. This list supersedes the metric + * publishers set on the client. */ public List metricPublishers() { return metricPublishers; @@ -160,27 +162,32 @@ public List plugins() { } /** - * Returns the additional execution attributes to be added to this request. - * This collection of attributes is added in addition to the attributes set on the client. - * An attribute value added on the client within the collection of attributes is superseded by an - * attribute value added on the request. + * Return the progress listeners that will be used to listen in and track request progress + */ + public List progressListeners() { + return progressListeners; + } + + /** + * Returns the additional execution attributes to be added to this request. This collection of attributes is added in addition + * to the attributes set on the client. An attribute value added on the client within the collection of attributes is + * superseded by an attribute value added on the request. */ public ExecutionAttributes executionAttributes() { return executionAttributes; } /** - * Returns the endpoint provider for resolving the endpoint for this request. This supersedes the - * endpoint provider set on the client. + * Returns the endpoint provider for resolving the endpoint for this request. This supersedes the endpoint provider set on the + * client. */ public Optional endpointProvider() { return Optional.ofNullable(endpointProvider); } /** - * Returns the compression configuration object, if present, which includes options to enable/disable compression and set - * the minimum compression threshold. This compression config object supersedes the compression config object set on the - * client. + * Returns the compression configuration object, if present, which includes options to enable/disable compression and set the + * minimum compression threshold. This compression config object supersedes the compression config object set on the client. */ public Optional compressionConfiguration() { return Optional.ofNullable(compressionConfiguration); @@ -205,7 +212,8 @@ public boolean equals(Object o) { Objects.equals(executionAttributes, that.executionAttributes) && Objects.equals(endpointProvider, that.endpointProvider) && Objects.equals(compressionConfiguration, that.compressionConfiguration) && - Objects.equals(plugins, that.plugins); + Objects.equals(plugins, that.plugins) && + Objects.equals(progressListeners, that.progressListeners); } @Override @@ -222,6 +230,7 @@ public int hashCode() { hashCode = 31 * hashCode + Objects.hashCode(endpointProvider); hashCode = 31 * hashCode + Objects.hashCode(compressionConfiguration); hashCode = 31 * hashCode + Objects.hashCode(plugins); + hashCode = 31 * hashCode + Objects.hashCode(progressListeners); return hashCode; } @@ -243,14 +252,13 @@ public interface Builder { /** * Add a single header to be set on the HTTP request. *

    - * This overrides any values for the given header set on the request by default by the SDK, as well as header - * overrides set at the client level using - * {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. + * This overrides any values for the given header set on the request by default by the SDK, as well as header overrides + * set at the client level using {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. * *

    * This overrides any values already configured with this header name in the builder. * - * @param name The name of the header. + * @param name The name of the header. * @param value The value of the header. * @return This object for method chaining. */ @@ -262,14 +270,13 @@ default B putHeader(String name, String value) { /** * Add a single header with multiple values to be set on the HTTP request. *

    - * This overrides any values for the given header set on the request by default by the SDK, as well as header - * overrides set at the client level using - * {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. + * This overrides any values for the given header set on the request by default by the SDK, as well as header overrides + * set at the client level using {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. * *

    * This overrides any values already configured with this header name in the builder. * - * @param name The name of the header. + * @param name The name of the header. * @param values The values of the header. * @return This object for method chaining. */ @@ -278,9 +285,8 @@ default B putHeader(String name, String value) { /** * Add additional headers to be set on the HTTP request. *

    - * This overrides any values for the given headers set on the request by default by the SDK, as well as header - * overrides set at the client level using - * {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. + * This overrides any values for the given headers set on the request by default by the SDK, as well as header overrides + * set at the client level using {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. * *

    * This completely overrides any values currently configured in the builder. @@ -303,7 +309,7 @@ default B putHeader(String name, String value) { *

    * This overrides any values already configured with this query name in the builder. * - * @param name The query parameter name. + * @param name The query parameter name. * @param value The query parameter value. * @return This object for method chaining. */ @@ -318,7 +324,7 @@ default B putRawQueryParameter(String name, String value) { *

    * This overrides any values already configured with this query name in the builder. * - * @param name The query parameter name. + * @param name The query parameter name. * @param values The query parameter values. * @return This object for method chaining. */ @@ -346,7 +352,6 @@ default B putRawQueryParameter(String name, String value) { * Set the optional name of the higher level library that constructed the request. * * @param apiName The name of the library. - * * @return This object for method chaining. */ B addApiName(ApiName apiName); @@ -355,7 +360,6 @@ default B putRawQueryParameter(String name, String value) { * Set the optional name of the higher level library that constructed the request. * * @param apiNameConsumer A {@link Consumer} that accepts a {@link ApiName.Builder}. - * * @return This object for method chaining. */ B addApiName(Consumer apiNameConsumer); @@ -389,8 +393,8 @@ default B putRawQueryParameter(String name, String value) { * *

    The request timeout feature doesn't have strict guarantees on how quickly a request is aborted when the timeout is * breached. The typical case aborts the request within a few milliseconds but there may occasionally be requests that - * don't get aborted until several seconds after the timer has been breached. Because of this, the request timeout - * feature should not be used when absolute precision is needed. + * don't get aborted until several seconds after the timer has been breached. Because of this, the request timeout feature + * should not be used when absolute precision is needed. * *

    This may be used together with {@link #apiCallTimeout()} to enforce both a timeout on each individual HTTP * request (i.e. each retry) and the total time spent on all requests across retries (i.e. the 'api call' time). @@ -417,8 +421,8 @@ default B putRawQueryParameter(String name, String value) { Signer signer(); /** - * Sets the metric publishers for publishing the metrics collected for this request. This list supersedes - * the metric publisher set on the client. + * Sets the metric publishers for publishing the metrics collected for this request. This list supersedes the metric + * publisher set on the client. * * @param metricPublisher The list metric publisher for this request. * @return This object for method chaining. @@ -426,8 +430,8 @@ default B putRawQueryParameter(String name, String value) { B metricPublishers(List metricPublisher); /** - * Add a metric publisher to the existing list of previously set publishers to be used for publishing metrics - * for this request. + * Add a metric publisher to the existing list of previously set publishers to be used for publishing metrics for this + * request. * * @param metricPublisher The metric publisher to add. */ @@ -437,6 +441,7 @@ default B putRawQueryParameter(String name, String value) { /** * Sets the additional execution attributes collection for this request. + * * @param executionAttributes Execution attributes for this request * @return This object for method chaining. */ @@ -444,17 +449,18 @@ default B putRawQueryParameter(String name, String value) { /** * Add an execution attribute to the existing collection of execution attributes. + * * @param attribute The execution attribute object - * @param value The value of the execution attribute. + * @param value The value of the execution attribute. */ B putExecutionAttribute(ExecutionAttribute attribute, T value); ExecutionAttributes executionAttributes(); /** - * Sets the endpointProvider to use for resolving the endpoint of the request. This endpointProvider gets priority - * over the endpointProvider set on the client while resolving the endpoint for the requests. - * If this value is null, then the client level endpointProvider is used for resolving the endpoint. + * Sets the endpointProvider to use for resolving the endpoint of the request. This endpointProvider gets priority over + * the endpointProvider set on the client while resolving the endpoint for the requests. If this value is null, then the + * client level endpointProvider is used for resolving the endpoint. * * @param endpointProvider Endpoint Provider that will override the resolving the endpoint for the request. * @return This object for method chaining @@ -464,19 +470,18 @@ default B putRawQueryParameter(String name, String value) { EndpointProvider endpointProvider(); /** - * Sets the {@link CompressionConfiguration} for this request. The order of precedence, from highest to lowest, - * for this setting is: 1) Per request configuration 2) Client configuration 3) Environment variables 4) Profile setting. + * Sets the {@link CompressionConfiguration} for this request. The order of precedence, from highest to lowest, for this + * setting is: 1) Per request configuration 2) Client configuration 3) Environment variables 4) Profile setting. * * @param compressionConfiguration Request compression configuration object for this request. */ B compressionConfiguration(CompressionConfiguration compressionConfiguration); /** - * Sets the {@link CompressionConfiguration} for this request. The order of precedence, from highest to lowest, - * for this setting is: 1) Per request configuration 2) Client configuration 3) Environment variables 4) Profile setting. + * Sets the {@link CompressionConfiguration} for this request. The order of precedence, from highest to lowest, for this + * setting is: 1) Per request configuration 2) Client configuration 3) Environment variables 4) Profile setting. * * @param compressionConfigurationConsumer A {@link Consumer} that accepts a {@link CompressionConfiguration.Builder} - * * @return This object for method chaining */ B compressionConfiguration(Consumer compressionConfigurationConsumer); @@ -506,6 +511,26 @@ default B putRawQueryParameter(String name, String value) { @SdkPreviewApi List plugins(); + /** + * Sets the progress listeners used to perform request prorgess tracking + * + * @param progressListeners The list of plugins for this request. + * @return This object for method chaining. + */ + B progressListeners(List progressListeners); + + /** + * Add a progress listener used to track request progress + * + * @param progressListener The listener to add. + */ + B addProgressListener(ProgressListener progressListener); + + /** + * Returns the list of progress listeners attached to the request + */ + List progressListeners(); + /** * Create a new {@code SdkRequestOverrideConfiguration} with the properties set on this builder. * @@ -526,7 +551,7 @@ protected abstract static class BuilderImpl implements Builde private EndpointProvider endpointProvider; private CompressionConfiguration compressionConfiguration; private List plugins = new ArrayList<>(); - + private List progressListeners = new ArrayList<>(); protected BuilderImpl() { } @@ -543,6 +568,7 @@ protected BuilderImpl(RequestOverrideConfiguration sdkRequestOverrideConfig) { endpointProvider(sdkRequestOverrideConfig.endpointProvider); compressionConfiguration(sdkRequestOverrideConfig.compressionConfiguration); plugins(sdkRequestOverrideConfig.plugins); + progressListeners(sdkRequestOverrideConfig.progressListeners); } @Override @@ -747,5 +773,22 @@ public B addPlugin(SdkPlugin plugin) { public List plugins() { return Collections.unmodifiableList(plugins); } + + @Override + public B progressListeners(List progressListeners) { + this.progressListeners = new ArrayList<>(progressListeners); + return (B) this; + } + + @Override + public B addProgressListener(ProgressListener progressListener) { + this.progressListeners.add(progressListener); + return (B) this; + } + + @Override + public List progressListeners() { + return Collections.unmodifiableList(progressListeners); + } } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java new file mode 100644 index 000000000000..da905b69e95b --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java @@ -0,0 +1,162 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.progress; + +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * An SDK-internal implementation of {@link ProgressListener.Context.ExecutionSuccess} and its parent interfaces. + * + * @see ProgressListenerFailedContext + */ +@SdkProtectedApi +@Immutable +public final class ProgressListenerContext + implements ProgressListener.Context.ExecutionSuccess, + ToCopyableBuilder { + + private final SdkRequest sdkRequest; + private final SdkHttpRequest sdkHttpRequest; + private final ProgressSnapshot uploadProgressSnapshot; + private final ProgressSnapshot downloadProgressSnapshot; + private final SdkHttpResponse sdkHttpResponse; + private final SdkResponse executionSuccessfulSdkResponse; + + private ProgressListenerContext(Builder builder) { + this.sdkRequest = builder.sdkRequest; + this.sdkHttpRequest = builder.sdkHttpRequest; + this.uploadProgressSnapshot = builder.uploadProgressSnapshot; + this.downloadProgressSnapshot = builder.downloadProgressSnapshot; + this.sdkHttpResponse = builder.sdkHttpResponse; + this.executionSuccessfulSdkResponse = builder.executionSuccessfulSdkResponse; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + @Override + public SdkRequest request() { + return sdkRequest; + } + + @Override + public SdkHttpRequest httpRequest() { + return sdkHttpRequest; + } + + @Override + public ProgressSnapshot uploadProgressSnapshot() { + return uploadProgressSnapshot; + } + + @Override + public ProgressSnapshot downloadProgressSnapshot() { + return downloadProgressSnapshot; + } + + @Override + public SdkHttpResponse httpResponse() { + return sdkHttpResponse; + } + + @Override + public SdkResponse response() { + return executionSuccessfulSdkResponse; + } + + @Override + public String toString() { + return ToString.builder("ProgressListenerContext") + .add("sdkRequest", sdkRequest) + .add("uploadProgressSnapshot", uploadProgressSnapshot) + .add("downloadProgressSnapshot", downloadProgressSnapshot) + .add("executionSuccessfulSdkResponse", executionSuccessfulSdkResponse) + .build(); + } + + public static final class Builder implements CopyableBuilder { + private SdkRequest sdkRequest; + private SdkHttpRequest sdkHttpRequest; + private ProgressSnapshot uploadProgressSnapshot; + private ProgressSnapshot downloadProgressSnapshot; + private SdkHttpResponse sdkHttpResponse; + private SdkResponse executionSuccessfulSdkResponse; + + private Builder() { + } + + private Builder(ProgressListenerContext context) { + this.sdkRequest = context.sdkRequest; + this.sdkHttpRequest = context.sdkHttpRequest; + this.uploadProgressSnapshot = context.uploadProgressSnapshot; + this.downloadProgressSnapshot = context.downloadProgressSnapshot; + this.sdkHttpResponse = context.sdkHttpResponse; + this.executionSuccessfulSdkResponse = context.executionSuccessfulSdkResponse; + } + + public Builder request(SdkRequest sdkRequest) { + this.sdkRequest = sdkRequest; + return this; + } + + public Builder httpRequest(SdkHttpRequest sdkHttpRequest) { + this.sdkHttpRequest = sdkHttpRequest; + return this; + } + + public Builder uploadProgressSnapshot(ProgressSnapshot uploadProgressSnapshot) { + this.uploadProgressSnapshot = uploadProgressSnapshot; + return this; + } + + public Builder downloadProgressSnapshot(ProgressSnapshot downloadProgressSnapshot) { + this.downloadProgressSnapshot = downloadProgressSnapshot; + return this; + } + + public Builder httpResponse(SdkHttpResponse sdkHttpResponse) { + this.sdkHttpResponse = sdkHttpResponse; + return this; + } + + public Builder response(SdkResponse executionSuccessfulSdkResponse) { + this.executionSuccessfulSdkResponse = executionSuccessfulSdkResponse; + return this; + } + + @Override + public ProgressListenerContext build() { + return new ProgressListenerContext(this); + } + } +} + diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java new file mode 100644 index 000000000000..e505a2055c82 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java @@ -0,0 +1,121 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.progress; + +import java.util.concurrent.CompletionException; +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * An SDK-internal implementation of {@link ProgressListener.Context.ExecutionFailure}. + * + * @see ProgressListenerContext + */ +@SdkInternalApi +@Immutable +public class ProgressListenerFailedContext + implements ProgressListener.Context.ExecutionFailure, + ToCopyableBuilder { + + private final ProgressListenerContext progressListenerContext; + private final Throwable exception; + + private ProgressListenerFailedContext(Builder builder) { + this.exception = unwrap(Validate.paramNotNull(builder.exception, "exception")); + this.progressListenerContext = Validate.paramNotNull(builder.progressListenerContext, "progressListenerContext"); + } + + private Throwable unwrap(Throwable exception) { + while (exception instanceof CompletionException) { + exception = exception.getCause(); + } + return exception; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + @Override + public SdkRequest request() { + return progressListenerContext.request(); + } + + @Override + public SdkHttpRequest httpRequest() { + return progressListenerContext.httpRequest(); + } + + @Override + public ProgressSnapshot uploadProgressSnapshot() { + return progressListenerContext.uploadProgressSnapshot(); + } + + @Override + public Throwable exception() { + return exception; + } + + @Override + public String toString() { + return ToString.builder("ProgressListenerFailedContext") + .add("progressListenerContext", progressListenerContext) + .add("exception", exception) + .build(); + } + + public static final class Builder implements CopyableBuilder { + private ProgressListenerContext progressListenerContext; + private Throwable exception; + + private Builder() { + } + + private Builder(ProgressListenerFailedContext failedContext) { + this.exception = failedContext.exception; + this.progressListenerContext = failedContext.progressListenerContext; + } + + public Builder exception(Throwable exception) { + this.exception = exception; + return this; + } + + public Builder progressListenerContext(ProgressListenerContext progressListenerContext) { + this.progressListenerContext = progressListenerContext; + return this; + } + + @Override + public ProgressListenerFailedContext build() { + return new ProgressListenerFailedContext(this); + } + } +} + diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java new file mode 100644 index 000000000000..05ed703ba1cf --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java @@ -0,0 +1,104 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.progress.listener; + +import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Validate; + +@SdkInternalApi +public class ProgressListenerInvoker implements ProgressListener { + + private static final Logger log = Logger.loggerFor(ProgressListener.class); + + private final List listeners; + private final AtomicBoolean prepared = new AtomicBoolean(); + private final AtomicBoolean headerSent = new AtomicBoolean(); + private final AtomicBoolean headerRecieved = new AtomicBoolean(); + private final AtomicBoolean complete = new AtomicBoolean(); + + public ProgressListenerInvoker(List listeners) { + this.listeners = Validate.paramNotNull(listeners, "listeners"); + } + + @Override + public void requestPrepared(Context.RequestPrepared context) { + if (!prepared.getAndSet(true)) { + forEach(listener -> listener.requestPrepared(context)); + } + } + + @Override + public void requestHeaderSent(Context.RequestHeaderSent context) { + if (!headerSent.getAndSet(true)) { + forEach(listener -> listener.requestHeaderSent(context)); + } + } + + @Override + public void requestBytesSent(Context.RequestBytesSent context) { + forEach(listener -> listener.requestBytesSent(context)); + } + + @Override + public void responseHeaderReceived(Context.ResponseHeaderReceived context) { + if (!headerRecieved.getAndSet(true)) { + forEach(listener -> listener.responseHeaderReceived(context)); + } + } + + @Override + public void responseBytesReceived(Context.ResponseBytesReceived context) { + forEach(listener -> listener.responseBytesReceived(context)); + } + + @Override + public void attemptFailure(Context.AttemptFailure context) { + forEach(listener -> listener.attemptFailure(context)); + } + + @Override + public void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseBytesReceived context) { + forEach(listener -> listener.attemptFailureResponseBytesReceived(context)); + } + + @Override + public void executionFailure(Context.ExecutionFailure context) { + if (!complete.getAndSet(true)) { + forEach(listener -> listener.executionFailure(context)); + } + } + + @Override + public void executionSuccess(Context.ExecutionSuccess context) { + if (!complete.getAndSet(true)) { + forEach(listener -> listener.executionSuccess(context)); + } + } + + private void forEach(Consumer action) { + for (ProgressListener listener : listeners) { + runAndLogError(log.logger(), "Exception thrown in TransferListener, ignoring", + () -> action.accept(listener)); + } + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index ee0cff4d249e..3e58fd3e7d4f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -293,7 +293,7 @@ private Context() { * Available context attributes: *

      *
    1. {@link RequestPrepared#request()}
      2. - *
    2. {@link RequestPrepared#progressSnapshot()}
      3. + *
    3. {@link RequestPrepared#uploadProgressSnapshot()}
      4. *
    */ @Immutable From ba7b059aaa2d8a4a9454ac24b1d8ec59c8ede9a9 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Wed, 3 Jan 2024 01:17:21 -0800 Subject: [PATCH 21/50] Address PR comments --- .../core/RequestOverrideConfiguration.java | 105 ++++++++++-------- .../progress/ProgressListenerContext.java | 68 ++++++------ .../ProgressListenerFailedContext.java | 3 +- .../listener/DefaultSdkRequestProgress.java | 2 +- .../listener/ProgressListenerInvoker.java | 12 +- .../progress/listener/ProgressListener.java | 4 +- .../DefaultProgressSnapshotTest.java | 2 +- 7 files changed, 109 insertions(+), 87 deletions(-) rename core/sdk-core/src/test/java/software/amazon/awssdk/core/{progress => internal/progress/snapshot}/DefaultProgressSnapshotTest.java (99%) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java index f437c0dde2ea..ff68e9b338a4 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java @@ -56,6 +56,7 @@ public abstract class RequestOverrideConfiguration { private final EndpointProvider endpointProvider; private final CompressionConfiguration compressionConfiguration; private final List plugins; + // TODO : need to discuss this in SurfaceAPI review : defining List vs ListenerChain private final List progressListeners; protected RequestOverrideConfiguration(Builder builder) { @@ -106,9 +107,9 @@ public List apiNames() { * unmarshalling, etc. This value should always be positive, if present. * *

    The api call timeout feature doesn't have strict guarantees on how quickly a request is aborted when the - * timeout is breached. The typical case aborts the request within a few milliseconds but there may occasionally be requests - * that don't get aborted until several seconds after the timer has been breached. Because of this, the client execution - * timeout feature should not be used when absolute precision is needed. + * timeout is breached. The typical case aborts the request within a few milliseconds but there may occasionally be + * requests that don't get aborted until several seconds after the timer has been breached. Because of this, the client + * execution timeout feature should not be used when absolute precision is needed. * *

    This may be used together with {@link #apiCallAttemptTimeout()} to enforce both a timeout on each individual HTTP * request (i.e. each retry) and the total time spent on all requests across retries (i.e. the 'api call' time). @@ -124,12 +125,13 @@ public Optional apiCallTimeout() { * positive, if present. * *

    The request timeout feature doesn't have strict guarantees on how quickly a request is aborted when the timeout is - * breached. The typical case aborts the request within a few milliseconds but there may occasionally be requests that don't - * get aborted until several seconds after the timer has been breached. Because of this, the request timeout feature should - * not be used when absolute precision is needed. + * breached. The typical case aborts the request within a few milliseconds but there may occasionally be requests that + * don't get aborted until several seconds after the timer has been breached. Because of this, the request timeout + * feature should not be used when absolute precision is needed. * *

    This may be used together with {@link #apiCallTimeout()} to enforce both a timeout on each individual HTTP - * request (i.e. each retry) and the total time spent on all requests across retries (i.e. the 'api call' time). + * request + * (i.e. each retry) and the total time spent on all requests across retries (i.e. the 'api call' time). * * @see Builder#apiCallAttemptTimeout(Duration) */ @@ -138,16 +140,16 @@ public Optional apiCallAttemptTimeout() { } /** - * @return the signer for signing the request. This signer get priority over the signer set on the client while signing the - * requests. If this value is not set, then the client level signer is used for signing the request. + * @return the signer for signing the request. This signer get priority over the signer set on the client while + * signing the requests. If this value is not set, then the client level signer is used for signing the request. */ public Optional signer() { return Optional.ofNullable(signer); } /** - * Return the metric publishers for publishing the metrics collected for this request. This list supersedes the metric - * publishers set on the client. + * Return the metric publishers for publishing the metrics collected for this request. This list supersedes the + * metric publishers set on the client. */ public List metricPublishers() { return metricPublishers; @@ -162,32 +164,35 @@ public List plugins() { } /** - * Return the progress listeners that will be used to listen in and track request progress + * Return a list of progress listeners that will be used to track request progress. */ + @SdkPreviewApi public List progressListeners() { return progressListeners; } /** - * Returns the additional execution attributes to be added to this request. This collection of attributes is added in addition - * to the attributes set on the client. An attribute value added on the client within the collection of attributes is - * superseded by an attribute value added on the request. + * Returns the additional execution attributes to be added to this request. + * This collection of attributes is added in addition to the attributes set on the client. + * An attribute value added on the client within the collection of attributes is superseded by an + * attribute value added on the request. */ public ExecutionAttributes executionAttributes() { return executionAttributes; } /** - * Returns the endpoint provider for resolving the endpoint for this request. This supersedes the endpoint provider set on the - * client. + * Returns the endpoint provider for resolving the endpoint for this request. This supersedes the + * endpoint provider set on the client. */ public Optional endpointProvider() { return Optional.ofNullable(endpointProvider); } /** - * Returns the compression configuration object, if present, which includes options to enable/disable compression and set the - * minimum compression threshold. This compression config object supersedes the compression config object set on the client. + * Returns the compression configuration object, if present, which includes options to enable/disable compression and set + * the minimum compression threshold. This compression config object supersedes the compression config object set on the + * client. */ public Optional compressionConfiguration() { return Optional.ofNullable(compressionConfiguration); @@ -252,13 +257,14 @@ public interface Builder { /** * Add a single header to be set on the HTTP request. *

    - * This overrides any values for the given header set on the request by default by the SDK, as well as header overrides - * set at the client level using {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. + * This overrides any values for the given header set on the request by default by the SDK, as well as header + * overrides set at the client level using + * {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. * *

    * This overrides any values already configured with this header name in the builder. * - * @param name The name of the header. + * @param name The name of the header. * @param value The value of the header. * @return This object for method chaining. */ @@ -270,13 +276,14 @@ default B putHeader(String name, String value) { /** * Add a single header with multiple values to be set on the HTTP request. *

    - * This overrides any values for the given header set on the request by default by the SDK, as well as header overrides - * set at the client level using {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. + * This overrides any values for the given header set on the request by default by the SDK, as well as header + * overrides set at the client level using + * {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. * *

    * This overrides any values already configured with this header name in the builder. * - * @param name The name of the header. + * @param name The name of the header. * @param values The values of the header. * @return This object for method chaining. */ @@ -285,8 +292,9 @@ default B putHeader(String name, String value) { /** * Add additional headers to be set on the HTTP request. *

    - * This overrides any values for the given headers set on the request by default by the SDK, as well as header overrides - * set at the client level using {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. + * This overrides any values for the given headers set on the request by default by the SDK, as well as header + * overrides set at the client level using + * {@link software.amazon.awssdk.core.client.config.ClientOverrideConfiguration}. * *

    * This completely overrides any values currently configured in the builder. @@ -309,7 +317,7 @@ default B putHeader(String name, String value) { *

    * This overrides any values already configured with this query name in the builder. * - * @param name The query parameter name. + * @param name The query parameter name. * @param value The query parameter value. * @return This object for method chaining. */ @@ -324,7 +332,7 @@ default B putRawQueryParameter(String name, String value) { *

    * This overrides any values already configured with this query name in the builder. * - * @param name The query parameter name. + * @param name The query parameter name. * @param values The query parameter values. * @return This object for method chaining. */ @@ -352,6 +360,7 @@ default B putRawQueryParameter(String name, String value) { * Set the optional name of the higher level library that constructed the request. * * @param apiName The name of the library. + * * @return This object for method chaining. */ B addApiName(ApiName apiName); @@ -360,6 +369,7 @@ default B putRawQueryParameter(String name, String value) { * Set the optional name of the higher level library that constructed the request. * * @param apiNameConsumer A {@link Consumer} that accepts a {@link ApiName.Builder}. + * * @return This object for method chaining. */ B addApiName(Consumer apiNameConsumer); @@ -393,8 +403,8 @@ default B putRawQueryParameter(String name, String value) { * *

    The request timeout feature doesn't have strict guarantees on how quickly a request is aborted when the timeout is * breached. The typical case aborts the request within a few milliseconds but there may occasionally be requests that - * don't get aborted until several seconds after the timer has been breached. Because of this, the request timeout feature - * should not be used when absolute precision is needed. + * don't get aborted until several seconds after the timer has been breached. Because of this, the request timeout + * feature should not be used when absolute precision is needed. * *

    This may be used together with {@link #apiCallTimeout()} to enforce both a timeout on each individual HTTP * request (i.e. each retry) and the total time spent on all requests across retries (i.e. the 'api call' time). @@ -421,8 +431,8 @@ default B putRawQueryParameter(String name, String value) { Signer signer(); /** - * Sets the metric publishers for publishing the metrics collected for this request. This list supersedes the metric - * publisher set on the client. + * Sets the metric publishers for publishing the metrics collected for this request. This list supersedes + * the metric publisher set on the client. * * @param metricPublisher The list metric publisher for this request. * @return This object for method chaining. @@ -430,8 +440,8 @@ default B putRawQueryParameter(String name, String value) { B metricPublishers(List metricPublisher); /** - * Add a metric publisher to the existing list of previously set publishers to be used for publishing metrics for this - * request. + * Add a metric publisher to the existing list of previously set publishers to be used for publishing metrics + * for this request. * * @param metricPublisher The metric publisher to add. */ @@ -441,7 +451,6 @@ default B putRawQueryParameter(String name, String value) { /** * Sets the additional execution attributes collection for this request. - * * @param executionAttributes Execution attributes for this request * @return This object for method chaining. */ @@ -449,18 +458,17 @@ default B putRawQueryParameter(String name, String value) { /** * Add an execution attribute to the existing collection of execution attributes. - * * @param attribute The execution attribute object - * @param value The value of the execution attribute. + * @param value The value of the execution attribute. */ B putExecutionAttribute(ExecutionAttribute attribute, T value); ExecutionAttributes executionAttributes(); /** - * Sets the endpointProvider to use for resolving the endpoint of the request. This endpointProvider gets priority over - * the endpointProvider set on the client while resolving the endpoint for the requests. If this value is null, then the - * client level endpointProvider is used for resolving the endpoint. + * Sets the endpointProvider to use for resolving the endpoint of the request. This endpointProvider gets priority + * over the endpointProvider set on the client while resolving the endpoint for the requests. + * If this value is null, then the client level endpointProvider is used for resolving the endpoint. * * @param endpointProvider Endpoint Provider that will override the resolving the endpoint for the request. * @return This object for method chaining @@ -470,18 +478,19 @@ default B putRawQueryParameter(String name, String value) { EndpointProvider endpointProvider(); /** - * Sets the {@link CompressionConfiguration} for this request. The order of precedence, from highest to lowest, for this - * setting is: 1) Per request configuration 2) Client configuration 3) Environment variables 4) Profile setting. + * Sets the {@link CompressionConfiguration} for this request. The order of precedence, from highest to lowest, + * for this setting is: 1) Per request configuration 2) Client configuration 3) Environment variables 4) Profile setting. * * @param compressionConfiguration Request compression configuration object for this request. */ B compressionConfiguration(CompressionConfiguration compressionConfiguration); /** - * Sets the {@link CompressionConfiguration} for this request. The order of precedence, from highest to lowest, for this - * setting is: 1) Per request configuration 2) Client configuration 3) Environment variables 4) Profile setting. + * Sets the {@link CompressionConfiguration} for this request. The order of precedence, from highest to lowest, + * for this setting is: 1) Per request configuration 2) Client configuration 3) Environment variables 4) Profile setting. * * @param compressionConfigurationConsumer A {@link Consumer} that accepts a {@link CompressionConfiguration.Builder} + * * @return This object for method chaining */ B compressionConfiguration(Consumer compressionConfigurationConsumer); @@ -512,11 +521,13 @@ default B putRawQueryParameter(String name, String value) { List plugins(); /** - * Sets the progress listeners used to perform request prorgess tracking + * Assigns a list of progress listeners used to perform request progress tracking + * Refer to {@link ProgressListener} * * @param progressListeners The list of plugins for this request. * @return This object for method chaining. */ + @SdkPreviewApi B progressListeners(List progressListeners); /** @@ -524,11 +535,13 @@ default B putRawQueryParameter(String name, String value) { * * @param progressListener The listener to add. */ + @SdkPreviewApi B addProgressListener(ProgressListener progressListener); /** * Returns the list of progress listeners attached to the request */ + @SdkPreviewApi List progressListeners(); /** diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java index da905b69e95b..86af01d93395 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java @@ -16,7 +16,7 @@ package software.amazon.awssdk.core.internal.progress; import software.amazon.awssdk.annotations.Immutable; -import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.progress.listener.ProgressListener; @@ -29,29 +29,29 @@ /** * An SDK-internal implementation of {@link ProgressListener.Context.ExecutionSuccess} and its parent interfaces. - * - * @see ProgressListenerFailedContext + * An instance of this class can be used by ProgressListener methods to capture and store request progress + * @see ProgressListenerFailedContext for failed transaction context definitions */ -@SdkProtectedApi +@SdkInternalApi @Immutable public final class ProgressListenerContext implements ProgressListener.Context.ExecutionSuccess, ToCopyableBuilder { - private final SdkRequest sdkRequest; - private final SdkHttpRequest sdkHttpRequest; + private final SdkRequest request; + private final SdkHttpRequest httpRequest; private final ProgressSnapshot uploadProgressSnapshot; private final ProgressSnapshot downloadProgressSnapshot; - private final SdkHttpResponse sdkHttpResponse; - private final SdkResponse executionSuccessfulSdkResponse; + private final SdkHttpResponse httpResponse; + private final SdkResponse response; private ProgressListenerContext(Builder builder) { - this.sdkRequest = builder.sdkRequest; - this.sdkHttpRequest = builder.sdkHttpRequest; + this.request = builder.request; + this.httpRequest = builder.httpRequest; this.uploadProgressSnapshot = builder.uploadProgressSnapshot; this.downloadProgressSnapshot = builder.downloadProgressSnapshot; - this.sdkHttpResponse = builder.sdkHttpResponse; - this.executionSuccessfulSdkResponse = builder.executionSuccessfulSdkResponse; + this.httpResponse = builder.httpResponse; + this.response = builder.response; } public static Builder builder() { @@ -65,12 +65,12 @@ public Builder toBuilder() { @Override public SdkRequest request() { - return sdkRequest; + return request; } @Override public SdkHttpRequest httpRequest() { - return sdkHttpRequest; + return httpRequest; } @Override @@ -85,51 +85,51 @@ public ProgressSnapshot downloadProgressSnapshot() { @Override public SdkHttpResponse httpResponse() { - return sdkHttpResponse; + return httpResponse; } @Override public SdkResponse response() { - return executionSuccessfulSdkResponse; + return response; } @Override public String toString() { return ToString.builder("ProgressListenerContext") - .add("sdkRequest", sdkRequest) + .add("request", request) .add("uploadProgressSnapshot", uploadProgressSnapshot) .add("downloadProgressSnapshot", downloadProgressSnapshot) - .add("executionSuccessfulSdkResponse", executionSuccessfulSdkResponse) + .add("response", response) .build(); } public static final class Builder implements CopyableBuilder { - private SdkRequest sdkRequest; - private SdkHttpRequest sdkHttpRequest; + private SdkRequest request; + private SdkHttpRequest httpRequest; private ProgressSnapshot uploadProgressSnapshot; private ProgressSnapshot downloadProgressSnapshot; - private SdkHttpResponse sdkHttpResponse; - private SdkResponse executionSuccessfulSdkResponse; + private SdkHttpResponse httpResponse; + private SdkResponse response; private Builder() { } private Builder(ProgressListenerContext context) { - this.sdkRequest = context.sdkRequest; - this.sdkHttpRequest = context.sdkHttpRequest; + this.request = context.request; + this.httpRequest = context.httpRequest; this.uploadProgressSnapshot = context.uploadProgressSnapshot; this.downloadProgressSnapshot = context.downloadProgressSnapshot; - this.sdkHttpResponse = context.sdkHttpResponse; - this.executionSuccessfulSdkResponse = context.executionSuccessfulSdkResponse; + this.httpResponse = context.httpResponse; + this.response = context.response; } - public Builder request(SdkRequest sdkRequest) { - this.sdkRequest = sdkRequest; + public Builder request(SdkRequest request) { + this.request = request; return this; } - public Builder httpRequest(SdkHttpRequest sdkHttpRequest) { - this.sdkHttpRequest = sdkHttpRequest; + public Builder httpRequest(SdkHttpRequest httpRequest) { + this.httpRequest = httpRequest; return this; } @@ -143,13 +143,13 @@ public Builder downloadProgressSnapshot(ProgressSnapshot downloadProgressSnapsho return this; } - public Builder httpResponse(SdkHttpResponse sdkHttpResponse) { - this.sdkHttpResponse = sdkHttpResponse; + public Builder httpResponse(SdkHttpResponse httpResponse) { + this.httpResponse = httpResponse; return this; } - public Builder response(SdkResponse executionSuccessfulSdkResponse) { - this.executionSuccessfulSdkResponse = executionSuccessfulSdkResponse; + public Builder response(SdkResponse response) { + this.response = response; return this; } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java index e505a2055c82..6b277cc470c3 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java @@ -29,8 +29,9 @@ /** * An SDK-internal implementation of {@link ProgressListener.Context.ExecutionFailure}. + * An instance of this class can be used by ProgressListener methods to capture and store failed request * - * @see ProgressListenerContext + * @see ProgressListenerContext for a successful request progress state capturing */ @SdkInternalApi @Immutable diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java index 019c869ae060..49724fc16d0f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java @@ -27,7 +27,7 @@ /** * An SDK-internal implementation of {@link SdkRequestProgress}. This implementation acts as a thin wrapper around {@link * AtomicReference}, where calls to get the latest {@link #progressSnapshot()} simply return the latest reference, while {@link - * SdkRequestProgressUpdater} is responsible for continuously updating the latest reference. + * ProgressUpdater} is responsible for continuously updating the latest reference. * * @see SdkRequestProgress */ diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java index 05ed703ba1cf..ee6f35db8d2e 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java @@ -21,19 +21,25 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.ThreadSafe; import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.Validate; +/** + * ProgressListenerInvoker class exposes thread safe methods that facilitate invocation of all ProgressListener methods + * corresponding to different stages of the request lifecycle + */ @SdkInternalApi +@ThreadSafe public class ProgressListenerInvoker implements ProgressListener { - private static final Logger log = Logger.loggerFor(ProgressListener.class); + private static final Logger log = Logger.loggerFor(ProgressListenerInvoker.class); private final List listeners; private final AtomicBoolean prepared = new AtomicBoolean(); private final AtomicBoolean headerSent = new AtomicBoolean(); - private final AtomicBoolean headerRecieved = new AtomicBoolean(); + private final AtomicBoolean headerReceived = new AtomicBoolean(); private final AtomicBoolean complete = new AtomicBoolean(); public ProgressListenerInvoker(List listeners) { @@ -61,7 +67,7 @@ public void requestBytesSent(Context.RequestBytesSent context) { @Override public void responseHeaderReceived(Context.ResponseHeaderReceived context) { - if (!headerRecieved.getAndSet(true)) { + if (!headerReceived.getAndSet(true)) { forEach(listener -> listener.responseHeaderReceived(context)); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index 3e58fd3e7d4f..3cdfa494226b 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -154,6 +154,7 @@ default void requestBytesSent(Context.RequestBytesSent context) { /** * The service returns the response headers + * SdkHttpResponse httpResponse() denotes the unmarshalled response sent back by the service * After this, one among responseBytesReceived, attemptFailureResponseBytesReceived and attemptFailure will always be invoked *

    * Available context attributes: @@ -195,7 +196,7 @@ default void responseBytesReceived(Context.ResponseBytesReceived context) { *

  • {@link Context.AttemptFailureResponseBytesReceived#request()}
    • *
  • {@link Context.AttemptFailureResponseBytesReceived#httpRequest()}
    • *
  • {@link Context.AttemptFailureResponseBytesReceived#uploadProgressSnapshot()}
    • - *
  • {@link Context.AttemptFailureResponseBytesReceived#httpResponse()} ()}
    • + *
  • {@link Context.AttemptFailureResponseBytesReceived#httpResponse()}
    • *
  • {@link Context.AttemptFailureResponseBytesReceived#downloadProgressSnapshot()}
    • *
  • {@link Context.AttemptFailureResponseBytesReceived#exception()}
    • * @@ -205,6 +206,7 @@ default void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseB /** * Successful request execution + * SdkResponse response() denotes the marshalled response * This marks the end of the request path. *

    * Available context attributes: diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java similarity index 99% rename from core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java rename to core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java index 760e7a0c3c90..49acebb1da4f 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/progress/DefaultProgressSnapshotTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.progress; +package software.amazon.awssdk.core.internal.progress.snapshot; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; From dbdd27d7fb36babcf883ec64c43c93e7c086d55d Mon Sep 17 00:00:00 2001 From: Krishnan Date: Thu, 4 Jan 2024 13:47:13 -0800 Subject: [PATCH 22/50] Remove SDK Preview API annotations --- .../amazon/awssdk/core/RequestOverrideConfiguration.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java index ff68e9b338a4..3bdebad70a42 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java @@ -56,7 +56,6 @@ public abstract class RequestOverrideConfiguration { private final EndpointProvider endpointProvider; private final CompressionConfiguration compressionConfiguration; private final List plugins; - // TODO : need to discuss this in SurfaceAPI review : defining List vs ListenerChain private final List progressListeners; protected RequestOverrideConfiguration(Builder builder) { @@ -166,7 +165,6 @@ public List plugins() { /** * Return a list of progress listeners that will be used to track request progress. */ - @SdkPreviewApi public List progressListeners() { return progressListeners; } @@ -524,10 +522,8 @@ default B putRawQueryParameter(String name, String value) { * Assigns a list of progress listeners used to perform request progress tracking * Refer to {@link ProgressListener} * - * @param progressListeners The list of plugins for this request. - * @return This object for method chaining. + * @param progressListeners a list of Progress Listeners to be associated with the request. */ - @SdkPreviewApi B progressListeners(List progressListeners); /** @@ -535,13 +531,11 @@ default B putRawQueryParameter(String name, String value) { * * @param progressListener The listener to add. */ - @SdkPreviewApi B addProgressListener(ProgressListener progressListener); /** * Returns the list of progress listeners attached to the request */ - @SdkPreviewApi List progressListeners(); /** From 29259dff36259447db2ffa7be5b01ec3a5c4b003 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Thu, 4 Jan 2024 15:26:42 -0800 Subject: [PATCH 23/50] Fixed a typo with error logging --- .../internal/progress/listener/ProgressListenerInvoker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java index ee6f35db8d2e..0caa17930b07 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java @@ -103,7 +103,7 @@ public void executionSuccess(Context.ExecutionSuccess context) { private void forEach(Consumer action) { for (ProgressListener listener : listeners) { - runAndLogError(log.logger(), "Exception thrown in TransferListener, ignoring", + runAndLogError(log.logger(), "Exception thrown in ProgressListener, ignoring", () -> action.accept(listener)); } } From 16e06b71622fef1c7e62aa7ff0515022fd86053c Mon Sep 17 00:00:00 2001 From: Krishnan Date: Tue, 9 Jan 2024 14:39:09 -0800 Subject: [PATCH 24/50] Rectify typo in adding listeners to RequestOverrideConfiguration and add tests --- .../core/RequestOverrideConfiguration.java | 2 +- .../RequestOverrideConfigurationTest.java | 29 +++++++++++++++++++ .../TestAwsRequestOverrideConfiguration.java | 2 ++ .../amazon/awssdk/core/TestSdkRequest.java | 2 ++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/TestAwsRequestOverrideConfiguration.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/TestSdkRequest.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java index 3bdebad70a42..23d129bd597d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java @@ -70,7 +70,7 @@ protected RequestOverrideConfiguration(Builder builder) { this.endpointProvider = builder.endpointProvider(); this.compressionConfiguration = builder.compressionConfiguration(); this.plugins = Collections.unmodifiableList(new ArrayList<>(builder.plugins())); - this.progressListeners = Collections.unmodifiableList(new ArrayList<>()); + this.progressListeners = Collections.unmodifiableList(new ArrayList<>(builder.progressListeners())); } /** diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/RequestOverrideConfigurationTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/RequestOverrideConfigurationTest.java index 736c43ac476a..72248f46ed54 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/RequestOverrideConfigurationTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/RequestOverrideConfigurationTest.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.interceptor.ExecutionAttribute; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.core.signer.Signer; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.metrics.MetricPublisher; @@ -330,6 +331,34 @@ public void testConfigurationEquals() { assertThat(request1Override).isNotEqualTo(null); } + @Test + public void addProgressListenersList() { + ProgressListener listener1 = mock(ProgressListener.class); + ProgressListener listener2 = mock(ProgressListener.class); + List listProgressListener = Arrays.asList(listener1, listener2); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(listProgressListener); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + assertThat(overrideConfig.progressListeners()).isEqualTo(listProgressListener); + } + + @Test + public void addProgressListeners() { + ProgressListener listener1 = mock(ProgressListener.class); + ProgressListener listener2 = mock(ProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.addProgressListener(listener1); + builder.addProgressListener(listener2); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + assertThat(overrideConfig.progressListeners()).isEqualTo(Arrays.asList(listener1, listener2)); + } + private static class NoOpSigner implements Signer { @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestAwsRequestOverrideConfiguration.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestAwsRequestOverrideConfiguration.java new file mode 100644 index 000000000000..10ec5fc9ea81 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestAwsRequestOverrideConfiguration.java @@ -0,0 +1,2 @@ +package software.amazon.awssdk.core;public class TestAwsRequestOverrideConfiguration { +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestSdkRequest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestSdkRequest.java new file mode 100644 index 000000000000..570b3ba11f71 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestSdkRequest.java @@ -0,0 +1,2 @@ +package software.amazon.awssdk.core;public class TestSdkRequest { +} From 54caa47a69005f6dab043321a4bc4f35b28910d2 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 29 Jan 2024 15:45:21 -0800 Subject: [PATCH 25/50] Added Progress Updater Class to invoke listener methods --- .../progress/listener/ProgressUpdater.java | 243 +++++++++++++ .../snapshot/DefaultProgressSnapshot.java | 1 - .../awssdk/core/protocol/VoidSdkResponse.java | 4 +- .../TestAwsRequestOverrideConfiguration.java | 2 - .../amazon/awssdk/core/TestSdkRequest.java | 2 - .../awssdk/core/http/NoopTestRequest.java | 1 + .../listener/CaptureProgressListener.java | 105 ++++++ .../listener/ProgressUpdaterTest.java | 341 ++++++++++++++++++ .../snapshot/DefaultProgressSnapshotTest.java | 4 - 9 files changed, 692 insertions(+), 11 deletions(-) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java delete mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/TestAwsRequestOverrideConfiguration.java delete mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/TestSdkRequest.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java new file mode 100644 index 000000000000..3ac8c9677a9b --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java @@ -0,0 +1,243 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.progress.listener; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import org.reactivestreams.Subscriber; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.listener.AsyncRequestBodyListener; +import software.amazon.awssdk.core.internal.progress.ProgressListenerContext; +import software.amazon.awssdk.core.internal.progress.ProgressListenerFailedContext; +import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; +import software.amazon.awssdk.core.progress.listener.SdkRequestProgress; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; + +/** + * ProgressUpdater exposes methods that invokes listener methods to update and store request progress state + */ +@SdkInternalApi +public class ProgressUpdater { + private final DefaultSdkRequestProgress uploadProgress; + private final DefaultSdkRequestProgress downloadProgress; + private final ProgressListenerContext context; + private final ProgressListenerInvoker listenerInvoker; + private final CompletableFuture endOfStreamFuture; + + public ProgressUpdater(SdkRequest sdkRequest, + Long contentLength) { + DefaultProgressSnapshot.Builder uploadProgressSnapshotBuilder = DefaultProgressSnapshot.builder(); + uploadProgressSnapshotBuilder.transferredBytes(0L); + Optional.ofNullable(contentLength).ifPresent(uploadProgressSnapshotBuilder::totalBytes); + + ProgressSnapshot uploadProgressSnapshot = uploadProgressSnapshotBuilder.build(); + uploadProgress = new DefaultSdkRequestProgress(uploadProgressSnapshot); + + DefaultProgressSnapshot.Builder downloadProgressSnapshotBuilder = DefaultProgressSnapshot.builder(); + downloadProgressSnapshotBuilder.transferredBytes(0L); + Optional.ofNullable(contentLength).ifPresent(downloadProgressSnapshotBuilder::totalBytes); + + ProgressSnapshot downloadProgressSnapshot = downloadProgressSnapshotBuilder.build(); + downloadProgress = new DefaultSdkRequestProgress(downloadProgressSnapshot); + + context = ProgressListenerContext.builder() + .request(sdkRequest) + .uploadProgressSnapshot(uploadProgressSnapshot) + .downloadProgressSnapshot(downloadProgressSnapshot) + .build(); + + listenerInvoker = sdkRequest.overrideConfiguration().get().progressListeners() == null + ? new ProgressListenerInvoker((Collections.emptyList())) + : new ProgressListenerInvoker(sdkRequest.overrideConfiguration().get().progressListeners()); + + endOfStreamFuture = new CompletableFuture<>(); + } + + public SdkRequestProgress uploadProgress() { + return uploadProgress; + } + + public SdkRequestProgress downloadProgress() { + return downloadProgress; + } + + public AsyncRequestBody wrapUploadRequestBody(AsyncRequestBody requestBody) { + return AsyncRequestBodyListener.wrap( + requestBody, + new AsyncRequestBodyListener() { + final AtomicBoolean done = new AtomicBoolean(false); + + @Override + public void publisherSubscribe(Subscriber subscriber) { + resetBytesSent(); + } + + @Override + public void subscriberOnNext(ByteBuffer byteBuffer) { + incrementBytesSent(byteBuffer.limit()); + uploadProgress.progressSnapshot().ratioTransferred().ifPresent(ratioTransferred -> { + if (Double.compare(ratioTransferred, 1.0) == 0) { + endOfStreamFutureCompleted(); + } + }); + } + + @Override + public void subscriberOnError(Throwable t) { + attemptFailure(t); + } + + @Override + public void subscriberOnComplete() { + endOfStreamFutureCompleted(); + } + + private void endOfStreamFutureCompleted() { + if (done.compareAndSet(false, true)) { + endOfStreamFuture.complete(null); + } + } + }); + } + + public AsyncRequestBody wrapDownloadRequestBody(AsyncRequestBody requestBody) { + return AsyncRequestBodyListener.wrap( + requestBody, + new AsyncRequestBodyListener() { + final AtomicBoolean done = new AtomicBoolean(false); + + @Override + public void publisherSubscribe(Subscriber subscriber) { + resetBytesReceived(); + } + + @Override + public void subscriberOnNext(ByteBuffer byteBuffer) { + incrementBytesReceived(byteBuffer.limit()); + downloadProgress.progressSnapshot().ratioTransferred().ifPresent(ratioTransferred -> { + if (Double.compare(ratioTransferred, 1.0) == 0) { + endOfStreamFutureCompleted(); + } + }); + } + + @Override + public void subscriberOnError(Throwable t) { + attemptFailure(t); + } + + @Override + public void subscriberOnComplete() { + endOfStreamFutureCompleted(); + } + + private void endOfStreamFutureCompleted() { + if (done.compareAndSet(false, true)) { + endOfStreamFuture.complete(null); + } + } + }); + } + + public void requestPrepared() { + listenerInvoker.requestPrepared(context); + } + + public void requestHeaderSent() { + listenerInvoker.requestHeaderSent(context); + } + + public void resetBytesSent() { + uploadProgress.updateAndGet(b -> b.transferredBytes(0L)); + } + + public void resetBytesReceived() { + downloadProgress.updateAndGet(b -> b.transferredBytes(0L)); + } + + public void incrementBytesSent(long numBytes) { + long uploadBytes = uploadProgress.progressSnapshot().transferredBytes(); + + ProgressSnapshot snapshot = uploadProgress.updateAndGet(b -> b.transferredBytes(uploadBytes + numBytes)); + listenerInvoker.requestBytesSent(context.copy(b -> b.uploadProgressSnapshot(snapshot))); + } + + public void incrementBytesReceived(long numBytes) { + long downloadedBytes = downloadProgress.progressSnapshot().transferredBytes(); + + ProgressSnapshot snapshot = downloadProgress.updateAndGet(b -> b.transferredBytes(downloadedBytes + numBytes)); + listenerInvoker.responseBytesReceived(context.copy(b -> b.downloadProgressSnapshot(snapshot))); + } + + public void registerCompletion(CompletableFuture future) { + future.whenComplete((r, t) -> { + if (t == null) { + endOfStreamFuture.whenComplete((r2, t2) -> { + if (t2 == null) { + executionSuccess(r); + } else { + attemptFailure(t2); + } + }); + } else { + executionFailure(t); + } + }); + } + + public void responseHeaderReceived() { + listenerInvoker.responseHeaderReceived(context); + } + + public void executionSuccess(SdkResponse response) { + + listenerInvoker.executionSuccess(context.copy(b -> b.response(response))); + } + + private void executionFailure(Throwable t) { + listenerInvoker.executionFailure(ProgressListenerFailedContext.builder() + .progressListenerContext( + context.copy( + b -> { + b.uploadProgressSnapshot( + uploadProgress.progressSnapshot()); + b.downloadProgressSnapshot( + downloadProgress.progressSnapshot()); + })) + .exception(t) + .build()); + } + + private void attemptFailure(Throwable t) { + listenerInvoker.executionFailure(ProgressListenerFailedContext.builder() + .progressListenerContext( + context.copy( + b -> { + b.uploadProgressSnapshot( + uploadProgress.progressSnapshot()); + b.downloadProgressSnapshot( + downloadProgress.progressSnapshot()); + })) + .exception(t) + .build()); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshot.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshot.java index 86e9a64004d2..be95d5232b86 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshot.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshot.java @@ -52,7 +52,6 @@ public DefaultProgressSnapshot(Builder builder) { this.transferredBytes = Validate.isNotNegative(builder.transferredBytes, "transferredBytes"); this.totalBytes = builder.totalBytes; - Validate.paramNotNull(builder.startTime, "startTime"); this.startTime = builder.startTime; } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/protocol/VoidSdkResponse.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/protocol/VoidSdkResponse.java index 4773b9443706..d8b688b33bec 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/protocol/VoidSdkResponse.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/protocol/VoidSdkResponse.java @@ -31,7 +31,7 @@ public final class VoidSdkResponse extends SdkResponse { private static final List> SDK_FIELDS = Collections.unmodifiableList(Collections.emptyList()); - private VoidSdkResponse(Builder builder) { + public VoidSdkResponse(Builder builder) { super(builder); } @@ -51,7 +51,7 @@ public List> sdkFields() { public static final class Builder extends BuilderImpl implements SdkPojo, SdkBuilder { - private Builder() { + public Builder() { } @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestAwsRequestOverrideConfiguration.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestAwsRequestOverrideConfiguration.java deleted file mode 100644 index 10ec5fc9ea81..000000000000 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestAwsRequestOverrideConfiguration.java +++ /dev/null @@ -1,2 +0,0 @@ -package software.amazon.awssdk.core;public class TestAwsRequestOverrideConfiguration { -} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestSdkRequest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestSdkRequest.java deleted file mode 100644 index 570b3ba11f71..000000000000 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/TestSdkRequest.java +++ /dev/null @@ -1,2 +0,0 @@ -package software.amazon.awssdk.core;public class TestSdkRequest { -} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopTestRequest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopTestRequest.java index 3f2251a2626f..b9bf46a7d424 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopTestRequest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopTestRequest.java @@ -18,6 +18,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.function.Consumer; import software.amazon.awssdk.core.SdkField; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java new file mode 100644 index 000000000000..1f90ed3b6c02 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java @@ -0,0 +1,105 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.progress.listener; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.core.progress.listener.ProgressListener; + +public class CaptureProgressListener implements ProgressListener { + public Boolean isRequestPrepared() { + return requestPrepared; + } + + public Boolean isRequestHeaderSent() { + return requestHeaderSent; + } + + public Boolean isResponseHeaderReceived() { + return responseHeaderReceived; + } + + public Boolean isExecutionSuccess() { + return executionSuccess; + } + + public List getRatioTransferredList() { + return Collections.unmodifiableList(ratioTransferredList); + } + + public CompletableFuture getCompletionFuture() { + return completionFuture; + } + + public Throwable getExceptionCaught() { + return exceptionCaught; + } + + private Boolean requestPrepared = false; + private Boolean requestHeaderSent = false; + private Boolean responseHeaderReceived = false; + private Boolean executionSuccess = false; + CompletableFuture completionFuture = new CompletableFuture<>(); + + private final List ratioTransferredList = new ArrayList<>(); + private Throwable exceptionCaught; + + @Override + public void requestPrepared(Context.RequestPrepared context) { + requestPrepared = true; + context.uploadProgressSnapshot().ratioTransferred().ifPresent(ratioTransferredList::add); + + } + + @Override + public void requestHeaderSent(Context.RequestHeaderSent context) { + requestHeaderSent = true; + context.uploadProgressSnapshot().ratioTransferred().ifPresent(ratioTransferredList::add); + } + + @Override + public void requestBytesSent(Context.RequestBytesSent context) { + context.uploadProgressSnapshot().ratioTransferred().ifPresent(ratioTransferredList::add); + } + + @Override + public void responseHeaderReceived(Context.ResponseHeaderReceived context) { + responseHeaderReceived = true; + context.uploadProgressSnapshot().ratioTransferred().ifPresent(ratioTransferredList::add); + } + + @Override + public void responseBytesReceived(Context.ResponseBytesReceived context) { + context.uploadProgressSnapshot().ratioTransferred().ifPresent(ratioTransferredList::add); + } + + @Override + public void executionSuccess(Context.ExecutionSuccess context) { + context.uploadProgressSnapshot().ratioTransferred().ifPresent(ratioTransferredList::add); + executionSuccess = true; + completionFuture.complete(null); + + } + + @Override + public void executionFailure(Context.ExecutionFailure context) { + exceptionCaught = context.exception(); + completionFuture.completeExceptionally(exceptionCaught); + } +} + diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java new file mode 100644 index 000000000000..77282e9598c6 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java @@ -0,0 +1,341 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.progress.listener; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import org.reactivestreams.Subscriber; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.http.NoopTestRequest; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.core.protocol.VoidSdkResponse; +import software.amazon.awssdk.http.async.SimpleSubscriber; +import software.amazon.awssdk.testutils.RandomTempFile; + +public class ProgressUpdaterTest { + private CaptureProgressListener captureProgressListener; + + private static final long BYTES_TRANSFERRED = 5L; + + private static File sourceFile; + + private static final long OBJ_SIZE = 16 * 1024 * 1024; + + @BeforeEach + void initiate() { + captureProgressListener = new CaptureProgressListener(); + } + private static CompletableFuture completedSdkResponse(long millis) { + return CompletableFuture.supplyAsync(() -> { + quietSleep(millis); + + VoidSdkResponse.Builder builder = (VoidSdkResponse.Builder) new VoidSdkResponse.Builder().sdkHttpResponse(null); + return new VoidSdkResponse(builder); + }); + } + + private static Stream contentLength() { + return Stream.of( + Arguments.of(100L), + Arguments.of(200L), + Arguments.of(300L), + Arguments.of(400L), + Arguments.of(500L)); + } + + @Test + public void test_requestPrepared_transferredBytes_equals_zero() { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.requestPrepared(); + + assertEquals(0.0, progressUpdater.uploadProgress().progressSnapshot().transferredBytes(), 0.0); + Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, times(1)).requestPrepared(ArgumentMatchers.any(ProgressListener.Context.RequestPrepared.class)); + + + } + + @Test + public void test_requestHeaderSent_transferredBytes_equals_zero() { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.requestHeaderSent(); + + assertEquals(0.0, progressUpdater.uploadProgress().progressSnapshot().transferredBytes(), 0.0); + Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, times(1)).requestHeaderSent(ArgumentMatchers.any(ProgressListener.Context.RequestHeaderSent.class)); + + } + + @Test + public void test_requestBytesSent_transferredBytes() { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); + assertEquals(BYTES_TRANSFERRED, progressUpdater.uploadProgress().progressSnapshot().transferredBytes(), 0.0); + + progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); + assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.uploadProgress().progressSnapshot().transferredBytes(), 0.0); + + Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, times(2)).requestBytesSent(ArgumentMatchers.any(ProgressListener.Context.RequestBytesSent.class)); + + } + + @ParameterizedTest + @MethodSource("contentLength") + public void test_ratioTransferred_upload_transferredBytes(long contentLength) { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, contentLength); + progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); + assertEquals((double) BYTES_TRANSFERRED / contentLength, progressUpdater.uploadProgress().progressSnapshot().ratioTransferred().getAsDouble(), 0.0); + + } + + @Test + public void test_responseHeaderReceived_transferredBytes_equals_zero() { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.responseHeaderReceived(); + + assertEquals(0.0, progressUpdater.downloadProgress().progressSnapshot().transferredBytes(), 0.0); + Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, times(1)).responseHeaderReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseHeaderReceived.class)); + + } + + @Test + public void test_responseBytesReceived_transferredBytes() { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.incrementBytesReceived(BYTES_TRANSFERRED); + assertEquals(BYTES_TRANSFERRED, progressUpdater.downloadProgress().progressSnapshot().transferredBytes(), 0.0); + + progressUpdater.incrementBytesReceived(BYTES_TRANSFERRED); + assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.downloadProgress().progressSnapshot().transferredBytes(), 0.0); + + Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, times(2)).responseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseBytesReceived.class)); + + } + + @ParameterizedTest(name = "{index} - {1}, total bytes = {0}") + @MethodSource("contentLength") + void registerCompletion_differentTransferredByteRatios_alwaysCompletesOnce(Long givenContentLength) + throws Exception { + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + sourceFile = new RandomTempFile(OBJ_SIZE); + AsyncRequestBody requestBody = AsyncRequestBody.fromFile(sourceFile); + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, givenContentLength); + AsyncRequestBody asyncRequestBody = progressUpdater.wrapUploadRequestBody(requestBody); + + CompletableFuture completionFuture = completedSdkResponse(10); + progressUpdater.registerCompletion(completionFuture); + + AtomicReference publishedBuffer = new AtomicReference<>(); + Subscriber subscriber = new SimpleSubscriber(publishedBuffer::set); + asyncRequestBody.subscribe(subscriber); + + captureProgressListener.getCompletionFuture().get(5, TimeUnit.SECONDS); + + Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, times(1)).executionSuccess(ArgumentMatchers.any(ProgressListener.Context.ExecutionSuccess.class)); + } + + @Test + void executionFailure_when_SubscriptionErrors() throws Exception { + Long contentLength = 51L; + String inputString = RandomStringUtils.randomAlphanumeric(contentLength.intValue()); + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + sourceFile = new RandomTempFile(OBJ_SIZE); + AsyncRequestBody requestFileBody = AsyncRequestBody.fromInputStream( + new ExceptionThrowingByteArrayInputStream(inputString.getBytes(), 3), contentLength, + Executors.newSingleThreadExecutor()); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, contentLength); + AsyncRequestBody asyncRequestBody = progressUpdater.wrapUploadRequestBody(requestFileBody); + + CompletableFuture completionFuture = completedSdkResponse(10); + progressUpdater.registerCompletion(completionFuture); + + AtomicReference publishedBuffer = new AtomicReference<>(); + Subscriber subscriber = new SimpleSubscriber(publishedBuffer::set); + asyncRequestBody.subscribe(subscriber); + + assertThatExceptionOfType(ExecutionException.class).isThrownBy( + () -> captureProgressListener.getCompletionFuture().get(5, TimeUnit.SECONDS)); + + Mockito.verify(mockListener, times(1)).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, never()).executionSuccess(ArgumentMatchers.any(ProgressListener.Context.ExecutionSuccess.class)); + } + + private static class ExceptionThrowingByteArrayInputStream extends ByteArrayInputStream { + private final int exceptionPosition; + + public ExceptionThrowingByteArrayInputStream(byte[] buf, int exceptionPosition) { + super(buf); + this.exceptionPosition = exceptionPosition; + } + + @Override + public int read() { + return (exceptionPosition == pos + 1) ? exceptionThrowingRead() : super.read(); + } + + @Override + public int read(byte[] b, int off, int len) { + return (exceptionPosition >= pos && exceptionPosition < (pos + len)) ? + exceptionThrowingReadByteArr(b, off, len) : super.read(b, off, len); + } + + private int exceptionThrowingRead() { + throw new RuntimeException("Exception occurred at position " + (pos + 1)); + } + + private int exceptionThrowingReadByteArr(byte[] b, int off, int len) { + throw new RuntimeException("Exception occurred in read(byte[]) at position " + exceptionPosition); + } + } + + private static void quietSleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); // Restore interrupted status + throw new RuntimeException(e); + } + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java index 49acebb1da4f..b2d125f656d1 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java @@ -39,10 +39,6 @@ private static Stream getArgumentsForInvalidParameterValidationTests( .transferredBytes(2L) .totalBytes(1L), new IllegalArgumentException()), - Arguments.of("startTime must not be null.", - DefaultProgressSnapshot.builder() - .transferredBytes(2L), - new NullPointerException()), Arguments.of("transferredBytes must not be negative", DefaultProgressSnapshot.builder() .transferredBytes(-2L), From 631f6744bb003c89a09a694b5fd07b30d316c011 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Wed, 31 Jan 2024 15:51:31 -0800 Subject: [PATCH 26/50] ProgressUpdater class --- ...s.java => DefaultSdkExchangeProgress.java} | 10 +- .../listener/LoggingProgressListener.java | 91 ++++++++++ .../progress/listener/ProgressUpdater.java | 140 ++++------------ ...Progress.java => SdkExchangeProgress.java} | 3 +- .../awssdk/core/protocol/VoidSdkResponse.java | 4 +- .../awssdk/core/http/NoopTestRequest.java | 1 - .../listener/CaptureProgressListener.java | 22 +-- .../listener/ProgressUpdaterTest.java | 157 +++--------------- 8 files changed, 167 insertions(+), 261 deletions(-) rename core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/{DefaultSdkRequestProgress.java => DefaultSdkExchangeProgress.java} (84%) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListener.java rename core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/{SdkRequestProgress.java => SdkExchangeProgress.java} (92%) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkExchangeProgress.java similarity index 84% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkExchangeProgress.java index 49724fc16d0f..a7af7a3ab69c 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkRequestProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/DefaultSdkExchangeProgress.java @@ -21,24 +21,24 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.annotations.ThreadSafe; import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; -import software.amazon.awssdk.core.progress.listener.SdkRequestProgress; +import software.amazon.awssdk.core.progress.listener.SdkExchangeProgress; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; /** - * An SDK-internal implementation of {@link SdkRequestProgress}. This implementation acts as a thin wrapper around {@link + * An SDK-internal implementation of {@link SdkExchangeProgress}. This implementation acts as a thin wrapper around {@link * AtomicReference}, where calls to get the latest {@link #progressSnapshot()} simply return the latest reference, while {@link * ProgressUpdater} is responsible for continuously updating the latest reference. * - * @see SdkRequestProgress + * @see SdkExchangeProgress */ @Mutable @ThreadSafe @SdkInternalApi -public class DefaultSdkRequestProgress implements SdkRequestProgress { +public class DefaultSdkExchangeProgress implements SdkExchangeProgress { private final AtomicReference snapshot; - public DefaultSdkRequestProgress(ProgressSnapshot snapshot) { + public DefaultSdkExchangeProgress(ProgressSnapshot snapshot) { this.snapshot = new AtomicReference<>(snapshot); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListener.java new file mode 100644 index 000000000000..fa47cee4caca --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListener.java @@ -0,0 +1,91 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.progress.listener; + +import static software.amazon.awssdk.utils.StringUtils.repeat; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.concurrent.atomic.AtomicInteger; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; +import software.amazon.awssdk.utils.Logger; + +/** + * An implementation of {@link ProgressListener} that logs a progress bar at the {@code INFO} level for upload operations. This implementation + * effectively limits the frequency of updates by limiting logging to events of progress advancement. By default, the + * progress bar has {@value #DEFAULT_MAX_TICKS} ticks, meaning an update is logged for every 5% progression, at most. + */ +@SdkPublicApi +public final class LoggingProgressListener implements ProgressListener { + private static final Logger log = Logger.loggerFor(LoggingProgressListener.class); + private static final int DEFAULT_MAX_TICKS = 20; + private final ProgressBar progressBar; + + private LoggingProgressListener(int maxTicks) { + progressBar = new ProgressBar(maxTicks); + } + + @Override + public void requestPrepared(Context.RequestPrepared context) { + log.info(() -> "Request Prepared... "); + context.uploadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); + } + + @Override + public void requestHeaderSent(Context.RequestHeaderSent context) { + log.info(() -> "Request Header Sent... "); + context.uploadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); + } + + @Override + public void requestBytesSent(Context.RequestBytesSent context) { + log.info(() -> "Request Bytes Sent... "); + context.uploadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); + } + + @Override + public void responseHeaderReceived(Context.ResponseHeaderReceived context) { + log.info(() -> "Response Header Received... "); + context.downloadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); + } + + private static class ProgressBar { + private final int maxTicks; + private final AtomicInteger prevTicks = new AtomicInteger(-1); + private ProgressBar(int maxTicks) { + this.maxTicks = maxTicks; + } + + void update(double ratio) { + int ticks = (int) Math.floor(ratio * maxTicks); + if (prevTicks.getAndSet(ticks) < ticks) { + log.info(() -> String.format("|%s%s| %s", + repeat("=", ticks), + repeat(" ", maxTicks - ticks), + round(ratio * 100, 1) + "%")); + } + } + + private static double round(double value, int places) { + BigDecimal bd = BigDecimal.valueOf(value); + bd = bd.setScale(places, RoundingMode.FLOOR); + return bd.doubleValue(); + } + } + +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java index 3ac8c9677a9b..0fae71ed51e8 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java @@ -19,17 +19,13 @@ import java.util.Collections; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicBoolean; -import org.reactivestreams.Subscriber; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkResponse; -import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.core.async.listener.AsyncRequestBodyListener; import software.amazon.awssdk.core.internal.progress.ProgressListenerContext; import software.amazon.awssdk.core.internal.progress.ProgressListenerFailedContext; import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; -import software.amazon.awssdk.core.progress.listener.SdkRequestProgress; +import software.amazon.awssdk.core.progress.listener.SdkExchangeProgress; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; /** @@ -37,8 +33,8 @@ */ @SdkInternalApi public class ProgressUpdater { - private final DefaultSdkRequestProgress uploadProgress; - private final DefaultSdkRequestProgress downloadProgress; + private final DefaultSdkExchangeProgress requestBodyProgress; + private final DefaultSdkExchangeProgress responseBodyProgress; private final ProgressListenerContext context; private final ProgressListenerInvoker listenerInvoker; private final CompletableFuture endOfStreamFuture; @@ -50,14 +46,14 @@ public ProgressUpdater(SdkRequest sdkRequest, Optional.ofNullable(contentLength).ifPresent(uploadProgressSnapshotBuilder::totalBytes); ProgressSnapshot uploadProgressSnapshot = uploadProgressSnapshotBuilder.build(); - uploadProgress = new DefaultSdkRequestProgress(uploadProgressSnapshot); + requestBodyProgress = new DefaultSdkExchangeProgress(uploadProgressSnapshot); DefaultProgressSnapshot.Builder downloadProgressSnapshotBuilder = DefaultProgressSnapshot.builder(); downloadProgressSnapshotBuilder.transferredBytes(0L); Optional.ofNullable(contentLength).ifPresent(downloadProgressSnapshotBuilder::totalBytes); ProgressSnapshot downloadProgressSnapshot = downloadProgressSnapshotBuilder.build(); - downloadProgress = new DefaultSdkRequestProgress(downloadProgressSnapshot); + responseBodyProgress = new DefaultSdkExchangeProgress(downloadProgressSnapshot); context = ProgressListenerContext.builder() .request(sdkRequest) @@ -65,97 +61,19 @@ public ProgressUpdater(SdkRequest sdkRequest, .downloadProgressSnapshot(downloadProgressSnapshot) .build(); - listenerInvoker = sdkRequest.overrideConfiguration().get().progressListeners() == null + listenerInvoker = (!sdkRequest.overrideConfiguration().isPresent() || sdkRequest.overrideConfiguration().get().progressListeners() == null) ? new ProgressListenerInvoker((Collections.emptyList())) : new ProgressListenerInvoker(sdkRequest.overrideConfiguration().get().progressListeners()); endOfStreamFuture = new CompletableFuture<>(); } - public SdkRequestProgress uploadProgress() { - return uploadProgress; + public SdkExchangeProgress requestBodyProgress() { + return requestBodyProgress; } - public SdkRequestProgress downloadProgress() { - return downloadProgress; - } - - public AsyncRequestBody wrapUploadRequestBody(AsyncRequestBody requestBody) { - return AsyncRequestBodyListener.wrap( - requestBody, - new AsyncRequestBodyListener() { - final AtomicBoolean done = new AtomicBoolean(false); - - @Override - public void publisherSubscribe(Subscriber subscriber) { - resetBytesSent(); - } - - @Override - public void subscriberOnNext(ByteBuffer byteBuffer) { - incrementBytesSent(byteBuffer.limit()); - uploadProgress.progressSnapshot().ratioTransferred().ifPresent(ratioTransferred -> { - if (Double.compare(ratioTransferred, 1.0) == 0) { - endOfStreamFutureCompleted(); - } - }); - } - - @Override - public void subscriberOnError(Throwable t) { - attemptFailure(t); - } - - @Override - public void subscriberOnComplete() { - endOfStreamFutureCompleted(); - } - - private void endOfStreamFutureCompleted() { - if (done.compareAndSet(false, true)) { - endOfStreamFuture.complete(null); - } - } - }); - } - - public AsyncRequestBody wrapDownloadRequestBody(AsyncRequestBody requestBody) { - return AsyncRequestBodyListener.wrap( - requestBody, - new AsyncRequestBodyListener() { - final AtomicBoolean done = new AtomicBoolean(false); - - @Override - public void publisherSubscribe(Subscriber subscriber) { - resetBytesReceived(); - } - - @Override - public void subscriberOnNext(ByteBuffer byteBuffer) { - incrementBytesReceived(byteBuffer.limit()); - downloadProgress.progressSnapshot().ratioTransferred().ifPresent(ratioTransferred -> { - if (Double.compare(ratioTransferred, 1.0) == 0) { - endOfStreamFutureCompleted(); - } - }); - } - - @Override - public void subscriberOnError(Throwable t) { - attemptFailure(t); - } - - @Override - public void subscriberOnComplete() { - endOfStreamFutureCompleted(); - } - - private void endOfStreamFutureCompleted() { - if (done.compareAndSet(false, true)) { - endOfStreamFuture.complete(null); - } - } - }); + public SdkExchangeProgress responseBodyProgress() { + return responseBodyProgress; } public void requestPrepared() { @@ -167,37 +85,31 @@ public void requestHeaderSent() { } public void resetBytesSent() { - uploadProgress.updateAndGet(b -> b.transferredBytes(0L)); + requestBodyProgress.updateAndGet(b -> b.transferredBytes(0L)); } public void resetBytesReceived() { - downloadProgress.updateAndGet(b -> b.transferredBytes(0L)); + responseBodyProgress.updateAndGet(b -> b.transferredBytes(0L)); } public void incrementBytesSent(long numBytes) { - long uploadBytes = uploadProgress.progressSnapshot().transferredBytes(); + long uploadBytes = requestBodyProgress.progressSnapshot().transferredBytes(); - ProgressSnapshot snapshot = uploadProgress.updateAndGet(b -> b.transferredBytes(uploadBytes + numBytes)); + ProgressSnapshot snapshot = requestBodyProgress.updateAndGet(b -> b.transferredBytes(uploadBytes + numBytes)); listenerInvoker.requestBytesSent(context.copy(b -> b.uploadProgressSnapshot(snapshot))); } public void incrementBytesReceived(long numBytes) { - long downloadedBytes = downloadProgress.progressSnapshot().transferredBytes(); + long downloadedBytes = responseBodyProgress.progressSnapshot().transferredBytes(); - ProgressSnapshot snapshot = downloadProgress.updateAndGet(b -> b.transferredBytes(downloadedBytes + numBytes)); + ProgressSnapshot snapshot = responseBodyProgress.updateAndGet(b -> b.transferredBytes(downloadedBytes + numBytes)); listenerInvoker.responseBytesReceived(context.copy(b -> b.downloadProgressSnapshot(snapshot))); } public void registerCompletion(CompletableFuture future) { future.whenComplete((r, t) -> { if (t == null) { - endOfStreamFuture.whenComplete((r2, t2) -> { - if (t2 == null) { - executionSuccess(r); - } else { - attemptFailure(t2); - } - }); + attachEndOfStreamFutureCallback(r); } else { executionFailure(t); } @@ -219,9 +131,9 @@ private void executionFailure(Throwable t) { context.copy( b -> { b.uploadProgressSnapshot( - uploadProgress.progressSnapshot()); + requestBodyProgress.progressSnapshot()); b.downloadProgressSnapshot( - downloadProgress.progressSnapshot()); + responseBodyProgress.progressSnapshot()); })) .exception(t) .build()); @@ -233,11 +145,21 @@ private void attemptFailure(Throwable t) { context.copy( b -> { b.uploadProgressSnapshot( - uploadProgress.progressSnapshot()); + requestBodyProgress.progressSnapshot()); b.downloadProgressSnapshot( - downloadProgress.progressSnapshot()); + responseBodyProgress.progressSnapshot()); })) .exception(t) .build()); } + + public void attachEndOfStreamFutureCallback(SdkResponse response) { + endOfStreamFuture.whenComplete((r2, t2) -> { + if (t2 == null) { + executionSuccess(response); + } else { + attemptFailure(t2); + } + }); + } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkExchangeProgress.java similarity index 92% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkExchangeProgress.java index 837b30535868..04dbce4736c1 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkRequestProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkExchangeProgress.java @@ -23,9 +23,10 @@ @Immutable @ThreadSafe @SdkPublicApi -public interface SdkRequestProgress { +public interface SdkExchangeProgress { /** + * SdkExchange Progress makes it available for * Takes a snapshot of the request execution progress * represented by an immutable {@link ProgressSnapshot}. */ diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/protocol/VoidSdkResponse.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/protocol/VoidSdkResponse.java index d8b688b33bec..4773b9443706 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/protocol/VoidSdkResponse.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/protocol/VoidSdkResponse.java @@ -31,7 +31,7 @@ public final class VoidSdkResponse extends SdkResponse { private static final List> SDK_FIELDS = Collections.unmodifiableList(Collections.emptyList()); - public VoidSdkResponse(Builder builder) { + private VoidSdkResponse(Builder builder) { super(builder); } @@ -51,7 +51,7 @@ public List> sdkFields() { public static final class Builder extends BuilderImpl implements SdkPojo, SdkBuilder { - public Builder() { + private Builder() { } @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopTestRequest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopTestRequest.java index b9bf46a7d424..3f2251a2626f 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopTestRequest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopTestRequest.java @@ -18,7 +18,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.function.Consumer; import software.amazon.awssdk.core.SdkField; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java index 1f90ed3b6c02..9861293da2ba 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java @@ -22,38 +22,38 @@ import software.amazon.awssdk.core.progress.listener.ProgressListener; public class CaptureProgressListener implements ProgressListener { - public Boolean isRequestPrepared() { + public Boolean requestPrepared() { return requestPrepared; } - public Boolean isRequestHeaderSent() { + public Boolean requestHeaderSent() { return requestHeaderSent; } - public Boolean isResponseHeaderReceived() { + public Boolean responseHeaderReceived() { return responseHeaderReceived; } - public Boolean isExecutionSuccess() { + public Boolean executionSuccess() { return executionSuccess; } - public List getRatioTransferredList() { + public List ratioTransferredList() { return Collections.unmodifiableList(ratioTransferredList); } - public CompletableFuture getCompletionFuture() { + public CompletableFuture completionFuture() { return completionFuture; } - public Throwable getExceptionCaught() { + public Throwable exceptionCaught() { return exceptionCaught; } - private Boolean requestPrepared = false; - private Boolean requestHeaderSent = false; - private Boolean responseHeaderReceived = false; - private Boolean executionSuccess = false; + private volatile boolean requestPrepared = false; + private volatile boolean requestHeaderSent = false; + private volatile boolean responseHeaderReceived = false; + private volatile boolean executionSuccess = false; CompletableFuture completionFuture = new CompletableFuture<>(); private final List ratioTransferredList = new ArrayList<>(); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java index 77282e9598c6..65c07e60a220 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java @@ -17,6 +17,8 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -54,22 +56,10 @@ public class ProgressUpdaterTest { private static final long BYTES_TRANSFERRED = 5L; - private static File sourceFile; - - private static final long OBJ_SIZE = 16 * 1024 * 1024; - @BeforeEach void initiate() { captureProgressListener = new CaptureProgressListener(); } - private static CompletableFuture completedSdkResponse(long millis) { - return CompletableFuture.supplyAsync(() -> { - quietSleep(millis); - - VoidSdkResponse.Builder builder = (VoidSdkResponse.Builder) new VoidSdkResponse.Builder().sdkHttpResponse(null); - return new VoidSdkResponse(builder); - }); - } private static Stream contentLength() { return Stream.of( @@ -81,7 +71,7 @@ private static Stream contentLength() { } @Test - public void test_requestPrepared_transferredBytes_equals_zero() { + public void requestPrepared_transferredBytes_equals_zero() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -97,7 +87,10 @@ public void test_requestPrepared_transferredBytes_equals_zero() { ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); progressUpdater.requestPrepared(); - assertEquals(0.0, progressUpdater.uploadProgress().progressSnapshot().transferredBytes(), 0.0); + assertEquals(0.0, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); + assertTrue(captureProgressListener.requestPrepared()); + assertFalse(captureProgressListener.requestHeaderSent()); + assertFalse(captureProgressListener.responseHeaderReceived()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); @@ -107,7 +100,7 @@ public void test_requestPrepared_transferredBytes_equals_zero() { } @Test - public void test_requestHeaderSent_transferredBytes_equals_zero() { + public void requestHeaderSent_transferredBytes_equals_zero() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -123,7 +116,10 @@ public void test_requestHeaderSent_transferredBytes_equals_zero() { ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); progressUpdater.requestHeaderSent(); - assertEquals(0.0, progressUpdater.uploadProgress().progressSnapshot().transferredBytes(), 0.0); + assertEquals(0.0, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); + assertFalse(captureProgressListener.requestPrepared()); + assertTrue(captureProgressListener.requestHeaderSent()); + assertFalse(captureProgressListener.responseHeaderReceived()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); @@ -132,7 +128,7 @@ public void test_requestHeaderSent_transferredBytes_equals_zero() { } @Test - public void test_requestBytesSent_transferredBytes() { + public void requestBytesSent_transferredBytes() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -147,10 +143,10 @@ public void test_requestBytesSent_transferredBytes() { ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); - assertEquals(BYTES_TRANSFERRED, progressUpdater.uploadProgress().progressSnapshot().transferredBytes(), 0.0); + assertEquals(BYTES_TRANSFERRED, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); - assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.uploadProgress().progressSnapshot().transferredBytes(), 0.0); + assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); @@ -161,7 +157,7 @@ public void test_requestBytesSent_transferredBytes() { @ParameterizedTest @MethodSource("contentLength") - public void test_ratioTransferred_upload_transferredBytes(long contentLength) { + public void ratioTransferred_upload_transferredBytes(long contentLength) { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -176,12 +172,12 @@ public void test_ratioTransferred_upload_transferredBytes(long contentLength) { ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, contentLength); progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); - assertEquals((double) BYTES_TRANSFERRED / contentLength, progressUpdater.uploadProgress().progressSnapshot().ratioTransferred().getAsDouble(), 0.0); + assertEquals((double) BYTES_TRANSFERRED / contentLength, progressUpdater.requestBodyProgress().progressSnapshot().ratioTransferred().getAsDouble(), 0.0); } @Test - public void test_responseHeaderReceived_transferredBytes_equals_zero() { + public void responseHeaderReceived_transferredBytes_equals_zero() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -197,7 +193,10 @@ public void test_responseHeaderReceived_transferredBytes_equals_zero() { ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); progressUpdater.responseHeaderReceived(); - assertEquals(0.0, progressUpdater.downloadProgress().progressSnapshot().transferredBytes(), 0.0); + assertEquals(0.0, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); + assertFalse(captureProgressListener.requestPrepared()); + assertFalse(captureProgressListener.requestHeaderSent()); + assertTrue(captureProgressListener.responseHeaderReceived()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); @@ -206,7 +205,7 @@ public void test_responseHeaderReceived_transferredBytes_equals_zero() { } @Test - public void test_responseBytesReceived_transferredBytes() { + public void responseBytesReceived_transferredBytes_valid() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -221,10 +220,10 @@ public void test_responseBytesReceived_transferredBytes() { ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); progressUpdater.incrementBytesReceived(BYTES_TRANSFERRED); - assertEquals(BYTES_TRANSFERRED, progressUpdater.downloadProgress().progressSnapshot().transferredBytes(), 0.0); + assertEquals(BYTES_TRANSFERRED, progressUpdater.responseBodyProgress().progressSnapshot().transferredBytes(), 0.0); progressUpdater.incrementBytesReceived(BYTES_TRANSFERRED); - assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.downloadProgress().progressSnapshot().transferredBytes(), 0.0); + assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.responseBodyProgress().progressSnapshot().transferredBytes(), 0.0); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); @@ -232,110 +231,4 @@ public void test_responseBytesReceived_transferredBytes() { Mockito.verify(mockListener, times(2)).responseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseBytesReceived.class)); } - - @ParameterizedTest(name = "{index} - {1}, total bytes = {0}") - @MethodSource("contentLength") - void registerCompletion_differentTransferredByteRatios_alwaysCompletesOnce(Long givenContentLength) - throws Exception { - CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); - - SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); - builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); - - SdkRequestOverrideConfiguration overrideConfig = builder.build(); - - SdkRequest sdkRequest = NoopTestRequest.builder() - .overrideConfiguration(overrideConfig) - .build(); - - sourceFile = new RandomTempFile(OBJ_SIZE); - AsyncRequestBody requestBody = AsyncRequestBody.fromFile(sourceFile); - ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, givenContentLength); - AsyncRequestBody asyncRequestBody = progressUpdater.wrapUploadRequestBody(requestBody); - - CompletableFuture completionFuture = completedSdkResponse(10); - progressUpdater.registerCompletion(completionFuture); - - AtomicReference publishedBuffer = new AtomicReference<>(); - Subscriber subscriber = new SimpleSubscriber(publishedBuffer::set); - asyncRequestBody.subscribe(subscriber); - - captureProgressListener.getCompletionFuture().get(5, TimeUnit.SECONDS); - - Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, times(1)).executionSuccess(ArgumentMatchers.any(ProgressListener.Context.ExecutionSuccess.class)); - } - - @Test - void executionFailure_when_SubscriptionErrors() throws Exception { - Long contentLength = 51L; - String inputString = RandomStringUtils.randomAlphanumeric(contentLength.intValue()); - - CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); - SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); - builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); - - SdkRequestOverrideConfiguration overrideConfig = builder.build(); - SdkRequest sdkRequest = NoopTestRequest.builder() - .overrideConfiguration(overrideConfig) - .build(); - - sourceFile = new RandomTempFile(OBJ_SIZE); - AsyncRequestBody requestFileBody = AsyncRequestBody.fromInputStream( - new ExceptionThrowingByteArrayInputStream(inputString.getBytes(), 3), contentLength, - Executors.newSingleThreadExecutor()); - - ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, contentLength); - AsyncRequestBody asyncRequestBody = progressUpdater.wrapUploadRequestBody(requestFileBody); - - CompletableFuture completionFuture = completedSdkResponse(10); - progressUpdater.registerCompletion(completionFuture); - - AtomicReference publishedBuffer = new AtomicReference<>(); - Subscriber subscriber = new SimpleSubscriber(publishedBuffer::set); - asyncRequestBody.subscribe(subscriber); - - assertThatExceptionOfType(ExecutionException.class).isThrownBy( - () -> captureProgressListener.getCompletionFuture().get(5, TimeUnit.SECONDS)); - - Mockito.verify(mockListener, times(1)).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).executionSuccess(ArgumentMatchers.any(ProgressListener.Context.ExecutionSuccess.class)); - } - - private static class ExceptionThrowingByteArrayInputStream extends ByteArrayInputStream { - private final int exceptionPosition; - - public ExceptionThrowingByteArrayInputStream(byte[] buf, int exceptionPosition) { - super(buf); - this.exceptionPosition = exceptionPosition; - } - - @Override - public int read() { - return (exceptionPosition == pos + 1) ? exceptionThrowingRead() : super.read(); - } - - @Override - public int read(byte[] b, int off, int len) { - return (exceptionPosition >= pos && exceptionPosition < (pos + len)) ? - exceptionThrowingReadByteArr(b, off, len) : super.read(b, off, len); - } - - private int exceptionThrowingRead() { - throw new RuntimeException("Exception occurred at position " + (pos + 1)); - } - - private int exceptionThrowingReadByteArr(byte[] b, int off, int len) { - throw new RuntimeException("Exception occurred in read(byte[]) at position " + exceptionPosition); - } - } - - private static void quietSleep(long millis) { - try { - Thread.sleep(millis); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); // Restore interrupted status - throw new RuntimeException(e); - } - } } From 700bd5e3a41018b48b5ba45facf4fba702b99006 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Wed, 31 Jan 2024 18:48:30 -0800 Subject: [PATCH 27/50] Define ProgressUpdater and LoggingProgressListener --- .../listener/LoggingProgressListener.java | 49 ++++- .../progress/listener/ProgressUpdater.java | 5 +- .../listener/LoggingProgressListenerTest.java | 167 ++++++++++++++++++ .../listener/ProgressUpdaterTest.java | 16 -- 4 files changed, 210 insertions(+), 27 deletions(-) create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListenerTest.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListener.java index fa47cee4caca..fe75efcae75d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListener.java @@ -22,13 +22,12 @@ import java.util.concurrent.atomic.AtomicInteger; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.core.progress.listener.ProgressListener; -import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; import software.amazon.awssdk.utils.Logger; /** - * An implementation of {@link ProgressListener} that logs a progress bar at the {@code INFO} level for upload operations. This implementation - * effectively limits the frequency of updates by limiting logging to events of progress advancement. By default, the - * progress bar has {@value #DEFAULT_MAX_TICKS} ticks, meaning an update is logged for every 5% progression, at most. + * An implementation of {@link ProgressListener} that logs a progress bar at the {@code INFO} level for upload operations. This + * implementation effectively limits the frequency of updates by limiting logging to events of progress advancement. By default, + * the progress bar has {@value #DEFAULT_MAX_TICKS} ticks, meaning an update is logged for every 5% progression, at most. */ @SdkPublicApi public final class LoggingProgressListener implements ProgressListener { @@ -40,40 +39,72 @@ private LoggingProgressListener(int maxTicks) { progressBar = new ProgressBar(maxTicks); } + /** + * Create an instance of {@link LoggingProgressListener} with a custom {@code maxTicks} value. + * + * @param maxTicks the number of ticks in the logged progress bar + */ + public static LoggingProgressListener create(int maxTicks) { + return new LoggingProgressListener(maxTicks); + } + + /** + * Create an instance of {@link LoggingProgressListener} with the default configuration. + */ + public static LoggingProgressListener create() { + return new LoggingProgressListener(DEFAULT_MAX_TICKS); + } + + @Override public void requestPrepared(Context.RequestPrepared context) { - log.info(() -> "Request Prepared... "); + log.info(() -> "Request Prepared..."); context.uploadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); } @Override public void requestHeaderSent(Context.RequestHeaderSent context) { - log.info(() -> "Request Header Sent... "); context.uploadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); } @Override public void requestBytesSent(Context.RequestBytesSent context) { - log.info(() -> "Request Bytes Sent... "); context.uploadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); } @Override public void responseHeaderReceived(Context.ResponseHeaderReceived context) { - log.info(() -> "Response Header Received... "); + log.info(() -> "Upload Successful! Starting Download..."); + context.downloadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); + } + + @Override + public void responseBytesReceived(Context.ResponseBytesReceived context) { context.downloadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); } + @Override + public void executionSuccess(Context.ExecutionSuccess context) { + log.info(() -> "Execution Successful!"); + context.downloadProgressSnapshot().ratioTransferred().ifPresent(progressBar::update); + } + + @Override + public void executionFailure(Context.ExecutionFailure context) { + log.warn(() -> "Execution Failed!", context.exception()); + } + private static class ProgressBar { private final int maxTicks; private final AtomicInteger prevTicks = new AtomicInteger(-1); + private ProgressBar(int maxTicks) { this.maxTicks = maxTicks; } void update(double ratio) { int ticks = (int) Math.floor(ratio * maxTicks); - if (prevTicks.getAndSet(ticks) < ticks) { + if (prevTicks.getAndSet(ticks) != ticks) { log.info(() -> String.format("|%s%s| %s", repeat("=", ticks), repeat(" ", maxTicks - ticks), diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java index 0fae71ed51e8..e02fde82b408 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java @@ -15,7 +15,6 @@ package software.amazon.awssdk.core.internal.progress.listener; -import java.nio.ByteBuffer; import java.util.Collections; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -61,7 +60,9 @@ public ProgressUpdater(SdkRequest sdkRequest, .downloadProgressSnapshot(downloadProgressSnapshot) .build(); - listenerInvoker = (!sdkRequest.overrideConfiguration().isPresent() || sdkRequest.overrideConfiguration().get().progressListeners() == null) + listenerInvoker = + (!sdkRequest.overrideConfiguration().isPresent() || + sdkRequest.overrideConfiguration().get().progressListeners() == null) ? new ProgressListenerInvoker((Collections.emptyList())) : new ProgressListenerInvoker(sdkRequest.overrideConfiguration().get().progressListeners()); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListenerTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListenerTest.java new file mode 100644 index 000000000000..7c83b4de0fcf --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/LoggingProgressListenerTest.java @@ -0,0 +1,167 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.progress.listener; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import java.util.List; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.LogEvent; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.http.NoopTestRequest; +import software.amazon.awssdk.core.internal.progress.ProgressListenerContext; +import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; +import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; +import software.amazon.awssdk.testutils.LogCaptor; + +public class LoggingProgressListenerTest { + private static final long UPLOAD_SIZE_IN_BYTES = 1024L; + private static final long DOWNLOAD_SIZE_IN_BYTES = 1024L; + private DefaultSdkExchangeProgress requestBodyProgress; + private DefaultSdkExchangeProgress responseBodyProgress; + private ProgressListenerContext context; + private LoggingProgressListener listener; + + @BeforeEach + public void setUp() throws Exception { + ProgressSnapshot uploadSnapshot = DefaultProgressSnapshot.builder() + .transferredBytes(0L) + .totalBytes(UPLOAD_SIZE_IN_BYTES) + .build(); + + ProgressSnapshot downloadSnapshot = DefaultProgressSnapshot.builder() + .transferredBytes(0L) + .totalBytes(DOWNLOAD_SIZE_IN_BYTES) + .build(); + requestBodyProgress = new DefaultSdkExchangeProgress(uploadSnapshot); + responseBodyProgress = new DefaultSdkExchangeProgress(downloadSnapshot); + context = ProgressListenerContext.builder() + .request(mock(NoopTestRequest.class)) + .uploadProgressSnapshot(uploadSnapshot) + .downloadProgressSnapshot(downloadSnapshot) + .build(); + listener = LoggingProgressListener.create(); + } + + @Test + public void defaultListener_successfulTransfer() { + try (LogCaptor logCaptor = LogCaptor.create()) { + invokeSuccessfulLifecycle(); + List events = logCaptor.loggedEvents(); + assertLogged(events, Level.INFO, "Request Prepared..."); + assertLogged(events, Level.INFO, "| | 0.0%"); + assertLogged(events, Level.INFO, "|= | 5.0%"); + assertLogged(events, Level.INFO, "|== | 10.0%"); + assertLogged(events, Level.INFO, "|=== | 15.0%"); + assertLogged(events, Level.INFO, "|==== | 20.0%"); + assertLogged(events, Level.INFO, "|===== | 25.0%"); + assertLogged(events, Level.INFO, "|====== | 30.0%"); + assertLogged(events, Level.INFO, "|======= | 35.0%"); + assertLogged(events, Level.INFO, "|======== | 40.0%"); + assertLogged(events, Level.INFO, "|========= | 45.0%"); + assertLogged(events, Level.INFO, "|========== | 50.0%"); + assertLogged(events, Level.INFO, "|=========== | 55.0%"); + assertLogged(events, Level.INFO, "|============ | 60.0%"); + assertLogged(events, Level.INFO, "|============= | 65.0%"); + assertLogged(events, Level.INFO, "|============== | 70.0%"); + assertLogged(events, Level.INFO, "|=============== | 75.0%"); + assertLogged(events, Level.INFO, "|================ | 80.0%"); + assertLogged(events, Level.INFO, "|================= | 85.0%"); + assertLogged(events, Level.INFO, "|================== | 90.0%"); + assertLogged(events, Level.INFO, "|=================== | 95.0%"); + assertLogged(events, Level.INFO, "|====================| 100.0%"); + assertLogged(events, Level.INFO, "Upload Successful! Starting Download..."); + assertLogged(events, Level.INFO, "| | 0.0%"); + assertLogged(events, Level.INFO, "|= | 5.0%"); + assertLogged(events, Level.INFO, "|== | 10.0%"); + assertLogged(events, Level.INFO, "|=== | 15.0%"); + assertLogged(events, Level.INFO, "|==== | 20.0%"); + assertLogged(events, Level.INFO, "|===== | 25.0%"); + assertLogged(events, Level.INFO, "|====== | 30.0%"); + assertLogged(events, Level.INFO, "|======= | 35.0%"); + assertLogged(events, Level.INFO, "|======== | 40.0%"); + assertLogged(events, Level.INFO, "|========= | 45.0%"); + assertLogged(events, Level.INFO, "|========== | 50.0%"); + assertLogged(events, Level.INFO, "|=========== | 55.0%"); + assertLogged(events, Level.INFO, "|============ | 60.0%"); + assertLogged(events, Level.INFO, "|============= | 65.0%"); + assertLogged(events, Level.INFO, "|============== | 70.0%"); + assertLogged(events, Level.INFO, "|=============== | 75.0%"); + assertLogged(events, Level.INFO, "|================ | 80.0%"); + assertLogged(events, Level.INFO, "|================= | 85.0%"); + assertLogged(events, Level.INFO, "|================== | 90.0%"); + assertLogged(events, Level.INFO, "|=================== | 95.0%"); + assertLogged(events, Level.INFO, "|====================| 100.0%"); + assertLogged(events, Level.INFO, "Execution Successful!"); + assertThat(events).isEmpty(); + } + } + + @Test + public void test_customTicksListener_successfulTransfer() { + try (LogCaptor logCaptor = LogCaptor.create()) { + listener = LoggingProgressListener.create(5); + invokeSuccessfulLifecycle(); + List events = logCaptor.loggedEvents(); + assertLogged(events, Level.INFO, "Request Prepared..."); + assertLogged(events, Level.INFO, "| | 0.0%"); + assertLogged(events, Level.INFO, "|= | 20.0%"); + assertLogged(events, Level.INFO, "|== | 40.0%"); + assertLogged(events, Level.INFO, "|=== | 60.0%"); + assertLogged(events, Level.INFO, "|==== | 80.0%"); + assertLogged(events, Level.INFO, "|=====| 100.0%"); + assertLogged(events, Level.INFO, "Upload Successful! Starting Download..."); + assertLogged(events, Level.INFO, "| | 0.0%"); + assertLogged(events, Level.INFO, "|= | 20.0%"); + assertLogged(events, Level.INFO, "|== | 40.0%"); + assertLogged(events, Level.INFO, "|=== | 60.0%"); + assertLogged(events, Level.INFO, "|==== | 80.0%"); + assertLogged(events, Level.INFO, "|=====| 100.0%"); + assertLogged(events, Level.INFO, "Execution Successful!"); + assertThat(events).isEmpty(); + } + } + + private void invokeSuccessfulLifecycle() { + listener.requestPrepared(context); + + listener.requestHeaderSent(context); + + for (int i = 0; i <= UPLOAD_SIZE_IN_BYTES; i++) { + int bytes = i; + listener.requestBytesSent(context.copy(c -> c.uploadProgressSnapshot( + requestBodyProgress.updateAndGet(p -> p.transferredBytes((long) bytes))))); + } + listener.responseHeaderReceived(context); + for (int i = 0; i <= DOWNLOAD_SIZE_IN_BYTES; i++) { + int bytes = i; + listener.responseBytesReceived(context.copy(c -> c.downloadProgressSnapshot( + responseBodyProgress.updateAndGet(p -> p.transferredBytes((long) bytes))))); + } + + listener.executionSuccess(context.copy(b -> b.downloadProgressSnapshot(responseBodyProgress.progressSnapshot()))); + } + + private static void assertLogged(List events, org.apache.logging.log4j.Level level, String message) { + assertThat(events).withFailMessage("Expecting events to not be empty").isNotEmpty(); + LogEvent event = events.remove(0); + String msg = event.getMessage().getFormattedMessage(); + assertThat(msg).isEqualTo(message); + assertThat(event.getLevel()).isEqualTo(level); + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java index 65c07e60a220..e8f3700f6f60 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java @@ -15,24 +15,14 @@ package software.amazon.awssdk.core.internal.progress.listener; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; -import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -40,16 +30,10 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; -import org.reactivestreams.Subscriber; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; -import software.amazon.awssdk.core.SdkResponse; -import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.progress.listener.ProgressListener; -import software.amazon.awssdk.core.protocol.VoidSdkResponse; -import software.amazon.awssdk.http.async.SimpleSubscriber; -import software.amazon.awssdk.testutils.RandomTempFile; public class ProgressUpdaterTest { private CaptureProgressListener captureProgressListener; From 06f811b845cf9e25f7cd17721c6d981413661b14 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Tue, 13 Feb 2024 14:10:30 -0800 Subject: [PATCH 28/50] Added ProgressUpdater into ExecutionContext and removed asynchronous callback methods in ProgressUpdater --- .../awssdk/core/http/ExecutionContext.java | 16 ++++++ .../listener/ProgressListenerInvoker.java | 2 +- .../progress/listener/ProgressUpdater.java | 57 +++++++------------ .../progress/listener/ProgressListener.java | 36 +++--------- .../listener/ProgressUpdaterTest.java | 38 +++++++++++-- 5 files changed, 75 insertions(+), 74 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java index 141ba473986f..afe393fc2069 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java @@ -15,12 +15,15 @@ package software.amazon.awssdk.core.http; +import java.util.Optional; import software.amazon.awssdk.annotations.NotThreadSafe; import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain; import software.amazon.awssdk.core.interceptor.InterceptorContext; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.signer.Signer; +import software.amazon.awssdk.http.auth.aws.internal.signer.FlexibleChecksummer; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; @@ -36,6 +39,7 @@ public final class ExecutionContext implements ToCopyableBuilder progressUpdater() { + return progressUpdater != null ? Optional.of(progressUpdater) : Optional.empty(); + } + @Override public Builder toBuilder() { return new Builder(this); @@ -87,6 +96,7 @@ public static class Builder implements CopyableBuilder listener.attemptFailure(context)); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java index e02fde82b408..bd2673a3b3af 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java @@ -16,16 +16,22 @@ package software.amazon.awssdk.core.internal.progress.listener; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.RequestOverrideConfiguration; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.internal.progress.ProgressListenerContext; import software.amazon.awssdk.core.internal.progress.ProgressListenerFailedContext; import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; +import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.core.progress.listener.SdkExchangeProgress; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; /** * ProgressUpdater exposes methods that invokes listener methods to update and store request progress state @@ -34,23 +40,20 @@ public class ProgressUpdater { private final DefaultSdkExchangeProgress requestBodyProgress; private final DefaultSdkExchangeProgress responseBodyProgress; - private final ProgressListenerContext context; + private ProgressListenerContext context; private final ProgressListenerInvoker listenerInvoker; - private final CompletableFuture endOfStreamFuture; public ProgressUpdater(SdkRequest sdkRequest, - Long contentLength) { + Long requestContentLength) { DefaultProgressSnapshot.Builder uploadProgressSnapshotBuilder = DefaultProgressSnapshot.builder(); uploadProgressSnapshotBuilder.transferredBytes(0L); - Optional.ofNullable(contentLength).ifPresent(uploadProgressSnapshotBuilder::totalBytes); + Optional.ofNullable(requestContentLength).ifPresent(uploadProgressSnapshotBuilder::totalBytes); ProgressSnapshot uploadProgressSnapshot = uploadProgressSnapshotBuilder.build(); requestBodyProgress = new DefaultSdkExchangeProgress(uploadProgressSnapshot); DefaultProgressSnapshot.Builder downloadProgressSnapshotBuilder = DefaultProgressSnapshot.builder(); downloadProgressSnapshotBuilder.transferredBytes(0L); - Optional.ofNullable(contentLength).ifPresent(downloadProgressSnapshotBuilder::totalBytes); - ProgressSnapshot downloadProgressSnapshot = downloadProgressSnapshotBuilder.build(); responseBodyProgress = new DefaultSdkExchangeProgress(downloadProgressSnapshot); @@ -60,13 +63,11 @@ public ProgressUpdater(SdkRequest sdkRequest, .downloadProgressSnapshot(downloadProgressSnapshot) .build(); - listenerInvoker = - (!sdkRequest.overrideConfiguration().isPresent() || - sdkRequest.overrideConfiguration().get().progressListeners() == null) - ? new ProgressListenerInvoker((Collections.emptyList())) - : new ProgressListenerInvoker(sdkRequest.overrideConfiguration().get().progressListeners()); + listenerInvoker = new ProgressListenerInvoker(sdkRequest.overrideConfiguration().map(RequestOverrideConfiguration::progressListeners).orElse(Collections.emptyList())); + } - endOfStreamFuture = new CompletableFuture<>(); + public void updateResponseContentLength(Long responseContentLength) { + responseBodyProgress.updateAndGet(b -> b.totalBytes(responseContentLength)); } public SdkExchangeProgress requestBodyProgress() { @@ -107,16 +108,6 @@ public void incrementBytesReceived(long numBytes) { listenerInvoker.responseBytesReceived(context.copy(b -> b.downloadProgressSnapshot(snapshot))); } - public void registerCompletion(CompletableFuture future) { - future.whenComplete((r, t) -> { - if (t == null) { - attachEndOfStreamFutureCallback(r); - } else { - executionFailure(t); - } - }); - } - public void responseHeaderReceived() { listenerInvoker.responseHeaderReceived(context); } @@ -126,7 +117,7 @@ public void executionSuccess(SdkResponse response) { listenerInvoker.executionSuccess(context.copy(b -> b.response(response))); } - private void executionFailure(Throwable t) { + public void executionFailure(Throwable t) { listenerInvoker.executionFailure(ProgressListenerFailedContext.builder() .progressListenerContext( context.copy( @@ -140,9 +131,9 @@ private void executionFailure(Throwable t) { .build()); } - private void attemptFailure(Throwable t) { - listenerInvoker.executionFailure(ProgressListenerFailedContext.builder() - .progressListenerContext( + public void attemptFailure(Throwable t) { + listenerInvoker.attemptFailure(ProgressListenerFailedContext.builder() + .progressListenerContext( context.copy( b -> { b.uploadProgressSnapshot( @@ -150,17 +141,7 @@ private void attemptFailure(Throwable t) { b.downloadProgressSnapshot( responseBodyProgress.progressSnapshot()); })) - .exception(t) - .build()); - } - - public void attachEndOfStreamFutureCallback(SdkResponse response) { - endOfStreamFuture.whenComplete((r2, t2) -> { - if (t2 == null) { - executionSuccess(response); - } else { - attemptFailure(t2); - } - }); + .exception(t) + .build()); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index 3cdfa494226b..fff12aa64a12 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -22,6 +22,7 @@ import software.amazon.awssdk.annotations.ThreadSafe; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.internal.progress.listener.LoggingProgressListener; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.http.SdkHttpResponse; @@ -59,7 +60,7 @@ *

  • {@link #executionSuccess(Context.ExecutionSuccess)} - The transfer has completed successfully. This method is called * for every successful transfer.
    • * - * For every failed attempt {@link #attemptFailure(Context.AttemptFailure)}. + * For every failed attempt {@link #attemptFailure(Context.ExecutionFailure)}. * *

    * There are a few important rules and best practices that govern the usage of {@link ProgressListener}s: @@ -229,13 +230,13 @@ default void executionSuccess(Context.ExecutionSuccess context) { *

    * Available context attributes: *

      - *
    1. {@link Context.AttemptFailure#request()}
      2. - *
    2. {@link Context.AttemptFailure#httpRequest()}
      3. - *
    3. {@link Context.AttemptFailure#uploadProgressSnapshot()}
      4. - *
    4. {@link Context.AttemptFailure#exception()} ()}
      5. + *
    5. {@link Context.ExecutionFailure#request()}
      6. + *
    6. {@link Context.ExecutionFailure#httpRequest()}
      7. + *
    7. {@link Context.ExecutionFailure#uploadProgressSnapshot()}
      8. + *
    8. {@link Context.ExecutionFailure#exception()} ()}
      9. *
    */ - default void attemptFailure(Context.AttemptFailure context) { + default void attemptFailure(Context.ExecutionFailure context) { } /** @@ -268,7 +269,6 @@ default void executionFailure(Context.ExecutionFailure context) { * Failed transfer method hierarchy: *
      *
    1. {@link RequestPrepared}
      2. - *
    2. {@link AttemptFailure}
      3. *
    3. {@link ExecutionFailure}
      4. *
    * If the request header includes an Expect: 100-Continue and the service returns a different value, the method invokation @@ -444,28 +444,6 @@ public interface AttemptFailureResponseBytesReceived extends ResponseHeaderRecei Throwable exception(); } - /** - * The request execution attempt failed. - *

    - * Available context attributes: - *

      - *
    1. {@link AttemptFailure#request()}
      2. - *
    2. {@link AttemptFailure#httpRequest()}
      3. - *
    3. {@link AttemptFailure#uploadProgressSnapshot()}
      4. - *
    4. {@link AttemptFailure#exception()}
      5. - *
    - */ - @Immutable - @ThreadSafe - @SdkPublicApi - @SdkPreviewApi - public interface AttemptFailure extends RequestPrepared { - /** - * The exception associated with the failed request. - */ - Throwable exception(); - } - /** * The request execution failed. *

    diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java index e8f3700f6f60..99a4a0e17bec 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java @@ -32,8 +32,10 @@ import org.mockito.Mockito; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.core.protocol.VoidSdkResponse; public class ProgressUpdaterTest { private CaptureProgressListener captureProgressListener; @@ -76,7 +78,7 @@ public void requestPrepared_transferredBytes_equals_zero() { assertFalse(captureProgressListener.requestHeaderSent()); assertFalse(captureProgressListener.responseHeaderReceived()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); Mockito.verify(mockListener, times(1)).requestPrepared(ArgumentMatchers.any(ProgressListener.Context.RequestPrepared.class)); @@ -105,7 +107,7 @@ public void requestHeaderSent_transferredBytes_equals_zero() { assertTrue(captureProgressListener.requestHeaderSent()); assertFalse(captureProgressListener.responseHeaderReceived()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); Mockito.verify(mockListener, times(1)).requestHeaderSent(ArgumentMatchers.any(ProgressListener.Context.RequestHeaderSent.class)); @@ -133,7 +135,7 @@ public void requestBytesSent_transferredBytes() { assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); Mockito.verify(mockListener, times(2)).requestBytesSent(ArgumentMatchers.any(ProgressListener.Context.RequestBytesSent.class)); @@ -182,14 +184,14 @@ public void responseHeaderReceived_transferredBytes_equals_zero() { assertFalse(captureProgressListener.requestHeaderSent()); assertTrue(captureProgressListener.responseHeaderReceived()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); Mockito.verify(mockListener, times(1)).responseHeaderReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseHeaderReceived.class)); } @Test - public void responseBytesReceived_transferredBytes_valid() { + public void executionSuccess_transferredBytes_valid() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -209,10 +211,34 @@ public void responseBytesReceived_transferredBytes_valid() { progressUpdater.incrementBytesReceived(BYTES_TRANSFERRED); assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.responseBodyProgress().progressSnapshot().transferredBytes(), 0.0); + progressUpdater.executionSuccess(VoidSdkResponse.builder().sdkHttpResponse(null).build()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.AttemptFailure.class)); + Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); Mockito.verify(mockListener, times(2)).responseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseBytesReceived.class)); + Mockito.verify(mockListener, times(1)).executionSuccess(ArgumentMatchers.any(ProgressListener.Context.ExecutionSuccess.class)); + } + + @Test + public void executionFailure() { + + String EXCEPTION_MESSAGE = "TEST_EXCEPTION"; + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.attemptFailure(new Throwable(EXCEPTION_MESSAGE)); + progressUpdater.executionFailure(new Throwable(EXCEPTION_MESSAGE)); + Mockito.verify(mockListener, times(1)).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, times(1)).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); } } From 3faf34e6a2fbaa5f689851e7a79df7a6f0855b8f Mon Sep 17 00:00:00 2001 From: Krishnan Date: Tue, 13 Feb 2024 14:48:12 -0800 Subject: [PATCH 29/50] Fixed checkstyle issues --- .../internal/progress/listener/ProgressUpdater.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java index bd2673a3b3af..8fa12c963f64 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java @@ -16,10 +16,7 @@ package software.amazon.awssdk.core.internal.progress.listener; import java.util.Collections; -import java.util.List; import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.function.Supplier; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.RequestOverrideConfiguration; import software.amazon.awssdk.core.SdkRequest; @@ -27,11 +24,8 @@ import software.amazon.awssdk.core.internal.progress.ProgressListenerContext; import software.amazon.awssdk.core.internal.progress.ProgressListenerFailedContext; import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; -import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.core.progress.listener.SdkExchangeProgress; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; -import software.amazon.awssdk.utils.builder.CopyableBuilder; -import software.amazon.awssdk.utils.builder.ToCopyableBuilder; /** * ProgressUpdater exposes methods that invokes listener methods to update and store request progress state @@ -63,7 +57,9 @@ public ProgressUpdater(SdkRequest sdkRequest, .downloadProgressSnapshot(downloadProgressSnapshot) .build(); - listenerInvoker = new ProgressListenerInvoker(sdkRequest.overrideConfiguration().map(RequestOverrideConfiguration::progressListeners).orElse(Collections.emptyList())); + listenerInvoker = new ProgressListenerInvoker(sdkRequest.overrideConfiguration() + .map(RequestOverrideConfiguration::progressListeners) + .orElse(Collections.emptyList())); } public void updateResponseContentLength(Long responseContentLength) { From 8f1849661700c245f789425e77cfad25a686e95a Mon Sep 17 00:00:00 2001 From: Krishnan Date: Tue, 13 Feb 2024 14:52:07 -0800 Subject: [PATCH 30/50] Merged from master --- .../java/software/amazon/awssdk/core/http/ExecutionContext.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java index afe393fc2069..01cc3e0c5166 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java @@ -23,7 +23,6 @@ import software.amazon.awssdk.core.interceptor.InterceptorContext; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.signer.Signer; -import software.amazon.awssdk.http.auth.aws.internal.signer.FlexibleChecksummer; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; From 6eacf656fdde09ea1c47fdecbf97348ad37b5b74 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Wed, 14 Feb 2024 08:41:24 -0800 Subject: [PATCH 31/50] Added tests for attemptFailureResponseBytesReceived --- .../ProgressListenerFailedContext.java | 11 ++ .../listener/ProgressListenerInvoker.java | 2 +- .../progress/listener/ProgressUpdater.java | 14 ++ .../progress/listener/ProgressListener.java | 48 ++---- .../listener/SdkExchangeProgress.java | 4 +- .../listener/CaptureProgressListener.java | 12 ++ .../listener/ProgressUpdaterTest.java | 146 ++++++++++++++++-- 7 files changed, 187 insertions(+), 50 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java index 6b277cc470c3..5806abb499e5 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerFailedContext.java @@ -22,6 +22,7 @@ import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.SdkHttpResponse; import software.amazon.awssdk.utils.ToString; import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.CopyableBuilder; @@ -91,6 +92,16 @@ public String toString() { .build(); } + @Override + public SdkHttpResponse httpResponse() { + return progressListenerContext.httpResponse(); + } + + @Override + public ProgressSnapshot downloadProgressSnapshot() { + return progressListenerContext.downloadProgressSnapshot(); + } + public static final class Builder implements CopyableBuilder { private ProgressListenerContext progressListenerContext; private Throwable exception; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java index c8aba1cbd7e0..157eca5c449f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressListenerInvoker.java @@ -83,7 +83,7 @@ public void attemptFailure(Context.ExecutionFailure context) { } @Override - public void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseBytesReceived context) { + public void attemptFailureResponseBytesReceived(Context.ExecutionFailure context) { forEach(listener -> listener.attemptFailureResponseBytesReceived(context)); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java index 8fa12c963f64..3db626977b6a 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java @@ -140,4 +140,18 @@ public void attemptFailure(Throwable t) { .exception(t) .build()); } + + public void attemptFailureResponseBytesReceived(Throwable t) { + listenerInvoker.attemptFailureResponseBytesReceived(ProgressListenerFailedContext.builder() + .progressListenerContext( + context.copy( + b -> { + b.uploadProgressSnapshot( + requestBodyProgress.progressSnapshot()); + b.downloadProgressSnapshot( + responseBodyProgress.progressSnapshot()); + })) + .exception(t) + .build()); + } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index fff12aa64a12..4a2e2a4df24d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -194,15 +194,15 @@ default void responseBytesReceived(Context.ResponseBytesReceived context) { *

    * Available context attributes: *

      - *
    1. {@link Context.AttemptFailureResponseBytesReceived#request()}
      2. - *
    2. {@link Context.AttemptFailureResponseBytesReceived#httpRequest()}
      3. - *
    3. {@link Context.AttemptFailureResponseBytesReceived#uploadProgressSnapshot()}
      4. - *
    4. {@link Context.AttemptFailureResponseBytesReceived#httpResponse()}
      5. - *
    5. {@link Context.AttemptFailureResponseBytesReceived#downloadProgressSnapshot()}
      6. - *
    6. {@link Context.AttemptFailureResponseBytesReceived#exception()}
      7. + *
    7. {@link Context.ExecutionFailure#request()}
      8. + *
    8. {@link Context.ExecutionFailure#httpRequest()}
      9. + *
    9. {@link Context.ExecutionFailure#uploadProgressSnapshot()}
      10. + *
    10. {@link Context.ExecutionFailure#httpResponse()} ()}
      11. + *
    11. {@link Context.ExecutionFailure#downloadProgressSnapshot()}
      12. + *
    12. {@link Context.ExecutionFailure#exception()}
      13. *
    */ - default void attemptFailureResponseBytesReceived(Context.AttemptFailureResponseBytesReceived context) { + default void attemptFailureResponseBytesReceived(Context.ExecutionFailure context) { } /** @@ -233,7 +233,9 @@ default void executionSuccess(Context.ExecutionSuccess context) { *
  • {@link Context.ExecutionFailure#request()}
    • *
  • {@link Context.ExecutionFailure#httpRequest()}
    • *
  • {@link Context.ExecutionFailure#uploadProgressSnapshot()}
    • - *
  • {@link Context.ExecutionFailure#exception()} ()}
    • + *
  • {@link Context.ExecutionFailure#httpResponse()} ()}
    • + *
  • {@link Context.ExecutionFailure#downloadProgressSnapshot()}
    • + *
  • {@link Context.ExecutionFailure#exception()}
    • * */ default void attemptFailure(Context.ExecutionFailure context) { @@ -248,7 +250,9 @@ default void attemptFailure(Context.ExecutionFailure context) { *
  • {@link Context.ExecutionFailure#request()}
    • *
  • {@link Context.ExecutionFailure#httpRequest()}
    • *
  • {@link Context.ExecutionFailure#uploadProgressSnapshot()}
    • - *
  • {@link Context.ExecutionFailure#exception()} ()}
    • + *
  • {@link Context.ExecutionFailure#httpResponse()} ()}
    • + *
  • {@link Context.ExecutionFailure#downloadProgressSnapshot()}
    • + *
  • {@link Context.ExecutionFailure#exception()}
    • * */ default void executionFailure(Context.ExecutionFailure context) { @@ -278,7 +282,6 @@ default void executionFailure(Context.ExecutionFailure context) { *
  • {@link RequestHeaderSent}
    • *
  • {@link RequestBytesSent}
    • *
  • {@link ResponseHeaderReceived}
    • - *
  • {@link AttemptFailureResponseBytesReceived}
    • *
  • {@link ExecutionFailure}
    • * * @@ -423,27 +426,6 @@ public interface ExecutionSuccess extends ResponseBytesReceived { SdkResponse response(); } - /** - * This facilitates capturing and handling an error response returned by service - *

    - * Available context attributes: - *

      - *
    1. {@link AttemptFailureResponseBytesReceived#request()}
      2. - *
    2. {@link AttemptFailureResponseBytesReceived#httpRequest()}
      3. - *
    3. {@link AttemptFailureResponseBytesReceived#uploadProgressSnapshot()}
      4. - *
    4. {@link AttemptFailureResponseBytesReceived#httpResponse()} ()}
      5. - *
    5. {@link AttemptFailureResponseBytesReceived#downloadProgressSnapshot()}
      6. - *
    6. {@link AttemptFailureResponseBytesReceived#exception()}
      7. - *
    - */ - @Immutable - @ThreadSafe - @SdkPublicApi - @SdkPreviewApi - public interface AttemptFailureResponseBytesReceived extends ResponseHeaderReceived { - Throwable exception(); - } - /** * The request execution failed. *

    @@ -452,6 +434,8 @@ public interface AttemptFailureResponseBytesReceived extends ResponseHeaderRecei *

  • {@link ExecutionFailure#request()}
    • *
  • {@link ExecutionFailure#httpRequest()}
    • *
  • {@link ExecutionFailure#uploadProgressSnapshot()}
    • + *
  • {@link ExecutionFailure#httpResponse()} ()}
    • + *
  • {@link ExecutionFailure#downloadProgressSnapshot()}
    • *
  • {@link ExecutionFailure#exception()}
    • * */ @@ -459,7 +443,7 @@ public interface AttemptFailureResponseBytesReceived extends ResponseHeaderRecei @ThreadSafe @SdkPublicApi @SdkPreviewApi - public interface ExecutionFailure extends RequestPrepared { + public interface ExecutionFailure extends ResponseBytesReceived { /** * The exception associated with the failed request. */ diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkExchangeProgress.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkExchangeProgress.java index 04dbce4736c1..2ea1325b242a 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkExchangeProgress.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/SdkExchangeProgress.java @@ -26,8 +26,8 @@ public interface SdkExchangeProgress { /** - * SdkExchange Progress makes it available for - * Takes a snapshot of the request execution progress + * SdkExchange Progress class stores the Progress Snapshot + * and is used to track request or response progress * represented by an immutable {@link ProgressSnapshot}. */ ProgressSnapshot progressSnapshot(); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java index 9861293da2ba..f5e9db0e11ff 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/CaptureProgressListener.java @@ -101,5 +101,17 @@ public void executionFailure(Context.ExecutionFailure context) { exceptionCaught = context.exception(); completionFuture.completeExceptionally(exceptionCaught); } + + @Override + public void attemptFailure(Context.ExecutionFailure context) { + exceptionCaught = context.exception(); + completionFuture.completeExceptionally(exceptionCaught); + } + + @Override + public void attemptFailureResponseBytesReceived(Context.ExecutionFailure context) { + exceptionCaught = context.exception(); + completionFuture.completeExceptionally(exceptionCaught); + } } diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java index 99a4a0e17bec..0cb887561449 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -32,22 +33,24 @@ import org.mockito.Mockito; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; -import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.core.protocol.VoidSdkResponse; public class ProgressUpdaterTest { private CaptureProgressListener captureProgressListener; - private static final long BYTES_TRANSFERRED = 5L; + private static final Throwable attemptFailure = new Throwable("AttemptFailureException"); + private static final Throwable executionFailure = new Throwable("ExecutionFailureException"); + private static final Throwable attemptFailureResponseBytesReceived + = new Throwable("AttemptFailureResponseBytesReceivedException"); @BeforeEach void initiate() { captureProgressListener = new CaptureProgressListener(); } - private static Stream contentLength() { + private static Stream contentLength() { return Stream.of( Arguments.of(100L), Arguments.of(200L), @@ -79,7 +82,7 @@ public void requestPrepared_transferredBytes_equals_zero() { assertFalse(captureProgressListener.responseHeaderReceived()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, times(1)).requestPrepared(ArgumentMatchers.any(ProgressListener.Context.RequestPrepared.class)); @@ -108,7 +111,7 @@ public void requestHeaderSent_transferredBytes_equals_zero() { assertFalse(captureProgressListener.responseHeaderReceived()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, times(1)).requestHeaderSent(ArgumentMatchers.any(ProgressListener.Context.RequestHeaderSent.class)); } @@ -132,15 +135,62 @@ public void requestBytesSent_transferredBytes() { assertEquals(BYTES_TRANSFERRED, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); - assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); + assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, + progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, times(2)).requestBytesSent(ArgumentMatchers.any(ProgressListener.Context.RequestBytesSent.class)); } + @Test + public void validate_resetBytesSent() { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); + assertEquals(BYTES_TRANSFERRED, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); + + progressUpdater.resetBytesSent(); + assertEquals(0, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); + + } + + @Test + public void validate_resetBytesReceived() { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.incrementBytesReceived(BYTES_TRANSFERRED); + assertEquals(BYTES_TRANSFERRED, progressUpdater.responseBodyProgress().progressSnapshot().transferredBytes(), 0.0); + + progressUpdater.resetBytesReceived(); + assertEquals(0, progressUpdater.responseBodyProgress().progressSnapshot().transferredBytes(), 0.0); + + } + @ParameterizedTest @MethodSource("contentLength") public void ratioTransferred_upload_transferredBytes(long contentLength) { @@ -158,7 +208,8 @@ public void ratioTransferred_upload_transferredBytes(long contentLength) { ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, contentLength); progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); - assertEquals((double) BYTES_TRANSFERRED / contentLength, progressUpdater.requestBodyProgress().progressSnapshot().ratioTransferred().getAsDouble(), 0.0); + assertEquals((double) BYTES_TRANSFERRED / contentLength, + progressUpdater.requestBodyProgress().progressSnapshot().ratioTransferred().getAsDouble(), 0.0); } @@ -185,7 +236,7 @@ public void responseHeaderReceived_transferredBytes_equals_zero() { assertTrue(captureProgressListener.responseHeaderReceived()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, times(1)).responseHeaderReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseHeaderReceived.class)); } @@ -209,20 +260,20 @@ public void executionSuccess_transferredBytes_valid() { assertEquals(BYTES_TRANSFERRED, progressUpdater.responseBodyProgress().progressSnapshot().transferredBytes(), 0.0); progressUpdater.incrementBytesReceived(BYTES_TRANSFERRED); - assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, progressUpdater.responseBodyProgress().progressSnapshot().transferredBytes(), 0.0); + assertEquals(BYTES_TRANSFERRED + BYTES_TRANSFERRED, + progressUpdater.responseBodyProgress().progressSnapshot().transferredBytes(), 0.0); progressUpdater.executionSuccess(VoidSdkResponse.builder().sdkHttpResponse(null).build()); Mockito.verify(mockListener, never()).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, never()).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); - Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.AttemptFailureResponseBytesReceived.class)); + Mockito.verify(mockListener, never()).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, times(2)).responseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseBytesReceived.class)); Mockito.verify(mockListener, times(1)).executionSuccess(ArgumentMatchers.any(ProgressListener.Context.ExecutionSuccess.class)); } @Test - public void executionFailure() { + public void attemptFailureResponseBytesReceived() { - String EXCEPTION_MESSAGE = "TEST_EXCEPTION"; CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); @@ -235,10 +286,75 @@ public void executionFailure() { .build(); ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); - progressUpdater.attemptFailure(new Throwable(EXCEPTION_MESSAGE)); - progressUpdater.executionFailure(new Throwable(EXCEPTION_MESSAGE)); + progressUpdater.requestPrepared(); + progressUpdater.responseHeaderReceived(); + progressUpdater.attemptFailureResponseBytesReceived(attemptFailureResponseBytesReceived); + Mockito.verify(mockListener, times(1)).requestPrepared(ArgumentMatchers.any(ProgressListener.Context.RequestPrepared.class)); + Mockito.verify(mockListener, times(1)).responseHeaderReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseHeaderReceived.class)); + Mockito.verify(mockListener, times(1)).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, times(0)).responseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseBytesReceived.class)); + Mockito.verify(mockListener, times(0)).executionSuccess(ArgumentMatchers.any(ProgressListener.Context.ExecutionSuccess.class)); + + Assertions.assertEquals(captureProgressListener.exceptionCaught().getMessage(), attemptFailureResponseBytesReceived.getMessage()); + } + + @Test + public void attemptFailure() { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.requestPrepared(); + progressUpdater.attemptFailure(attemptFailure); + + Mockito.verify(mockListener, times(1)).requestPrepared(ArgumentMatchers.any(ProgressListener.Context.RequestPrepared.class)); + Mockito.verify(mockListener, times(0)).responseHeaderReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseHeaderReceived.class)); + Mockito.verify(mockListener, times(0)).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, times(0)).responseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseBytesReceived.class)); + Mockito.verify(mockListener, times(0)).executionSuccess(ArgumentMatchers.any(ProgressListener.Context.ExecutionSuccess.class)); Mockito.verify(mockListener, times(1)).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, times(0)).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + + Assertions.assertEquals(captureProgressListener.exceptionCaught().getMessage(), attemptFailure.getMessage()); + } + + @Test + public void executionFailure() { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.requestPrepared(); + progressUpdater.executionFailure(executionFailure); + + + Mockito.verify(mockListener, times(1)).requestPrepared(ArgumentMatchers.any(ProgressListener.Context.RequestPrepared.class)); + Mockito.verify(mockListener, times(0)).responseHeaderReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseHeaderReceived.class)); + Mockito.verify(mockListener, times(0)).attemptFailureResponseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + Mockito.verify(mockListener, times(0)).responseBytesReceived(ArgumentMatchers.any(ProgressListener.Context.ResponseBytesReceived.class)); + Mockito.verify(mockListener, times(0)).executionSuccess(ArgumentMatchers.any(ProgressListener.Context.ExecutionSuccess.class)); + Mockito.verify(mockListener, times(0)).attemptFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); Mockito.verify(mockListener, times(1)).executionFailure(ArgumentMatchers.any(ProgressListener.Context.ExecutionFailure.class)); + + Assertions.assertEquals(captureProgressListener.exceptionCaught().getMessage(), executionFailure.getMessage()); } } From 43dd6a236fbef3a5bdfb1d671f6c1052565b04ed Mon Sep 17 00:00:00 2001 From: Krishnan Date: Thu, 15 Feb 2024 11:57:59 -0800 Subject: [PATCH 32/50] Asynchronous Code Path Progress Listener WIP --- .../imds/internal/AsyncHttpRequestHelper.java | 3 +- .../async/SimpleHttpContentPublisher.java | 21 ++++++- .../AsyncApiCallMetricCollectionStage.java | 1 + ...ecutionFailureExceptionReportingStage.java | 5 +- .../stages/MakeAsyncHttpRequestStage.java | 51 ++++++++++++++-- .../stages/UnwrapResponseContainer.java | 4 ++ .../metrics/BytesReadTrackingPublisher.java | 18 +++++- .../metrics/BytesSentTrackingSubscriber.java | 58 +++++++++++++++++++ .../progress/listener/ProgressListener.java | 3 +- .../async/SimpleRequestProviderTckTest.java | 3 +- .../BytesReadTrackingPublisherTckTest.java | 3 +- .../BytesReadTrackingPublisherTest.java | 5 +- .../internal/PresignRequestHandlerTest.java | 2 + 13 files changed, 159 insertions(+), 18 deletions(-) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java diff --git a/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java b/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java index 73a70ab4ea02..f5443b00990f 100644 --- a/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java +++ b/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java @@ -61,7 +61,8 @@ private static CompletableFuture sendAsync(SdkAsyncHttpClient client, SdkHttpFullRequest request, HttpResponseHandler handler, CompletableFuture parentFuture) { - SdkHttpContentPublisher requestContentPublisher = new SimpleHttpContentPublisher(request); + SdkHttpContentPublisher requestContentPublisher = new SimpleHttpContentPublisher(request, + Optional.empty()); TransformingAsyncResponseHandler responseHandler = new AsyncResponseHandler<>(handler, Function.identity(), new ExecutionAttributes()); CompletableFuture responseHandlerFuture = responseHandler.prepare(); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java index 0471c6199dc1..fd2a6d8008ff 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java @@ -22,6 +22,7 @@ import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.utils.IoUtils; @@ -36,10 +37,15 @@ public final class SimpleHttpContentPublisher implements SdkHttpContentPublisher private final byte[] content; private final int length; - public SimpleHttpContentPublisher(SdkHttpFullRequest request) { + private ProgressUpdater progressUpdater; + + public SimpleHttpContentPublisher(SdkHttpFullRequest request, Optional progressUpdater) { this.content = request.contentStreamProvider().map(p -> invokeSafely(() -> IoUtils.toByteArray(p.newStream()))) .orElseGet(() -> new byte[0]); this.length = content.length; + progressUpdater.ifPresent(value -> { + this.progressUpdater = value; + }); } @Override @@ -49,15 +55,21 @@ public Optional contentLength() { @Override public void subscribe(Subscriber s) { - s.onSubscribe(new SubscriptionImpl(s)); + s.onSubscribe(new SubscriptionImpl(s, + Optional.ofNullable(progressUpdater))); } private class SubscriptionImpl implements Subscription { private boolean running = true; private final Subscriber s; + private ProgressUpdater progressUpdater; - private SubscriptionImpl(Subscriber s) { + private SubscriptionImpl(Subscriber s, + Optional progressUpdater) { this.s = s; + progressUpdater.ifPresent(value -> { + this.progressUpdater = value; + }); } @Override @@ -68,6 +80,9 @@ public void request(long n) { s.onError(new IllegalArgumentException("Demand must be positive")); } else { s.onNext(ByteBuffer.wrap(content)); + if(progressUpdater != null) { + progressUpdater.incrementBytesSent(content.length); + } s.onComplete(); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallMetricCollectionStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallMetricCollectionStage.java index 09016026be1c..4e4b9c36216d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallMetricCollectionStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallMetricCollectionStage.java @@ -18,6 +18,7 @@ import java.time.Duration; import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; import software.amazon.awssdk.core.internal.util.MetricUtils; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncExecutionFailureExceptionReportingStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncExecutionFailureExceptionReportingStage.java index 4a0cbf3b6dba..8c2a60b7f8a2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncExecutionFailureExceptionReportingStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncExecutionFailureExceptionReportingStage.java @@ -42,12 +42,15 @@ public CompletableFuture execute(SdkHttpFullRequest input, RequestExecu CompletableFuture executeFuture = wrappedExecute.handle((o, t) -> { if (t != null) { Throwable toReport = t; - if (toReport instanceof CompletionException) { toReport = toReport.getCause(); } toReport = reportFailureToInterceptors(context, toReport); + context.executionContext().progressUpdater().ifPresent(progressUpdater -> { + progressUpdater.attemptFailure(t); + }); + throw CompletableFutureUtils.errorAsCompletionException(ThrowableUtils.asSdkException(toReport)); } else { return o; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index 5c443f07a9a5..92830009d872 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -21,6 +21,8 @@ import java.nio.ByteBuffer; import java.time.Duration; +import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; @@ -31,6 +33,7 @@ import org.reactivestreams.Subscriber; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.Response; +import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.core.client.config.SdkClientOption; @@ -47,17 +50,22 @@ import software.amazon.awssdk.core.internal.http.timers.TimeoutTracker; import software.amazon.awssdk.core.internal.http.timers.TimerUtils; import software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher; +import software.amazon.awssdk.core.internal.metrics.BytesSentTrackingSubscriber; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.internal.util.MetricUtils; import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpResponse; import software.amazon.awssdk.http.async.AsyncExecuteRequest; import software.amazon.awssdk.http.async.SdkAsyncHttpClient; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; +import software.amazon.awssdk.http.auth.aws.internal.signer.FlexibleChecksummer; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.StringUtils; /** * Delegate to the HTTP implementation to make an HTTP request and receive the response. @@ -129,9 +137,19 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque CompletableFuture> responseHandlerFuture = responseHandler.prepare(); + if(progressListenerAttached(context.originalRequest())) { + Long requestContentLength = context.requestProvider().contentLength().isPresent() ? + context.requestProvider().contentLength().get() : null; + ProgressUpdater progressUpdater = new ProgressUpdater(context.originalRequest(), requestContentLength); + progressUpdater.requestPrepared(); + context.executionContext().toBuilder().progressUpdater(progressUpdater); + } + SdkHttpContentPublisher requestProvider = context.requestProvider() == null - ? new SimpleHttpContentPublisher(request) - : new SdkHttpContentPublisherAdapter(context.requestProvider()); + ? new SimpleHttpContentPublisher(request, + context.executionContext().progressUpdater()) + : new SdkHttpContentPublisherAdapter(context.requestProvider(), + context.executionContext().progressUpdater()); // Set content length if it hasn't been set already. SdkHttpFullRequest requestWithContentLength = getRequestWithContentLength(request, requestProvider); @@ -263,10 +281,14 @@ private void completeResponseFuture(CompletableFuture> respons */ private static final class SdkHttpContentPublisherAdapter implements SdkHttpContentPublisher { + private ProgressUpdater progressUpdater; private final AsyncRequestBody asyncRequestBody; - private SdkHttpContentPublisherAdapter(AsyncRequestBody asyncRequestBody) { + private SdkHttpContentPublisherAdapter(AsyncRequestBody asyncRequestBody, Optional progressUpdater) { this.asyncRequestBody = asyncRequestBody; + progressUpdater.ifPresent(value -> { + this.progressUpdater = value; + }); } @Override @@ -276,6 +298,7 @@ public Optional contentLength() { @Override public void subscribe(Subscriber s) { + s = progressUpdater == null ? s : new BytesSentTrackingSubscriber(s, Optional.of(progressUpdater)); asyncRequestBody.subscribe(s); } } @@ -303,14 +326,34 @@ public void onHeaders(SdkHttpResponse headers) { long d = now - startTime; context.attemptMetricCollector().reportMetric(CoreMetric.TIME_TO_FIRST_BYTE, Duration.ofNanos(d)); super.onHeaders(headers); + + context.executionContext().progressUpdater().ifPresent(progressUpdater -> { + progressUpdater.responseHeaderReceived(); + headers.firstMatchingHeader(CONTENT_LENGTH).ifPresent(value -> { + if (!StringUtils.isNotBlank(value)) { + progressUpdater.updateResponseContentLength(Long.parseLong(value)); + } + }); + }); } @Override public void onStream(Publisher stream) { + BytesReadTrackingPublisher bytesReadTrackingPublisher; AtomicLong bytesReadCounter = context.executionAttributes() .getAttribute(SdkInternalExecutionAttribute.RESPONSE_BYTES_READ); - BytesReadTrackingPublisher bytesReadTrackingPublisher = new BytesReadTrackingPublisher(stream, bytesReadCounter); + + bytesReadTrackingPublisher = new BytesReadTrackingPublisher(stream, bytesReadCounter, context.executionContext().progressUpdater()); + super.onStream(bytesReadTrackingPublisher); } } + public boolean progressListenerAttached(SdkRequest request) { + if (request.overrideConfiguration().isPresent() && + !request.overrideConfiguration().get().progressListeners().isEmpty()) { + return true; + } + + return false; + } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java index d7290a29c1cb..0abb9915f18b 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java @@ -17,6 +17,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.Response; +import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; @@ -31,6 +32,9 @@ public class UnwrapResponseContainer implements RequestPipeline, OutputT> { @Override public OutputT execute(Response input, RequestExecutionContext context) throws Exception { + context.executionContext().progressUpdater().ifPresent(progressUpdater -> { + progressUpdater.executionSuccess((SdkResponse) input.response()); + }); return input.response(); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java index dd8ef03b7312..a096aa2ff963 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java @@ -16,11 +16,13 @@ package software.amazon.awssdk.core.internal.metrics; import java.nio.ByteBuffer; +import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; /** * Publisher that tracks how many bytes are published from the wrapped publisher to the downstream subscriber. @@ -29,15 +31,19 @@ public final class BytesReadTrackingPublisher implements Publisher { private final Publisher upstream; private final AtomicLong bytesRead; + private ProgressUpdater progressUpdater; - public BytesReadTrackingPublisher(Publisher upstream, AtomicLong bytesRead) { + public BytesReadTrackingPublisher(Publisher upstream, AtomicLong bytesRead, Optional progressUpdater) { this.upstream = upstream; this.bytesRead = bytesRead; + progressUpdater.ifPresent(value -> { + this.progressUpdater = value; + }); } @Override public void subscribe(Subscriber subscriber) { - upstream.subscribe(new BytesReadTracker(subscriber, bytesRead)); + upstream.subscribe(new BytesReadTracker(subscriber, bytesRead, progressUpdater)); } public long bytesRead() { @@ -47,10 +53,12 @@ public long bytesRead() { private static final class BytesReadTracker implements Subscriber { private final Subscriber downstream; private final AtomicLong bytesRead; + private final ProgressUpdater progressUpdater; - private BytesReadTracker(Subscriber downstream, AtomicLong bytesRead) { + private BytesReadTracker(Subscriber downstream, AtomicLong bytesRead, ProgressUpdater progressUpdater) { this.downstream = downstream; this.bytesRead = bytesRead; + this.progressUpdater = progressUpdater; } @Override @@ -62,6 +70,10 @@ public void onSubscribe(Subscription subscription) { public void onNext(ByteBuffer byteBuffer) { bytesRead.addAndGet(byteBuffer.remaining()); downstream.onNext(byteBuffer); + + if(progressUpdater != null) { + progressUpdater.incrementBytesReceived(bytesRead.get()); + } } @Override diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java new file mode 100644 index 000000000000..ea3de596f685 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java @@ -0,0 +1,58 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.metrics; +import java.nio.ByteBuffer; +import java.util.Optional; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; + +public class BytesSentTrackingSubscriber implements Subscriber { + + private Subscriber subscriber; + private ProgressUpdater progressUpdater; + + public BytesSentTrackingSubscriber(Subscriber subscriber, Optional progressUpdater) { + this.subscriber = subscriber; + progressUpdater.ifPresent(value -> { + this.progressUpdater = value; + }); + } + + @Override + public void onSubscribe(Subscription subscription) { + subscriber.onSubscribe(subscription); + } + + @Override + public void onNext(ByteBuffer byteBuffer) { + subscriber.onNext(byteBuffer); + + if(progressUpdater != null) { + progressUpdater.incrementBytesSent(byteBuffer.remaining()); + } + } + + @Override + public void onError(Throwable throwable) { + subscriber.onError(throwable); + } + + @Override + public void onComplete() { + subscriber.onComplete(); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index 4a2e2a4df24d..082b6933ba55 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -189,8 +189,7 @@ default void responseBytesReceived(Context.ResponseBytesReceived context) { /** * For Expect: 100-continue embedded requests, the service returning anything other than 100 continue * indicates a request failure. This method captures the error in the payload - * After this, either executionFailure or requestHeaderSent will always be invoked depending on - * whether the error type is retryable or not + * After this it will either be an executionFailure or a request retry *

    * Available context attributes: *

      diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java index aad8819b0860..c4a1ed29f47d 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java @@ -18,6 +18,7 @@ import java.io.ByteArrayInputStream; import java.net.URI; import java.nio.ByteBuffer; +import java.util.Optional; import org.reactivestreams.Publisher; import org.reactivestreams.tck.PublisherVerification; import org.reactivestreams.tck.TestEnvironment; @@ -36,7 +37,7 @@ public SimpleRequestProviderTckTest() { @Override public Publisher createPublisher(long l) { - return new SimpleHttpContentPublisher(makeFullRequest()); + return new SimpleHttpContentPublisher(makeFullRequest(), Optional.empty()); } @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java index 2afffec30592..813a96ed4dfd 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java @@ -17,6 +17,7 @@ import io.reactivex.Flowable; import java.nio.ByteBuffer; +import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -34,7 +35,7 @@ public BytesReadTrackingPublisherTckTest() { @Override public Publisher createPublisher(long l) { - return new BytesReadTrackingPublisher(createUpstreamPublisher(l), new AtomicLong(0)); + return new BytesReadTrackingPublisher(createUpstreamPublisher(l), new AtomicLong(0), Optional.empty()); } @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java index 259b13d53329..d790d8d59a5f 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java @@ -19,6 +19,7 @@ import io.reactivex.Flowable; import java.nio.ByteBuffer; +import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -35,7 +36,7 @@ public void test_requestAll_calculatesCorrectTotal() { long nElements = 1024; int elementSize = 4; Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0)); + BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0), Optional.empty()); readFully(trackingPublisher); assertThat(trackingPublisher.bytesRead()).isEqualTo(nElements * elementSize); @@ -50,7 +51,7 @@ public void test_requestAll_updatesInputCount() { AtomicLong bytesRead = new AtomicLong(baseBytesRead); Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, bytesRead); + BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, bytesRead, Optional.empty()); readFully(trackingPublisher); long expectedRead = baseBytesRead + nElements * elementSize; diff --git a/services/rds/src/test/java/software/amazon/awssdk/services/rds/internal/PresignRequestHandlerTest.java b/services/rds/src/test/java/software/amazon/awssdk/services/rds/internal/PresignRequestHandlerTest.java index 65a04d3a2f2a..d313998e92c2 100644 --- a/services/rds/src/test/java/software/amazon/awssdk/services/rds/internal/PresignRequestHandlerTest.java +++ b/services/rds/src/test/java/software/amazon/awssdk/services/rds/internal/PresignRequestHandlerTest.java @@ -42,6 +42,7 @@ import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.profiles.ProfileFile; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.rds.RdsClient; import software.amazon.awssdk.services.rds.model.CopyDbSnapshotRequest; import software.amazon.awssdk.services.rds.model.RdsRequest; import software.amazon.awssdk.services.rds.transform.CopyDbSnapshotRequestMarshaller; @@ -142,6 +143,7 @@ public void testParsesDestinationRegionfromRequestEndpoint() throws URISyntaxExc final URI presignedUrl = new URI(presignedRequest.rawQueryParameters().get("PreSignedUrl").get(0)); assertTrue(presignedUrl.toString().contains("DestinationRegion=" + destination.id())); + } @Test From 6b16403e45d64844d1297c1b871db88906ea17d6 Mon Sep 17 00:00:00 2001 From: John Viegas Date: Sat, 17 Feb 2024 17:13:20 -0800 Subject: [PATCH 33/50] Added Pre and Post Execution stages in Request Pipeline to update the Progress Updater --- .../internal/http/AmazonSyncHttpClient.java | 4 + .../PostExecutionProgressUpdateStage.java | 36 ++++++++ .../stages/PreExecuteProgressUpdateStage.java | 55 ++++++++++++ .../stages/ProgressUpdateStageTest.java | 90 +++++++++++++++++++ 4 files changed, 185 insertions(+) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionProgressUpdateStage.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecuteProgressUpdateStage.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java index 34536aca528d..7ccfc7153a45 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java @@ -46,6 +46,8 @@ import software.amazon.awssdk.core.internal.http.pipeline.stages.MakeRequestMutableStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.MergeCustomHeadersStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.MergeCustomQueryParamsStage; +import software.amazon.awssdk.core.internal.http.pipeline.stages.PostExecutionProgressUpdateStage; +import software.amazon.awssdk.core.internal.http.pipeline.stages.PreExecuteProgressUpdateStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.QueryParametersToBodyStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.SigningStage; @@ -206,6 +208,7 @@ public OutputT execute(HttpResponseHandler> response .then(RequestPipelineBuilder .first(SigningStage::new) .then(BeforeTransmissionExecutionInterceptorsStage::new) + .then(PreExecuteProgressUpdateStage::new) .then(MakeHttpRequestStage::new) .then(AfterTransmissionExecutionInterceptorsStage::new) .then(BeforeUnmarshallingExecutionInterceptorsStage::new) @@ -218,6 +221,7 @@ public OutputT execute(HttpResponseHandler> response .wrappedWith(ApiCallTimeoutTrackingStage::new)::build) .wrappedWith((deps, wrapped) -> new ApiCallMetricCollectionStage<>(wrapped)) .then(() -> new UnwrapResponseContainer<>()) + .then(() -> new PostExecutionProgressUpdateStage<>()) .then(() -> new AfterExecutionInterceptorsStage<>()) .wrappedWith(ExecutionFailureExceptionReportingStage::new) .build(httpClientDependencies) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionProgressUpdateStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionProgressUpdateStage.java new file mode 100644 index 000000000000..3347c7583bb7 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionProgressUpdateStage.java @@ -0,0 +1,36 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; + +@SdkInternalApi +public class PostExecutionProgressUpdateStage implements RequestPipeline { + @Override + public OutputT execute(OutputT input, RequestExecutionContext context) throws Exception { + + + context.executionContext().progressUpdater().ifPresent(progressUpdater -> { + progressUpdater.executionSuccess((SdkResponse) input); + }); + return input; + } + + +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecuteProgressUpdateStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecuteProgressUpdateStage.java new file mode 100644 index 000000000000..d54d15c86c38 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecuteProgressUpdateStage.java @@ -0,0 +1,55 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.http.pipeline.RequestToRequestPipeline; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.http.SdkHttpFullRequest; + +@SdkInternalApi +public class PreExecuteProgressUpdateStage implements RequestToRequestPipeline { + @Override + public SdkHttpFullRequest execute(SdkHttpFullRequest input, RequestExecutionContext context) throws Exception { + + if (progressListenerAttached(context.originalRequest())) { + Long requestContentLength = + context.requestProvider() != null && context.requestProvider().contentLength().isPresent() ? + context.requestProvider().contentLength().get() : null; + ProgressUpdater progressUpdater = new ProgressUpdater(context.originalRequest(), requestContentLength); + progressUpdater.requestPrepared(); + progressUpdater.requestHeaderSent(); + + // I think the right way is to set the EXECUTION Attribute or something better we can modify the Context here to + // add updater. + + context.executionContext().toBuilder().progressUpdater(progressUpdater); + } + + return input; + } + + public boolean progressListenerAttached(SdkRequest request) { + if (request.overrideConfiguration().isPresent() && + !request.overrideConfiguration().get().progressListeners().isEmpty()) { + return true; + } + + return false; + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java new file mode 100644 index 000000000000..26cea7d0c0fd --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java @@ -0,0 +1,90 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import java.net.URI; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.http.NoopTestRequest; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; + +@RunWith(MockitoJUnitRunner.class) +public class ProgressUpdateStageTest { + + private static final AsyncRequestBody ASYNC_REQUEST_BODY = AsyncRequestBody.fromString("TestBody"); + private static final RequestBody REQUEST_BODY = RequestBody.fromString("TestBody"); + + @Test + public void sync_progressListener_calledFrom_ExecutionPipeline() throws Exception { + ProgressListener progressListener = Mockito.mock(ProgressListener.class); + SdkHttpFullRequest.Builder requestBuilder = createHttpRequestBuilder(); + boolean isAsyncStreaming = false; + RequestExecutionContext.Builder ctx = progressListenerContext(isAsyncStreaming, progressListener); + PreExecuteProgressUpdateStage preExecutionProgressUpdateStage = new PreExecuteProgressUpdateStage(); + preExecutionProgressUpdateStage.execute(requestBuilder.build(), ctx.build()); + Mockito.verify(progressListener, Mockito.times(1)).requestPrepared(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(1)).requestHeaderSent(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).responseHeaderReceived(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).executionSuccess(Mockito.any()); + + + } + + @Test + public void sync_postExecuteProgressListener_calledFrom_ExecutionPipeline() throws Exception { + ProgressListener progressListener = Mockito.mock(ProgressListener.class); + SdkHttpFullRequest.Builder requestBuilder = createHttpRequestBuilder(); + boolean isAsyncStreaming = false; + RequestExecutionContext.Builder ctx = progressListenerContext(isAsyncStreaming, progressListener); + + PreExecuteProgressUpdateStage preExecutionProgressUpdateStage = new PreExecuteProgressUpdateStage(); + RequestExecutionContext context = ctx.build(); + preExecutionProgressUpdateStage.execute(requestBuilder.build(), context); + + PostExecutionProgressUpdateStage postExecutionProgressUpdateStage = new PostExecutionProgressUpdateStage(); + postExecutionProgressUpdateStage.execute(requestBuilder.build(), context); + // TODO : This test cases are failing since context is passes as args and is immutable. + // Mockito.verify(progressListener, Mockito.times(1)).responseHeaderReceived(Mockito.any()); + // Mockito.verify(progressListener, Mockito.times(1)).executionSuccess(Mockito.any()); + + + } + + + private RequestExecutionContext.Builder progressListenerContext(boolean isAsyncStreaming, ProgressListener progressListener) { + + RequestExecutionContext.Builder builder = + RequestExecutionContext.builder().executionContext(ExecutionContext.builder().build()).originalRequest(NoopTestRequest.builder().overrideConfiguration(SdkRequestOverrideConfiguration.builder().addProgressListener(progressListener).build()).build()); + if (isAsyncStreaming) { + builder.requestProvider(ASYNC_REQUEST_BODY); + } + + return builder; + } + + private SdkHttpFullRequest.Builder createHttpRequestBuilder() { + return SdkHttpFullRequest.builder().uri(URI.create("https://endpoint.host")).method(SdkHttpMethod.GET).contentStreamProvider(REQUEST_BODY.contentStreamProvider()); + } +} From 7a3db5a64c88efd30b45a4627b63742191323a1e Mon Sep 17 00:00:00 2001 From: Krishnan Date: Wed, 20 Mar 2024 13:29:03 -0700 Subject: [PATCH 34/50] Adding Pre, Post execution stages for tracking http requests invoked to prepare the request --- .../PostExecutionUpdateProgressStage.java | 35 +++++++ .../PreExecutionUpdateProgressStage.java | 56 +++++++++++ .../metrics/BytesSentTrackingSubscriber.java | 10 +- .../stages/ProgressUpdateStageTest.java | 92 ++++++++++++++----- 4 files changed, 168 insertions(+), 25 deletions(-) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionUpdateProgressStage.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionUpdateProgressStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionUpdateProgressStage.java new file mode 100644 index 000000000000..c887be837e60 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionUpdateProgressStage.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; + +@SdkInternalApi +public class PostExecutionUpdateProgressStage implements RequestPipeline { + @Override + public OutputT execute(OutputT input, RequestExecutionContext context) throws Exception { + + context.executionContext().progressUpdater().ifPresent(progressUpdater -> { + progressUpdater.executionSuccess((SdkResponse) input); + }); + return input; + } + + +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java new file mode 100644 index 000000000000..8bf5c4f8ecd7 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java @@ -0,0 +1,56 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.http.pipeline.RequestToRequestPipeline; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.http.SdkHttpFullRequest; + +@SdkInternalApi +public class PreExecutionUpdateProgressStage implements RequestToRequestPipeline { + + @Override + public SdkHttpFullRequest execute(SdkHttpFullRequest input, RequestExecutionContext context) throws Exception { + + if (progressListenerAttached(context.originalRequest())) { + Long requestContentLength = + context.requestProvider() != null && context.requestProvider().contentLength().isPresent() ? + context.requestProvider().contentLength().get() : null; + + if (context.executionContext().progressUpdater().isPresent()) { + context.executionContext().progressUpdater().get().requestPrepared(); + } + else { + ProgressUpdater progressUpdater = new ProgressUpdater(context.originalRequest(), requestContentLength); + progressUpdater.requestPrepared(); + context.executionContext().toBuilder().progressUpdater(progressUpdater); + } + } + + return input; + } + + public boolean progressListenerAttached(SdkRequest request) { + if (request.overrideConfiguration().isPresent() && + !request.overrideConfiguration().get().progressListeners().isEmpty()) { + return true; + } + return false; + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java index ea3de596f685..fccc86db6d07 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java @@ -14,16 +14,21 @@ */ package software.amazon.awssdk.core.internal.metrics; + import java.nio.ByteBuffer; import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; +import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +@SdkInternalApi public class BytesSentTrackingSubscriber implements Subscriber { private Subscriber subscriber; private ProgressUpdater progressUpdater; + private AtomicLong bytesSent; public BytesSentTrackingSubscriber(Subscriber subscriber, Optional progressUpdater) { this.subscriber = subscriber; @@ -40,9 +45,10 @@ public void onSubscribe(Subscription subscription) { @Override public void onNext(ByteBuffer byteBuffer) { subscriber.onNext(byteBuffer); + bytesSent.addAndGet(byteBuffer.remaining()); - if(progressUpdater != null) { - progressUpdater.incrementBytesSent(byteBuffer.remaining()); + if (progressUpdater != null) { + progressUpdater.incrementBytesSent(bytesSent.get()); } } diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java index 26cea7d0c0fd..3ab646a663c6 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java @@ -16,16 +16,21 @@ package software.amazon.awssdk.core.internal.http.pipeline.stages; import java.net.URI; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.http.ExecutionContext; import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.core.protocol.VoidSdkResponse; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; @@ -37,46 +42,76 @@ public class ProgressUpdateStageTest { private static final RequestBody REQUEST_BODY = RequestBody.fromString("TestBody"); @Test - public void sync_progressListener_calledFrom_ExecutionPipeline() throws Exception { + public void sync_preExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { ProgressListener progressListener = Mockito.mock(ProgressListener.class); - SdkHttpFullRequest.Builder requestBuilder = createHttpRequestBuilder(); - boolean isAsyncStreaming = false; - RequestExecutionContext.Builder ctx = progressListenerContext(isAsyncStreaming, progressListener); - PreExecuteProgressUpdateStage preExecutionProgressUpdateStage = new PreExecuteProgressUpdateStage(); - preExecutionProgressUpdateStage.execute(requestBuilder.build(), ctx.build()); + + SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() + .addProgressListener(progressListener) + .build(); + + SdkHttpFullRequest requestBuilder = createHttpRequestBuilder().build(); + + SdkRequest request = createSdkHttpRequest(config).build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(request, null); + ExecutionContext executionContext = ExecutionContext.builder() + .progressUpdater(progressUpdater) + .build(); + + RequestExecutionContext requestExecutionContext = progressListenerContext(false, request, + executionContext).build(); + + PreExecutionUpdateProgressStage preExecutionUpdateProgressStage = new PreExecutionUpdateProgressStage(); + preExecutionUpdateProgressStage.execute(requestBuilder, requestExecutionContext); + Mockito.verify(progressListener, Mockito.times(1)).requestPrepared(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(1)).requestHeaderSent(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).requestBytesSent(Mockito.any()); Mockito.verify(progressListener, Mockito.times(0)).responseHeaderReceived(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).responseBytesReceived(Mockito.any()); Mockito.verify(progressListener, Mockito.times(0)).executionSuccess(Mockito.any()); - + Mockito.verify(progressListener, Mockito.times(0)).executionFailure(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).attemptFailure(Mockito.any()); } @Test - public void sync_postExecuteProgressListener_calledFrom_ExecutionPipeline() throws Exception { + public void sync_postExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { ProgressListener progressListener = Mockito.mock(ProgressListener.class); - SdkHttpFullRequest.Builder requestBuilder = createHttpRequestBuilder(); - boolean isAsyncStreaming = false; - RequestExecutionContext.Builder ctx = progressListenerContext(isAsyncStreaming, progressListener); - PreExecuteProgressUpdateStage preExecutionProgressUpdateStage = new PreExecuteProgressUpdateStage(); - RequestExecutionContext context = ctx.build(); - preExecutionProgressUpdateStage.execute(requestBuilder.build(), context); + SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() + .addProgressListener(progressListener) + .build(); - PostExecutionProgressUpdateStage postExecutionProgressUpdateStage = new PostExecutionProgressUpdateStage(); - postExecutionProgressUpdateStage.execute(requestBuilder.build(), context); - // TODO : This test cases are failing since context is passes as args and is immutable. - // Mockito.verify(progressListener, Mockito.times(1)).responseHeaderReceived(Mockito.any()); - // Mockito.verify(progressListener, Mockito.times(1)).executionSuccess(Mockito.any()); + SdkRequest request = createSdkHttpRequest(config).build(); + ProgressUpdater progressUpdater = new ProgressUpdater(request, null); + ExecutionContext executionContext = ExecutionContext.builder() + .progressUpdater(progressUpdater) + .build(); + RequestExecutionContext requestExecutionContext = progressListenerContext(false, request, + executionContext).build(); + + SdkResponse response = createSdkResponseBuilder().build(); + + PostExecutionUpdateProgressStage postExecutionUpdateProgressStage = new PostExecutionUpdateProgressStage(); + postExecutionUpdateProgressStage.execute(response, requestExecutionContext); + + Mockito.verify(progressListener, Mockito.times(0)).requestPrepared(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).requestBytesSent(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).responseHeaderReceived(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).responseBytesReceived(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(1)).executionSuccess(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).executionFailure(Mockito.any()); } - private RequestExecutionContext.Builder progressListenerContext(boolean isAsyncStreaming, ProgressListener progressListener) { + private RequestExecutionContext.Builder progressListenerContext(boolean isAsyncStreaming, SdkRequest sdkRequest, + ExecutionContext executionContext) { RequestExecutionContext.Builder builder = - RequestExecutionContext.builder().executionContext(ExecutionContext.builder().build()).originalRequest(NoopTestRequest.builder().overrideConfiguration(SdkRequestOverrideConfiguration.builder().addProgressListener(progressListener).build()).build()); + RequestExecutionContext.builder().executionContext(executionContext). + originalRequest(sdkRequest); if (isAsyncStreaming) { builder.requestProvider(ASYNC_REQUEST_BODY); } @@ -85,6 +120,17 @@ private RequestExecutionContext.Builder progressListenerContext(boolean isAsyncS } private SdkHttpFullRequest.Builder createHttpRequestBuilder() { - return SdkHttpFullRequest.builder().uri(URI.create("https://endpoint.host")).method(SdkHttpMethod.GET).contentStreamProvider(REQUEST_BODY.contentStreamProvider()); + return SdkHttpFullRequest.builder().uri(URI.create("https://endpoint.host")) + .method(SdkHttpMethod.GET) + .contentStreamProvider(REQUEST_BODY.contentStreamProvider()); + } + + private SdkResponse.Builder createSdkResponseBuilder() { + return VoidSdkResponse.builder(); + } + + private SdkRequest.Builder createSdkHttpRequest(SdkRequestOverrideConfiguration config) { + return NoopTestRequest.builder() + .overrideConfiguration(config); } } From 3f0db6b289d4b62cad803b147722d027c82123da Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 25 Mar 2024 19:23:46 -0700 Subject: [PATCH 35/50] Add Progress Listener lifecycle methods to Asynchronous request path --- .../internal/http/AmazonAsyncHttpClient.java | 6 +- .../internal/http/AmazonSyncHttpClient.java | 10 +- .../async/SimpleHttpContentPublisher.java | 21 +--- .../AsyncApiCallMetricCollectionStage.java | 1 - ...ecutionFailureExceptionReportingStage.java | 4 + .../stages/MakeAsyncHttpRequestStage.java | 58 +++++----- .../PostExecutionProgressUpdateStage.java | 36 ------- .../stages/PreExecuteProgressUpdateStage.java | 55 ---------- .../PreExecutionUpdateProgressStage.java | 7 +- .../metrics/BytesReadTrackingPublisher.java | 12 ++- .../metrics/BytesSentTrackingPublisher.java | 102 ++++++++++++++++++ .../metrics/BytesSentTrackingSubscriber.java | 64 ----------- .../progress/listener/ProgressUpdater.java | 9 +- .../async/SimpleContentPublisherTest.java | 51 +++++++++ .../async/SimpleRequestProviderTckTest.java | 2 +- .../stages/ProgressUpdateStageTest.java | 1 - ...ientExecutionAndRequestTimerTestUtils.java | 17 +++ .../BytesReadTrackingPublisherTest.java | 35 ++++++ .../BytesSentTrackingPublisherTckTest.java | 58 ++++++++++ .../BytesSentTrackingPublisherTest.java | 90 ++++++++++++++++ .../listener/ProgressUpdaterTest.java | 43 +++++++- 21 files changed, 452 insertions(+), 230 deletions(-) delete mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionProgressUpdateStage.java delete mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecuteProgressUpdateStage.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisher.java delete mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleContentPublisherTest.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTckTest.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTest.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonAsyncHttpClient.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonAsyncHttpClient.java index 992f0b730f24..2a121b8ec42c 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonAsyncHttpClient.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonAsyncHttpClient.java @@ -46,6 +46,8 @@ import software.amazon.awssdk.core.internal.http.pipeline.stages.MakeRequestMutableStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.MergeCustomHeadersStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.MergeCustomQueryParamsStage; +import software.amazon.awssdk.core.internal.http.pipeline.stages.PostExecutionUpdateProgressStage; +import software.amazon.awssdk.core.internal.http.pipeline.stages.PreExecutionUpdateProgressStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.QueryParametersToBodyStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.UnwrapResponseContainer; import software.amazon.awssdk.core.internal.util.ThrowableUtils; @@ -201,12 +203,14 @@ public CompletableFuture execute( .then(() -> new HttpChecksumStage(ClientType.ASYNC)) .then(MakeRequestImmutableStage::new) .then(RequestPipelineBuilder - .first(AsyncSigningStage::new) + .first(PreExecutionUpdateProgressStage::new) + .then(AsyncSigningStage::new) .then(AsyncBeforeTransmissionExecutionInterceptorsStage::new) .then(d -> new MakeAsyncHttpRequestStage<>(responseHandler, d)) .wrappedWith(AsyncApiCallAttemptMetricCollectionStage::new) .wrappedWith((deps, wrapped) -> new AsyncRetryableStage<>(responseHandler, deps, wrapped)) .then(async(() -> new UnwrapResponseContainer<>())) + .then(() -> new PostExecutionUpdateProgressStage<>()) .then(async(() -> new AfterExecutionInterceptorsStage<>())) .wrappedWith(AsyncExecutionFailureExceptionReportingStage::new) .wrappedWith(AsyncApiCallTimeoutTrackingStage::new) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java index 7ccfc7153a45..41c9dcf2eea7 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java @@ -46,8 +46,8 @@ import software.amazon.awssdk.core.internal.http.pipeline.stages.MakeRequestMutableStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.MergeCustomHeadersStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.MergeCustomQueryParamsStage; -import software.amazon.awssdk.core.internal.http.pipeline.stages.PostExecutionProgressUpdateStage; -import software.amazon.awssdk.core.internal.http.pipeline.stages.PreExecuteProgressUpdateStage; +import software.amazon.awssdk.core.internal.http.pipeline.stages.PostExecutionUpdateProgressStage; +import software.amazon.awssdk.core.internal.http.pipeline.stages.PreExecutionUpdateProgressStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.QueryParametersToBodyStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.SigningStage; @@ -206,9 +206,9 @@ public OutputT execute(HttpResponseHandler> response .then(MakeRequestImmutableStage::new) // End of mutating request .then(RequestPipelineBuilder - .first(SigningStage::new) + .first(PreExecutionUpdateProgressStage::new) + .then(SigningStage::new) .then(BeforeTransmissionExecutionInterceptorsStage::new) - .then(PreExecuteProgressUpdateStage::new) .then(MakeHttpRequestStage::new) .then(AfterTransmissionExecutionInterceptorsStage::new) .then(BeforeUnmarshallingExecutionInterceptorsStage::new) @@ -221,8 +221,8 @@ public OutputT execute(HttpResponseHandler> response .wrappedWith(ApiCallTimeoutTrackingStage::new)::build) .wrappedWith((deps, wrapped) -> new ApiCallMetricCollectionStage<>(wrapped)) .then(() -> new UnwrapResponseContainer<>()) - .then(() -> new PostExecutionProgressUpdateStage<>()) .then(() -> new AfterExecutionInterceptorsStage<>()) + .then(() -> new PostExecutionUpdateProgressStage<>()) .wrappedWith(ExecutionFailureExceptionReportingStage::new) .build(httpClientDependencies) .execute(request, createRequestExecutionDependencies()); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java index fd2a6d8008ff..0471c6199dc1 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java @@ -22,7 +22,6 @@ import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.utils.IoUtils; @@ -37,15 +36,10 @@ public final class SimpleHttpContentPublisher implements SdkHttpContentPublisher private final byte[] content; private final int length; - private ProgressUpdater progressUpdater; - - public SimpleHttpContentPublisher(SdkHttpFullRequest request, Optional progressUpdater) { + public SimpleHttpContentPublisher(SdkHttpFullRequest request) { this.content = request.contentStreamProvider().map(p -> invokeSafely(() -> IoUtils.toByteArray(p.newStream()))) .orElseGet(() -> new byte[0]); this.length = content.length; - progressUpdater.ifPresent(value -> { - this.progressUpdater = value; - }); } @Override @@ -55,21 +49,15 @@ public Optional contentLength() { @Override public void subscribe(Subscriber s) { - s.onSubscribe(new SubscriptionImpl(s, - Optional.ofNullable(progressUpdater))); + s.onSubscribe(new SubscriptionImpl(s)); } private class SubscriptionImpl implements Subscription { private boolean running = true; private final Subscriber s; - private ProgressUpdater progressUpdater; - private SubscriptionImpl(Subscriber s, - Optional progressUpdater) { + private SubscriptionImpl(Subscriber s) { this.s = s; - progressUpdater.ifPresent(value -> { - this.progressUpdater = value; - }); } @Override @@ -80,9 +68,6 @@ public void request(long n) { s.onError(new IllegalArgumentException("Demand must be positive")); } else { s.onNext(ByteBuffer.wrap(content)); - if(progressUpdater != null) { - progressUpdater.incrementBytesSent(content.length); - } s.onComplete(); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallMetricCollectionStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallMetricCollectionStage.java index 4e4b9c36216d..09016026be1c 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallMetricCollectionStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallMetricCollectionStage.java @@ -18,7 +18,6 @@ import java.time.Duration; import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; import software.amazon.awssdk.core.internal.util.MetricUtils; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java index 322796d331ae..8929d6799170 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java @@ -37,6 +37,10 @@ public OutputT execute(SdkHttpFullRequest input, RequestExecutionContext context return wrapped.execute(input, context); } catch (Exception e) { Throwable throwable = reportFailureToInterceptors(context, e); + + context.executionContext().progressUpdater().ifPresent(progressUpdater -> { + progressUpdater.attemptFailure(e); + }); throw failure(throwable); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index 92830009d872..e011de36fb8a 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -21,8 +21,6 @@ import java.nio.ByteBuffer; import java.time.Duration; -import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; @@ -33,7 +31,6 @@ import org.reactivestreams.Subscriber; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.Response; -import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.core.client.config.SdkClientOption; @@ -50,18 +47,16 @@ import software.amazon.awssdk.core.internal.http.timers.TimeoutTracker; import software.amazon.awssdk.core.internal.http.timers.TimerUtils; import software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher; -import software.amazon.awssdk.core.internal.metrics.BytesSentTrackingSubscriber; +import software.amazon.awssdk.core.internal.metrics.BytesSentTrackingPublisher; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.internal.util.MetricUtils; import software.amazon.awssdk.core.metrics.CoreMetric; -import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpResponse; import software.amazon.awssdk.http.async.AsyncExecuteRequest; import software.amazon.awssdk.http.async.SdkAsyncHttpClient; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; -import software.amazon.awssdk.http.auth.aws.internal.signer.FlexibleChecksummer; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Logger; @@ -137,24 +132,32 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque CompletableFuture> responseHandlerFuture = responseHandler.prepare(); - if(progressListenerAttached(context.originalRequest())) { - Long requestContentLength = context.requestProvider().contentLength().isPresent() ? - context.requestProvider().contentLength().get() : null; - ProgressUpdater progressUpdater = new ProgressUpdater(context.originalRequest(), requestContentLength); - progressUpdater.requestPrepared(); - context.executionContext().toBuilder().progressUpdater(progressUpdater); - } - SdkHttpContentPublisher requestProvider = context.requestProvider() == null - ? new SimpleHttpContentPublisher(request, - context.executionContext().progressUpdater()) - : new SdkHttpContentPublisherAdapter(context.requestProvider(), - context.executionContext().progressUpdater()); + ? new SimpleHttpContentPublisher( + request) + : new SdkHttpContentPublisherAdapter( + context.requestProvider()); + // Set content length if it hasn't been set already. SdkHttpFullRequest requestWithContentLength = getRequestWithContentLength(request, requestProvider); MetricCollector httpMetricCollector = MetricUtils.createHttpMetricsCollector(context); + //If Progress Listening is enabled, wrap around BytesSentTrackingPublisher to track progress on bytes sent + if (context.executionContext().progressUpdater().isPresent()) { + + ProgressUpdater progressUpdater = context.executionContext().progressUpdater().get(); + + if (shouldSetContentLength(request, requestProvider)) { + progressUpdater.updateRequestContentLength(requestProvider.contentLength().get()); + } + + requestProvider = new BytesSentTrackingPublisher(requestProvider, + progressUpdater, + requestProvider.contentLength()); + + } + AsyncExecuteRequest.Builder executeRequestBuilder = AsyncExecuteRequest.builder() .request(requestWithContentLength) .requestContentPublisher(requestProvider) @@ -281,14 +284,10 @@ private void completeResponseFuture(CompletableFuture> respons */ private static final class SdkHttpContentPublisherAdapter implements SdkHttpContentPublisher { - private ProgressUpdater progressUpdater; private final AsyncRequestBody asyncRequestBody; - private SdkHttpContentPublisherAdapter(AsyncRequestBody asyncRequestBody, Optional progressUpdater) { + private SdkHttpContentPublisherAdapter(AsyncRequestBody asyncRequestBody) { this.asyncRequestBody = asyncRequestBody; - progressUpdater.ifPresent(value -> { - this.progressUpdater = value; - }); } @Override @@ -298,7 +297,6 @@ public Optional contentLength() { @Override public void subscribe(Subscriber s) { - s = progressUpdater == null ? s : new BytesSentTrackingSubscriber(s, Optional.of(progressUpdater)); asyncRequestBody.subscribe(s); } } @@ -343,17 +341,11 @@ public void onStream(Publisher stream) { AtomicLong bytesReadCounter = context.executionAttributes() .getAttribute(SdkInternalExecutionAttribute.RESPONSE_BYTES_READ); - bytesReadTrackingPublisher = new BytesReadTrackingPublisher(stream, bytesReadCounter, context.executionContext().progressUpdater()); + bytesReadTrackingPublisher = new BytesReadTrackingPublisher(stream, + bytesReadCounter, + context.executionContext().progressUpdater()); super.onStream(bytesReadTrackingPublisher); } } - public boolean progressListenerAttached(SdkRequest request) { - if (request.overrideConfiguration().isPresent() && - !request.overrideConfiguration().get().progressListeners().isEmpty()) { - return true; - } - - return false; - } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionProgressUpdateStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionProgressUpdateStage.java deleted file mode 100644 index 3347c7583bb7..000000000000 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionProgressUpdateStage.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.internal.http.pipeline.stages; - -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.SdkResponse; -import software.amazon.awssdk.core.internal.http.RequestExecutionContext; -import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; - -@SdkInternalApi -public class PostExecutionProgressUpdateStage implements RequestPipeline { - @Override - public OutputT execute(OutputT input, RequestExecutionContext context) throws Exception { - - - context.executionContext().progressUpdater().ifPresent(progressUpdater -> { - progressUpdater.executionSuccess((SdkResponse) input); - }); - return input; - } - - -} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecuteProgressUpdateStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecuteProgressUpdateStage.java deleted file mode 100644 index d54d15c86c38..000000000000 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecuteProgressUpdateStage.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.internal.http.pipeline.stages; - -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.SdkRequest; -import software.amazon.awssdk.core.internal.http.RequestExecutionContext; -import software.amazon.awssdk.core.internal.http.pipeline.RequestToRequestPipeline; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; -import software.amazon.awssdk.http.SdkHttpFullRequest; - -@SdkInternalApi -public class PreExecuteProgressUpdateStage implements RequestToRequestPipeline { - @Override - public SdkHttpFullRequest execute(SdkHttpFullRequest input, RequestExecutionContext context) throws Exception { - - if (progressListenerAttached(context.originalRequest())) { - Long requestContentLength = - context.requestProvider() != null && context.requestProvider().contentLength().isPresent() ? - context.requestProvider().contentLength().get() : null; - ProgressUpdater progressUpdater = new ProgressUpdater(context.originalRequest(), requestContentLength); - progressUpdater.requestPrepared(); - progressUpdater.requestHeaderSent(); - - // I think the right way is to set the EXECUTION Attribute or something better we can modify the Context here to - // add updater. - - context.executionContext().toBuilder().progressUpdater(progressUpdater); - } - - return input; - } - - public boolean progressListenerAttached(SdkRequest request) { - if (request.overrideConfiguration().isPresent() && - !request.overrideConfiguration().get().progressListeners().isEmpty()) { - return true; - } - - return false; - } -} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java index 8bf5c4f8ecd7..b402e866a562 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java @@ -34,11 +34,10 @@ public SdkHttpFullRequest execute(SdkHttpFullRequest input, RequestExecutionCont context.requestProvider().contentLength().get() : null; if (context.executionContext().progressUpdater().isPresent()) { - context.executionContext().progressUpdater().get().requestPrepared(); - } - else { + context.executionContext().progressUpdater().get().requestPrepared(input); + } else { ProgressUpdater progressUpdater = new ProgressUpdater(context.originalRequest(), requestContentLength); - progressUpdater.requestPrepared(); + progressUpdater.requestPrepared(input); context.executionContext().toBuilder().progressUpdater(progressUpdater); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java index a096aa2ff963..8b6364a4326d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java @@ -33,7 +33,8 @@ public final class BytesReadTrackingPublisher implements Publisher { private final AtomicLong bytesRead; private ProgressUpdater progressUpdater; - public BytesReadTrackingPublisher(Publisher upstream, AtomicLong bytesRead, Optional progressUpdater) { + public BytesReadTrackingPublisher(Publisher upstream, AtomicLong bytesRead, + Optional progressUpdater) { this.upstream = upstream; this.bytesRead = bytesRead; progressUpdater.ifPresent(value -> { @@ -55,7 +56,8 @@ private static final class BytesReadTracker implements Subscriber { private final AtomicLong bytesRead; private final ProgressUpdater progressUpdater; - private BytesReadTracker(Subscriber downstream, AtomicLong bytesRead, ProgressUpdater progressUpdater) { + private BytesReadTracker(Subscriber downstream, + AtomicLong bytesRead, ProgressUpdater progressUpdater) { this.downstream = downstream; this.bytesRead = bytesRead; this.progressUpdater = progressUpdater; @@ -64,14 +66,16 @@ private BytesReadTracker(Subscriber downstream, AtomicLong b @Override public void onSubscribe(Subscription subscription) { downstream.onSubscribe(subscription); + if (progressUpdater != null) { + progressUpdater.resetBytesReceived(); + } } @Override public void onNext(ByteBuffer byteBuffer) { bytesRead.addAndGet(byteBuffer.remaining()); downstream.onNext(byteBuffer); - - if(progressUpdater != null) { + if (progressUpdater != null) { progressUpdater.incrementBytesReceived(bytesRead.get()); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisher.java new file mode 100644 index 000000000000..475469b88926 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisher.java @@ -0,0 +1,102 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.metrics; + +import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import org.reactivestreams.Publisher; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.http.async.SdkHttpContentPublisher; + +@SdkInternalApi +public class BytesSentTrackingPublisher implements SdkHttpContentPublisher { + + private final Publisher upstream; + private final AtomicLong bytesSent; + private ProgressUpdater progressUpdater; + private Long contentLength; + + public BytesSentTrackingPublisher(Publisher upstream, + ProgressUpdater progressUpdater, + Optional contentLength) { + this.upstream = upstream; + this.bytesSent = new AtomicLong(0L); + this.progressUpdater = progressUpdater; + contentLength.ifPresent(value -> { + this.contentLength = value; + }); + } + + @Override + public void subscribe(Subscriber subscriber) { + upstream.subscribe(new BytesSentTrackingPublisher.BytesSentTracker(subscriber, + bytesSent, + progressUpdater)); + } + + public long bytesSent() { + return bytesSent.get(); + } + + @Override + public Optional contentLength() { + return Optional.ofNullable(contentLength); + } + + private static final class BytesSentTracker implements Subscriber { + private final Subscriber downstream; + private final AtomicLong bytesSent; + private final ProgressUpdater progressUpdater; + + private BytesSentTracker(Subscriber downstream, AtomicLong bytesSent, + ProgressUpdater progressUpdater) { + this.downstream = downstream; + this.bytesSent = bytesSent; + this.progressUpdater = progressUpdater; + } + + @Override + public void onSubscribe(Subscription subscription) { + downstream.onSubscribe(subscription); + if (progressUpdater != null) { + progressUpdater.resetBytesSent(); + } + } + + @Override + public void onNext(ByteBuffer byteBuffer) { + downstream.onNext(byteBuffer); + bytesSent.addAndGet(byteBuffer.remaining()); + if (progressUpdater != null) { + progressUpdater.incrementBytesSent(bytesSent.get()); + } + } + + @Override + public void onError(Throwable throwable) { + downstream.onError(throwable); + } + + @Override + public void onComplete() { + downstream.onComplete(); + } + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java deleted file mode 100644 index fccc86db6d07..000000000000 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingSubscriber.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.internal.metrics; - -import java.nio.ByteBuffer; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; -import org.reactivestreams.Subscriber; -import org.reactivestreams.Subscription; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; - -@SdkInternalApi -public class BytesSentTrackingSubscriber implements Subscriber { - - private Subscriber subscriber; - private ProgressUpdater progressUpdater; - private AtomicLong bytesSent; - - public BytesSentTrackingSubscriber(Subscriber subscriber, Optional progressUpdater) { - this.subscriber = subscriber; - progressUpdater.ifPresent(value -> { - this.progressUpdater = value; - }); - } - - @Override - public void onSubscribe(Subscription subscription) { - subscriber.onSubscribe(subscription); - } - - @Override - public void onNext(ByteBuffer byteBuffer) { - subscriber.onNext(byteBuffer); - bytesSent.addAndGet(byteBuffer.remaining()); - - if (progressUpdater != null) { - progressUpdater.incrementBytesSent(bytesSent.get()); - } - } - - @Override - public void onError(Throwable throwable) { - subscriber.onError(throwable); - } - - @Override - public void onComplete() { - subscriber.onComplete(); - } -} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java index 3db626977b6a..ef6729b374f2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java @@ -26,6 +26,7 @@ import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; import software.amazon.awssdk.core.progress.listener.SdkExchangeProgress; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; +import software.amazon.awssdk.http.SdkHttpRequest; /** * ProgressUpdater exposes methods that invokes listener methods to update and store request progress state @@ -62,6 +63,10 @@ public ProgressUpdater(SdkRequest sdkRequest, .orElse(Collections.emptyList())); } + public void updateRequestContentLength(Long requestContentLength) { + requestBodyProgress.updateAndGet(b -> b.totalBytes(requestContentLength)); + } + public void updateResponseContentLength(Long responseContentLength) { responseBodyProgress.updateAndGet(b -> b.totalBytes(responseContentLength)); } @@ -74,8 +79,8 @@ public SdkExchangeProgress responseBodyProgress() { return responseBodyProgress; } - public void requestPrepared() { - listenerInvoker.requestPrepared(context); + public void requestPrepared(SdkHttpRequest httpRequest) { + listenerInvoker.requestPrepared(context.copy(b -> b.httpRequest(httpRequest))); } public void requestHeaderSent() { diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleContentPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleContentPublisherTest.java new file mode 100644 index 000000000000..ccddb23fc7f5 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleContentPublisherTest.java @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.async; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.reactivex.Flowable; +import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher; + +public class SimpleContentPublisherTest { + @Test + public void test_requestAll_calculatesCorrectTotal() { + long nElements = 1024; + int elementSize = 4; + Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); + BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0), Optional.empty()); + readFully(trackingPublisher); + + assertThat(trackingPublisher.bytesRead()).isEqualTo(nElements * elementSize); + } + + private Publisher createUpstreamPublisher(long elements, int elementSize) { + return Flowable.fromIterable(Stream.generate(() -> ByteBuffer.wrap(new byte[elementSize])) + .limit(elements) + .collect(Collectors.toList())); + } + + private void readFully(Publisher publisher) { + Flowable.fromPublisher(publisher).toList().blockingGet(); + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java index c4a1ed29f47d..de19d7bdc983 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java @@ -37,7 +37,7 @@ public SimpleRequestProviderTckTest() { @Override public Publisher createPublisher(long l) { - return new SimpleHttpContentPublisher(makeFullRequest(), Optional.empty()); + return new SimpleHttpContentPublisher(makeFullRequest()); } @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java index 3ab646a663c6..0c6294324c10 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java @@ -16,7 +16,6 @@ package software.amazon.awssdk.core.internal.http.pipeline.stages; import java.net.URI; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java index b4923e2b363e..96f6aced2ba0 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java @@ -31,6 +31,7 @@ import software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient; import software.amazon.awssdk.core.internal.http.response.ErrorDuringUnmarshallingResponseHandler; import software.amazon.awssdk.core.internal.http.response.NullErrorResponseHandler; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.signer.NoOpSigner; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; @@ -121,6 +122,22 @@ public static ExecutionContext executionContext(SdkHttpFullRequest request) { .build(); } + public static ExecutionContext executionContextWithProgressUpdater(SdkHttpFullRequest request, ProgressUpdater progressUpdater) { + InterceptorContext incerceptorContext = + InterceptorContext.builder() + .request(NoopTestRequest.builder().build()) + .httpRequest(request) + .build(); + return ExecutionContext.builder() + .signer(new NoOpSigner()) + .interceptorChain(new ExecutionInterceptorChain(Collections.emptyList())) + .executionAttributes(new ExecutionAttributes()) + .progressUpdater(progressUpdater) + .interceptorContext(incerceptorContext) + .metricCollector(MetricCollector.create("ApiCall")) + .build(); + } + private static void waitBeforeAssertOnExecutor() { try { Thread.sleep(WAIT_BEFORE_ASSERT_ON_EXECUTOR); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java index d790d8d59a5f..4becab26f6f4 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java @@ -24,7 +24,15 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; import org.reactivestreams.Publisher; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.http.NoopTestRequest; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.http.SdkHttpFullRequest; /** * Functional tests for {@link BytesReadTrackingPublisher}. @@ -59,6 +67,33 @@ public void test_requestAll_updatesInputCount() { assertThat(trackingPublisher.bytesRead()).isEqualTo(expectedRead); } + @Test + public void test_progressUpdater_invokes_incrementBytesReceived() { + int nElements = 8; + int elementSize = 2; + + long baseBytesRead = 1024; + AtomicLong bytesRead = new AtomicLong(baseBytesRead); + + ProgressListener progressListener = Mockito.mock(ProgressListener.class); + + SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() + .addProgressListener(progressListener) + .build(); + + SdkRequest request = NoopTestRequest.builder() + .overrideConfiguration(config) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(request, null); + + Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); + Publisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, bytesRead, Optional.of(progressUpdater)); + readFully(trackingPublisher); + + Mockito.verify(progressListener, Mockito.times(nElements)).responseBytesReceived(ArgumentMatchers.any()); + } + private Publisher createUpstreamPublisher(long elements, int elementSize) { return Flowable.fromIterable(Stream.generate(() -> ByteBuffer.wrap(new byte[elementSize])) .limit(elements) diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTckTest.java new file mode 100644 index 000000000000..4c1ca5ec13e5 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTckTest.java @@ -0,0 +1,58 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.metrics; + +import io.reactivex.Flowable; +import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.mockito.Mockito; +import org.reactivestreams.Publisher; +import org.reactivestreams.tck.PublisherVerification; +import org.reactivestreams.tck.TestEnvironment; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; + +public class BytesSentTrackingPublisherTckTest extends PublisherVerification { + + ProgressUpdater progressUpdater = Mockito.mock(ProgressUpdater.class); + + public BytesSentTrackingPublisherTckTest() { + super(new TestEnvironment()); + } + + @Override + public Publisher createPublisher(long l) { + return new BytesSentTrackingPublisher(createUpstreamPublisher(l), progressUpdater, Optional.empty()); + } + + @Override + public long maxElementsFromPublisher() { + return 1024; + } + + @Override + public Publisher createFailedPublisher() { + return null; + } + + private Publisher createUpstreamPublisher(long elements) { + return Flowable.fromIterable(Stream.generate(() -> ByteBuffer.wrap(new byte[1])) + .limit(elements) + .collect(Collectors.toList())); + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTest.java new file mode 100644 index 000000000000..e8a51634bc09 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTest.java @@ -0,0 +1,90 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.metrics; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.reactivex.Flowable; +import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.http.NoopTestRequest; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.progress.listener.ProgressListener; + +public class BytesSentTrackingPublisherTest { + + @Test + public void test_updatesBytesSent() { + int nElements = 8; + int elementSize = 2; + + ProgressUpdater progressUpdater = Mockito.mock(ProgressUpdater.class); + + Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); + BytesSentTrackingPublisher trackingPublisher = new BytesSentTrackingPublisher(upstreamPublisher, progressUpdater, Optional.empty()); + readFully(trackingPublisher); + + long expectedSent = nElements * elementSize; + + assertThat(trackingPublisher.bytesSent()).isEqualTo(expectedSent); + } + + @Test + public void test_progressUpdater_invokes_incrementBytesSent() { + int nElements = 8; + int elementSize = 2; + + ProgressListener progressListener = Mockito.mock(ProgressListener.class); + + SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() + .addProgressListener(progressListener) + .build(); + + SdkRequest request = NoopTestRequest.builder() + .overrideConfiguration(config) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(request, null); + + Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); + BytesSentTrackingPublisher trackingPublisher = new BytesSentTrackingPublisher(upstreamPublisher, progressUpdater, Optional.empty()); + readFully(trackingPublisher); + + long expectedSent = nElements * elementSize; + + assertThat(trackingPublisher.bytesSent()).isEqualTo(expectedSent); + Mockito.verify(progressListener, Mockito.times(nElements)).requestBytesSent(ArgumentMatchers.any()); + } + + private Publisher createUpstreamPublisher(long elements, int elementSize) { + return Flowable.fromIterable(Stream.generate(() -> ByteBuffer.wrap(new byte[elementSize])) + .limit(elements) + .collect(Collectors.toList())); + } + + private void readFully(Publisher publisher) { + Flowable.fromPublisher(publisher).toList().blockingGet(); + } +} + diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java index 0cb887561449..f4d08c19872f 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; +import java.net.URI; import java.util.Arrays; import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; @@ -36,6 +37,8 @@ import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.core.protocol.VoidSdkResponse; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; public class ProgressUpdaterTest { private CaptureProgressListener captureProgressListener; @@ -74,7 +77,7 @@ public void requestPrepared_transferredBytes_equals_zero() { .build(); ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); - progressUpdater.requestPrepared(); + progressUpdater.requestPrepared(createHttpRequest()); assertEquals(0.0, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); assertTrue(captureProgressListener.requestPrepared()); @@ -206,13 +209,37 @@ public void ratioTransferred_upload_transferredBytes(long contentLength) { .overrideConfiguration(overrideConfig) .build(); - ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, contentLength); + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.updateRequestContentLength(contentLength); progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); assertEquals((double) BYTES_TRANSFERRED / contentLength, progressUpdater.requestBodyProgress().progressSnapshot().ratioTransferred().getAsDouble(), 0.0); } + @ParameterizedTest + @MethodSource("contentLength") + public void ratioTransferred_download_transferredBytes(long contentLength) { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.updateResponseContentLength(contentLength); + progressUpdater.incrementBytesReceived(BYTES_TRANSFERRED); + assertEquals((double) BYTES_TRANSFERRED / contentLength, + progressUpdater.responseBodyProgress().progressSnapshot().ratioTransferred().getAsDouble(), 0.0); + + } + @Test public void responseHeaderReceived_transferredBytes_equals_zero() { @@ -286,7 +313,7 @@ public void attemptFailureResponseBytesReceived() { .build(); ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); - progressUpdater.requestPrepared(); + progressUpdater.requestPrepared(createHttpRequest()); progressUpdater.responseHeaderReceived(); progressUpdater.attemptFailureResponseBytesReceived(attemptFailureResponseBytesReceived); @@ -314,7 +341,7 @@ public void attemptFailure() { .build(); ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); - progressUpdater.requestPrepared(); + progressUpdater.requestPrepared(createHttpRequest()); progressUpdater.attemptFailure(attemptFailure); Mockito.verify(mockListener, times(1)).requestPrepared(ArgumentMatchers.any(ProgressListener.Context.RequestPrepared.class)); @@ -343,7 +370,7 @@ public void executionFailure() { .build(); ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); - progressUpdater.requestPrepared(); + progressUpdater.requestPrepared(createHttpRequest()); progressUpdater.executionFailure(executionFailure); @@ -357,4 +384,10 @@ public void executionFailure() { Assertions.assertEquals(captureProgressListener.exceptionCaught().getMessage(), executionFailure.getMessage()); } + + private SdkHttpFullRequest createHttpRequest() { + return SdkHttpFullRequest.builder().uri(URI.create("https://endpoint.host")) + .method(SdkHttpMethod.GET) + .build(); + } } From d3efc8ea115c9134371b96f6a8ddb44103915933 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 25 Mar 2024 22:43:10 -0700 Subject: [PATCH 36/50] Fix rebase --- .../internal/PresignRequestHandlerTest.java | 364 +++++++++++++----- 1 file changed, 276 insertions(+), 88 deletions(-) diff --git a/services/rds/src/test/java/software/amazon/awssdk/services/rds/internal/PresignRequestHandlerTest.java b/services/rds/src/test/java/software/amazon/awssdk/services/rds/internal/PresignRequestHandlerTest.java index 0b4d489efe38..fff1e649fc40 100644 --- a/services/rds/src/test/java/software/amazon/awssdk/services/rds/internal/PresignRequestHandlerTest.java +++ b/services/rds/src/test/java/software/amazon/awssdk/services/rds/internal/PresignRequestHandlerTest.java @@ -15,32 +15,39 @@ package software.amazon.awssdk.services.rds.internal; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; +import java.io.IOException; +import java.io.UncheckedIOException; import java.net.URI; -import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.time.Clock; -import java.util.Calendar; -import java.util.GregorianCalendar; -import java.util.TimeZone; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; -import software.amazon.awssdk.awscore.endpoint.DefaultServiceEndpointBuilder; -import software.amazon.awssdk.core.Protocol; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.interceptor.Context; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; -import software.amazon.awssdk.core.interceptor.InterceptorContext; -import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpRequest; -import software.amazon.awssdk.profiles.ProfileFile; +import software.amazon.awssdk.http.auth.aws.scheme.AwsV4AuthScheme; +import software.amazon.awssdk.http.auth.aws.signer.AwsV4FamilyHttpSigner; +import software.amazon.awssdk.http.auth.spi.scheme.AuthSchemeOption; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.rds.RdsClient; import software.amazon.awssdk.services.rds.RdsClientBuilder; @@ -53,96 +60,277 @@ /** * Unit Tests for {@link RdsPresignInterceptor} */ -public class PresignRequestHandlerTest { - private static final AwsBasicCredentials CREDENTIALS = AwsBasicCredentials.create("foo", "bar"); - private static final Region DESTINATION_REGION = Region.of("us-west-2"); - - private static RdsPresignInterceptor presignInterceptor = new CopyDbSnapshotPresignInterceptor(); - private final CopyDbSnapshotRequestMarshaller marshaller = - new CopyDbSnapshotRequestMarshaller(RdsPresignInterceptor.PROTOCOL_FACTORY); +class PresignRequestHandlerTest { + private static String TEST_KMS_KEY_ID = "arn:aws:kms:us-west-2:123456789012:key/" + + "11111111-2222-3333-4444-555555555555"; + + @ParameterizedTest + @MethodSource("testCases") + public void testExpectations(TestCase testCase) { + // Arrange + CapturingInterceptor interceptor = new CapturingInterceptor(); + RdsClientBuilder clientBuilder = client(interceptor, testCase.signingClockOverride); + testCase.clientConfigure.accept(clientBuilder); + RdsClient client = clientBuilder.build(); + + // Act + assertThatThrownBy(() -> testCase.clientConsumer.accept(client)) + .hasMessageContaining("boom!"); + + // Assert + SdkHttpFullRequest request = (SdkHttpFullRequest) interceptor.httpRequest(); + Map> rawQueryParameters = rawQueryParameters(request); + + // The following params should not be included in the outgoing request + assertFalse(rawQueryParameters.containsKey("SourceRegion")); + assertFalse(rawQueryParameters.containsKey("DestinationRegion")); + + if (testCase.shouldContainPreSignedUrl) { + List rawPresignedUrlValue = rawQueryParameters.get("PreSignedUrl"); + assertNotNull(rawPresignedUrlValue); + assertTrue(rawPresignedUrlValue.size() == 1); + String presignedUrl = rawPresignedUrlValue.get(0); + assertNotNull(presignedUrl); + // Validate that the URL can be parsed back + URI presignedUrlAsUri = URI.create(presignedUrl); + assertNotNull(presignedUrlAsUri); + if (testCase.expectedDestinationRegion != null) { + assertTrue(presignedUrl.contains("DestinationRegion=" + testCase.expectedDestinationRegion)); + } + if (testCase.expectedUri != null) { + assertEquals(normalize(URI.create(testCase.expectedUri)), normalize(presignedUrlAsUri)); + } + } else { + assertFalse(rawQueryParameters.containsKey("PreSignedUrl")); + } + } - @Test - public void testSetsPresignedUrl() { - CopyDbSnapshotRequest request = makeTestRequest(); - SdkHttpRequest presignedRequest = modifyHttpRequest(presignInterceptor, request, marshallRequest(request)); + public static List testCases() { + return Arrays.asList( + builder("CopyDbClusterSnapshot - Sets pre-signed URL when sourceRegion is set") + .clientConsumer(c -> c.copyDBClusterSnapshot(makeTestRequestBuilder() + .sourceRegion("us-east-1") + .build())) + .shouldContainPreSignedUrl(true) + .expectedDestinationRegion("us-east-1") + .build(), + builder("CopyDbClusterSnapshot - Doesn't set pre-signed URL when sourceRegion is NOT set") + .clientConsumer(c -> c.copyDBClusterSnapshot(makeTestRequestBuilder().build())) + .shouldContainPreSignedUrl(false) + .build(), + builder("CopyDbClusterSnapshot - Does not override pre-signed URL") + .clientConsumer(c -> c.copyDBClusterSnapshot( + makeTestRequestBuilder() + .sourceRegion("us-west-2") + .preSignedUrl("http://localhost?foo=bar") + .build())) + .shouldContainPreSignedUrl(true) + .expectedUri("http://localhost?foo=bar") + .build(), + builder("CopyDbClusterSnapshot - Fixed time") + .clientConfigure(c -> c.region(Region.US_WEST_2)) + .clientConsumer(c -> c.copyDBClusterSnapshot( + makeTestRequestBuilder() + .sourceRegion("us-east-1") + .build())) + .shouldContainPreSignedUrl(true) + .signingClockOverride(Clock.fixed(Instant.parse("2016-12-21T18:07:35.000Z"), ZoneId.of("UTC"))) + .expectedUri(fixedTimePresignedUrl()) + .build(), + + builder("CreateDBCluster With SourceRegion Sends Presigned Url") + .clientConsumer(c -> c.createDBCluster(r -> r.kmsKeyId(TEST_KMS_KEY_ID) + .sourceRegion("us-west-2"))) + .shouldContainPreSignedUrl(true) + .expectedDestinationRegion("us-east-1") + .build(), + builder("CreateDBCluster Without SourceRegion Does NOT Send PresignedUrl") + .clientConsumer(c -> c.createDBCluster(r -> r.kmsKeyId(TEST_KMS_KEY_ID))) + .shouldContainPreSignedUrl(false) + .build(), + + builder("CreateDBInstanceReadReplica With SourceRegion Sends Presigned Url") + .clientConsumer(c -> c.createDBInstanceReadReplica(r -> r.kmsKeyId(TEST_KMS_KEY_ID) + .sourceRegion("us-west-2"))) + .shouldContainPreSignedUrl(true) + .expectedDestinationRegion("us-east-1") + .build(), + builder("CreateDBInstanceReadReplica Without SourceRegion Does NOT Send PresignedUrl") + .clientConsumer(c -> c.createDBInstanceReadReplica(r -> r.kmsKeyId(TEST_KMS_KEY_ID))) + .shouldContainPreSignedUrl(false) + .build(), + + builder("StartDBInstanceAutomatedBackupsReplication With SourceRegion Sends Presigned Url") + .clientConsumer(c -> c.startDBInstanceAutomatedBackupsReplication(r -> r.kmsKeyId(TEST_KMS_KEY_ID) + .sourceRegion("us-west-2"))) + .shouldContainPreSignedUrl(true) + .expectedDestinationRegion("us-east-1") + .build(), + builder("StartDBInstanceAutomatedBackupsReplication Without SourceRegion Does NOT Send PresignedUrl") + .clientConsumer(c -> c.startDBInstanceAutomatedBackupsReplication(r -> r.kmsKeyId(TEST_KMS_KEY_ID))) + .shouldContainPreSignedUrl(false) + .build() + ); + } - assertNotNull(presignedRequest.rawQueryParameters().get("PreSignedUrl").get(0)); + private static CopyDbClusterSnapshotRequest.Builder makeTestRequestBuilder() { + return CopyDbClusterSnapshotRequest + .builder() + .sourceDBClusterSnapshotIdentifier("arn:aws:rds:us-east-1:123456789012:snapshot:rds" + + ":test-instance-ss-2016-12-20-23-19") + .targetDBClusterSnapshotIdentifier("test-instance-ss-copy-2") + .kmsKeyId(TEST_KMS_KEY_ID); } - @Test - public void testComputesPresignedUrlCorrectly() { - // Note: test data was baselined by performing actual calls, with real - // credentials to RDS and checking that they succeeded. Then the - // request was recreated with all the same parameters but with test - // credentials. - final CopyDbSnapshotRequest request = CopyDbSnapshotRequest.builder() - .sourceDBSnapshotIdentifier("arn:aws:rds:us-east-1:123456789012:snapshot:rds:test-instance-ss-2016-12-20-23-19") - .targetDBSnapshotIdentifier("test-instance-ss-copy-2") - .sourceRegion("us-east-1") - .kmsKeyId("arn:aws:kms:us-west-2:123456789012:key/11111111-2222-3333-4444-555555555555") - .build(); - - Calendar c = new GregorianCalendar(); - c.setTimeZone(TimeZone.getTimeZone("UTC")); - // 20161221T180735Z - // Note: month is 0-based - c.set(2016, Calendar.DECEMBER, 21, 18, 7, 35); - - Clock signingDateOverride = Mockito.mock(Clock.class); - when(signingDateOverride.millis()).thenReturn(c.getTimeInMillis()); - - RdsPresignInterceptor interceptor = new CopyDbSnapshotPresignInterceptor(signingDateOverride); - - SdkHttpRequest presignedRequest = modifyHttpRequest(interceptor, request, marshallRequest(request)); - - final String expectedPreSignedUrl = "https://rds.us-east-1.amazonaws.com?" + - "Action=CopyDBSnapshot" + - "&Version=2014-10-31" + - "&SourceDBSnapshotIdentifier=arn%3Aaws%3Ards%3Aus-east-1%3A123456789012%3Asnapshot%3Ards%3Atest-instance-ss-2016-12-20-23-19" + - "&TargetDBSnapshotIdentifier=test-instance-ss-copy-2" + - "&KmsKeyId=arn%3Aaws%3Akms%3Aus-west-2%3A123456789012%3Akey%2F11111111-2222-3333-4444-555555555555" + - "&DestinationRegion=us-west-2" + - "&X-Amz-Algorithm=AWS4-HMAC-SHA256" + - "&X-Amz-Date=20161221T180735Z" + - "&X-Amz-SignedHeaders=host" + - "&X-Amz-Expires=604800" + - "&X-Amz-Credential=foo%2F20161221%2Fus-east-1%2Frds%2Faws4_request" + - "&X-Amz-Signature=f839ca3c728dc96e7c978befeac648296b9f778f6724073de4217173859d13d9"; - - assertEquals(expectedPreSignedUrl, presignedRequest.rawQueryParameters().get("PreSignedUrl").get(0)); + private static RdsClientBuilder client(CapturingInterceptor interceptor, Clock signingClockOverride) { + RdsClientBuilder builder = RdsClient + .builder() + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create("foo", "bar"))) + .region(Region.US_EAST_1) + .addPlugin(c -> { + // Adds the capturing interceptor. + RdsServiceClientConfiguration.Builder config = + Validate.isInstanceOf(RdsServiceClientConfiguration.Builder.class, c, + "\uD83E\uDD14"); + config.overrideConfiguration(oc -> oc.addExecutionInterceptor(interceptor)); + }); + + if (signingClockOverride != null) { + // Adds a auth scheme wrapper that handles the clock override + builder.addPlugin(c -> { + RdsServiceClientConfiguration.Builder config = + Validate.isInstanceOf(RdsServiceClientConfiguration.Builder.class, c, "\uD83E\uDD14"); + config.authSchemeProvider(clockOverridingAuthScheme(config.authSchemeProvider(), signingClockOverride)); + }); + } + return builder; } - @Test - public void testSkipsPresigningIfUrlSet() { - CopyDbSnapshotRequest request = CopyDbSnapshotRequest.builder() - .sourceRegion("us-west-2") - .preSignedUrl("PRESIGNED") - .build(); + private static RdsAuthSchemeProvider clockOverridingAuthScheme(RdsAuthSchemeProvider source, Clock signingClockOverride) { + return authSchemeParams -> { + List authSchemeOptions = source.resolveAuthScheme(authSchemeParams); + List result = new ArrayList<>(authSchemeOptions.size()); + for (AuthSchemeOption option : authSchemeOptions) { + if (option.schemeId().equals(AwsV4AuthScheme.SCHEME_ID)) { + option = option.toBuilder() + .putSignerProperty(AwsV4FamilyHttpSigner.SIGNING_CLOCK, signingClockOverride) + .build(); + } + result.add(option); + } + return result; + }; + } + static String fixedTimePresignedUrl() { + return + "https://rds.us-east-1.amazonaws.com?" + + "Action=CopyDBClusterSnapshot" + + "&Version=2014-10-31" + + "&SourceDBClusterSnapshotIdentifier=arn%3Aaws%3Ards%3Aus-east-1%3A123456789012" + + "%3Asnapshot%3Ards%3Atest-instance-ss-2016-12-20-23-19" + + "&TargetDBClusterSnapshotIdentifier=test-instance-ss-copy-2" + + "&KmsKeyId=arn%3Aaws%3Akms%3Aus-west-2%3A123456789012%3Akey%2F11111111-2222-3333" + + "-4444-555555555555" + + "&DestinationRegion=us-west-2" + + "&X-Amz-Algorithm=AWS4-HMAC-SHA256" + + "&X-Amz-Date=20161221T180735Z" + + "&X-Amz-SignedHeaders=host" + + "&X-Amz-Credential=foo%2F20161221%2Fus-east-1%2Frds%2Faws4_request" + + "&X-Amz-Expires=604800" + + "&X-Amz-Signature=00822ebbba95e2e6ac09112aa85621fbef060a596e3e1480f9f4ac61493e9821"; + } - SdkHttpRequest presignedRequest = modifyHttpRequest(presignInterceptor, request, marshallRequest(request)); + private Map> rawQueryParameters(SdkHttpFullRequest request) { + // Retrieve back from the query parameters from the body, this is best-effort only. + try { + String decodedQueryParams = IoUtils.toUtf8String(request.contentStreamProvider().get().newStream()); + String[] keyValuePairs = decodedQueryParams.split("&"); + Map> result = new LinkedHashMap<>(); + for (String keyValuePair : keyValuePairs) { + String[] kvpParts = keyValuePair.split("=", 2); + String value = URLDecoder.decode(kvpParts.length > 1 ? kvpParts[1] : "", StandardCharsets.UTF_8.name()); + result.computeIfAbsent(kvpParts[0], x -> new ArrayList<>()).add(value); + } + return result; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - assertEquals("PRESIGNED", presignedRequest.rawQueryParameters().get("PreSignedUrl").get(0)); + static TestCaseBuilder builder(String name) { + return new TestCaseBuilder() + .clientConfigure(c -> { + }) + .name(name); } - @Test - public void testSkipsPresigningIfSourceRegionNotSet() { - CopyDbSnapshotRequest request = CopyDbSnapshotRequest.builder().build(); + private static String normalize(URI uri) { + String uriAsString = uri.toString(); + int queryStart = uriAsString.indexOf('?'); + if (queryStart == -1) { + return uriAsString; + } + String uriQueryPrefix = uriAsString.substring(0, queryStart); + String query = uri.getQuery(); + if (query == null) { + return uriAsString; + } + if (!query.isEmpty()) { + String[] queryParts = query.split("&"); + query = Arrays.stream(queryParts) + .sorted() + .collect(Collectors.joining("&")); - SdkHttpRequest presignedRequest = modifyHttpRequest(presignInterceptor, request, marshallRequest(request)); + } + return uriQueryPrefix + "?" + query; + } - assertNull(presignedRequest.rawQueryParameters().get("PreSignedUrl")); + static class TestCase { + private final String name; + private final Consumer clientConfigure; + private final Consumer clientConsumer; + private final Boolean shouldContainPreSignedUrl; + private final String expectedDestinationRegion; + private final Clock signingClockOverride; + private final String expectedUri; + + TestCase(TestCaseBuilder builder) { + this.name = Validate.notNull(builder.name, "name"); + this.clientConsumer = Validate.notNull(builder.clientConsumer, "clientConsumer"); + this.clientConfigure = Validate.notNull(builder.clientConfigure, "clientConfigure"); + this.shouldContainPreSignedUrl = builder.shouldContainPreSignedUrl; + this.expectedDestinationRegion = builder.expectedDestinationRegion; + this.signingClockOverride = builder.signingClockOverride; + this.expectedUri = builder.expectedUri; + } } - @Test - public void testParsesDestinationRegionfromRequestEndpoint() throws URISyntaxException { - CopyDbSnapshotRequest request = CopyDbSnapshotRequest.builder() - .sourceRegion("us-east-1") - .build(); - Region destination = Region.of("us-west-2"); - SdkHttpFullRequest marshalled = marshallRequest(request); + static class TestCaseBuilder { + private String name; + private Consumer clientConfigure; + private Consumer clientConsumer; + private Boolean shouldContainPreSignedUrl; + private String expectedDestinationRegion; + private Clock signingClockOverride; + private String expectedUri; + + private TestCaseBuilder name(String name) { + this.name = name; + return this; + } + + private TestCaseBuilder clientConfigure(Consumer clientConfigure) { + this.clientConfigure = clientConfigure; + return this; + } - final SdkHttpRequest presignedRequest = modifyHttpRequest(presignInterceptor, request, marshalled); + private TestCaseBuilder clientConsumer(Consumer clientConsumer) { + this.clientConsumer = clientConsumer; + return this; + } private TestCaseBuilder shouldContainPreSignedUrl(Boolean value) { this.shouldContainPreSignedUrl = value; From 98a2a85022261e63de7ece593343b80dfb2cb88a Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 25 Mar 2024 23:35:57 -0700 Subject: [PATCH 37/50] Fix method call --- .../amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java b/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java index f5443b00990f..73a70ab4ea02 100644 --- a/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java +++ b/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java @@ -61,8 +61,7 @@ private static CompletableFuture sendAsync(SdkAsyncHttpClient client, SdkHttpFullRequest request, HttpResponseHandler handler, CompletableFuture parentFuture) { - SdkHttpContentPublisher requestContentPublisher = new SimpleHttpContentPublisher(request, - Optional.empty()); + SdkHttpContentPublisher requestContentPublisher = new SimpleHttpContentPublisher(request); TransformingAsyncResponseHandler responseHandler = new AsyncResponseHandler<>(handler, Function.identity(), new ExecutionAttributes()); CompletableFuture responseHandlerFuture = responseHandler.prepare(); From 51ac00322cc76c8bd2b4732f5e370687b98a6c83 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 25 Mar 2024 23:40:40 -0700 Subject: [PATCH 38/50] Remove unintentionally added test class --- .../async/SimpleContentPublisherTest.java | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleContentPublisherTest.java diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleContentPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleContentPublisherTest.java deleted file mode 100644 index ccddb23fc7f5..000000000000 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleContentPublisherTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.internal.http.async; - -import static org.assertj.core.api.Assertions.assertThat; - -import io.reactivex.Flowable; -import java.nio.ByteBuffer; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.junit.jupiter.api.Test; -import org.reactivestreams.Publisher; -import software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher; - -public class SimpleContentPublisherTest { - @Test - public void test_requestAll_calculatesCorrectTotal() { - long nElements = 1024; - int elementSize = 4; - Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0), Optional.empty()); - readFully(trackingPublisher); - - assertThat(trackingPublisher.bytesRead()).isEqualTo(nElements * elementSize); - } - - private Publisher createUpstreamPublisher(long elements, int elementSize) { - return Flowable.fromIterable(Stream.generate(() -> ByteBuffer.wrap(new byte[elementSize])) - .limit(elements) - .collect(Collectors.toList())); - } - - private void readFully(Publisher publisher) { - Flowable.fromPublisher(publisher).toList().blockingGet(); - } -} From 210c8ed34edc566d351fca54874f83b93d68a0f6 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Tue, 26 Mar 2024 12:42:41 -0700 Subject: [PATCH 39/50] Remove needles indentation changes --- .../http/pipeline/stages/MakeAsyncHttpRequestStage.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index e011de36fb8a..20ecde1c8347 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -133,11 +133,8 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque CompletableFuture> responseHandlerFuture = responseHandler.prepare(); SdkHttpContentPublisher requestProvider = context.requestProvider() == null - ? new SimpleHttpContentPublisher( - request) - : new SdkHttpContentPublisherAdapter( - context.requestProvider()); - + ? new SimpleHttpContentPublisher(request) + : new SdkHttpContentPublisherAdapter(context.requestProvider()); // Set content length if it hasn't been set already. SdkHttpFullRequest requestWithContentLength = getRequestWithContentLength(request, requestProvider); From d5f1258999d6e206005d84b0087132ca87f73005 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 22 Apr 2024 18:32:02 -0700 Subject: [PATCH 40/50] WIP --- .../awssdk/core/http/ExecutionContext.java | 15 -- .../internal/http/AmazonAsyncHttpClient.java | 8 +- .../internal/http/AmazonSyncHttpClient.java | 8 +- .../http/RequestExecutionContext.java | 11 ++ ...AfterExecutionProgressReportingStage.java} | 8 +- ...ecutionFailureExceptionReportingStage.java | 11 +- ...eforeExecutionProgressReportingStage.java} | 24 +--- ...ecutionFailureExceptionReportingStage.java | 5 +- .../stages/MakeAsyncHttpRequestStage.java | 35 ++--- .../stages/UnwrapResponseContainer.java | 3 - .../stages/utils/ExceptionReportingUtils.java | 19 ++- .../internal/util/ProgressListenerUtils.java | 76 ++++++++++ ...entHandlerTransformerVerificationTest.java | 1 - ...erExecutionProgressReportingStageTest.java | 64 +++++++++ ...reExecutionProgressReportingStageTest.java | 61 ++++++++ .../stages/ProgressUpdateStageTest.java | 135 ------------------ ...ientExecutionAndRequestTimerTestUtils.java | 16 --- .../BytesReadTrackingPublisherTest.java | 6 +- .../util/ProgressListenerTestUtils.java | 70 +++++++++ 19 files changed, 339 insertions(+), 237 deletions(-) rename core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/{PostExecutionUpdateProgressStage.java => AfterExecutionProgressReportingStage.java} (78%) rename core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/{PreExecutionUpdateProgressStage.java => BeforeExecutionProgressReportingStage.java} (60%) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStageTest.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStageTest.java delete mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java index 01cc3e0c5166..141ba473986f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java @@ -15,13 +15,11 @@ package software.amazon.awssdk.core.http; -import java.util.Optional; import software.amazon.awssdk.annotations.NotThreadSafe; import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain; import software.amazon.awssdk.core.interceptor.InterceptorContext; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.signer.Signer; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.builder.CopyableBuilder; @@ -38,7 +36,6 @@ public final class ExecutionContext implements ToCopyableBuilder progressUpdater() { - return progressUpdater != null ? Optional.of(progressUpdater) : Optional.empty(); - } - @Override public Builder toBuilder() { return new Builder(this); @@ -95,7 +87,6 @@ public static class Builder implements CopyableBuilder CompletableFuture execute( .then(() -> new HttpChecksumStage(ClientType.ASYNC)) .then(MakeRequestImmutableStage::new) .then(RequestPipelineBuilder - .first(PreExecutionUpdateProgressStage::new) + .first(BeforeExecutionProgressReportingStage::new) .then(AsyncSigningStage::new) .then(AsyncBeforeTransmissionExecutionInterceptorsStage::new) .then(d -> new MakeAsyncHttpRequestStage<>(responseHandler, d)) .wrappedWith(AsyncApiCallAttemptMetricCollectionStage::new) .wrappedWith((deps, wrapped) -> new AsyncRetryableStage<>(responseHandler, deps, wrapped)) .then(async(() -> new UnwrapResponseContainer<>())) - .then(() -> new PostExecutionUpdateProgressStage<>()) + .then(() -> new AfterExecutionProgressReportingStage<>()) .then(async(() -> new AfterExecutionInterceptorsStage<>())) .wrappedWith(AsyncExecutionFailureExceptionReportingStage::new) .wrappedWith(AsyncApiCallTimeoutTrackingStage::new) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java index 41c9dcf2eea7..94ed5892037b 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java @@ -28,6 +28,7 @@ import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder; import software.amazon.awssdk.core.internal.http.pipeline.stages.AfterExecutionInterceptorsStage; +import software.amazon.awssdk.core.internal.http.pipeline.stages.AfterExecutionProgressReportingStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.AfterTransmissionExecutionInterceptorsStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage; @@ -35,6 +36,7 @@ import software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.ApplyTransactionIdStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.ApplyUserAgentStage; +import software.amazon.awssdk.core.internal.http.pipeline.stages.BeforeExecutionProgressReportingStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.BeforeTransmissionExecutionInterceptorsStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.BeforeUnmarshallingExecutionInterceptorsStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.CompressRequestStage; @@ -46,8 +48,6 @@ import software.amazon.awssdk.core.internal.http.pipeline.stages.MakeRequestMutableStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.MergeCustomHeadersStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.MergeCustomQueryParamsStage; -import software.amazon.awssdk.core.internal.http.pipeline.stages.PostExecutionUpdateProgressStage; -import software.amazon.awssdk.core.internal.http.pipeline.stages.PreExecutionUpdateProgressStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.QueryParametersToBodyStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.SigningStage; @@ -206,7 +206,7 @@ public OutputT execute(HttpResponseHandler> response .then(MakeRequestImmutableStage::new) // End of mutating request .then(RequestPipelineBuilder - .first(PreExecutionUpdateProgressStage::new) + .first(BeforeExecutionProgressReportingStage::new) .then(SigningStage::new) .then(BeforeTransmissionExecutionInterceptorsStage::new) .then(MakeHttpRequestStage::new) @@ -221,8 +221,8 @@ public OutputT execute(HttpResponseHandler> response .wrappedWith(ApiCallTimeoutTrackingStage::new)::build) .wrappedWith((deps, wrapped) -> new ApiCallMetricCollectionStage<>(wrapped)) .then(() -> new UnwrapResponseContainer<>()) + .then(() -> new AfterExecutionProgressReportingStage<>()) .then(() -> new AfterExecutionInterceptorsStage<>()) - .then(() -> new PostExecutionUpdateProgressStage<>()) .wrappedWith(ExecutionFailureExceptionReportingStage::new) .build(httpClientDependencies) .execute(request, createRequestExecutionDependencies()); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/RequestExecutionContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/RequestExecutionContext.java index 73d9f2ea94f4..e7ea0b4b3e4f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/RequestExecutionContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/RequestExecutionContext.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.core.internal.http; +import java.util.Optional; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.RequestOverrideConfiguration; import software.amazon.awssdk.core.SdkRequest; @@ -26,6 +27,7 @@ import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; import software.amazon.awssdk.core.internal.http.timers.TimeoutTracker; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.signer.Signer; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.Validate; @@ -44,6 +46,7 @@ public final class RequestExecutionContext { private TimeoutTracker apiCallTimeoutTracker; private TimeoutTracker apiCallAttemptTimeoutTracker; private MetricCollector attemptMetricCollector; + private ProgressUpdater progressUpdater; private RequestExecutionContext(Builder builder) { this.requestProvider = builder.requestProvider; @@ -127,6 +130,14 @@ public void attemptMetricCollector(MetricCollector metricCollector) { this.attemptMetricCollector = metricCollector; } + public Optional progressUpdater() { + return progressUpdater != null ? Optional.of(progressUpdater) : Optional.empty(); + } + + public void progressUpdater(ProgressUpdater progressUpdater) { + this.progressUpdater = progressUpdater; + } + /** * Sets the request body provider. * Used for transforming the original body provider to sign events for diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionUpdateProgressStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStage.java similarity index 78% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionUpdateProgressStage.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStage.java index c887be837e60..5c53475103c9 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PostExecutionUpdateProgressStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStage.java @@ -19,15 +19,13 @@ import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; +import software.amazon.awssdk.core.internal.util.ProgressListenerUtils; @SdkInternalApi -public class PostExecutionUpdateProgressStage implements RequestPipeline { +public class AfterExecutionProgressReportingStage implements RequestPipeline { @Override public OutputT execute(OutputT input, RequestExecutionContext context) throws Exception { - - context.executionContext().progressUpdater().ifPresent(progressUpdater -> { - progressUpdater.executionSuccess((SdkResponse) input); - }); + ProgressListenerUtils.updateProgressListenersWithSuccessResponse((SdkResponse) input, context); return input; } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncExecutionFailureExceptionReportingStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncExecutionFailureExceptionReportingStage.java index 8c2a60b7f8a2..84b500fdb779 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncExecutionFailureExceptionReportingStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncExecutionFailureExceptionReportingStage.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.core.internal.http.pipeline.stages; import static software.amazon.awssdk.core.internal.http.pipeline.stages.utils.ExceptionReportingUtils.reportFailureToInterceptors; +import static software.amazon.awssdk.core.internal.http.pipeline.stages.utils.ExceptionReportingUtils.reportFailureToProgressListeners; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -47,14 +48,14 @@ public CompletableFuture execute(SdkHttpFullRequest input, RequestExecu } toReport = reportFailureToInterceptors(context, toReport); - context.executionContext().progressUpdater().ifPresent(progressUpdater -> { - progressUpdater.attemptFailure(t); - }); + // If Progress Listeners are attached to the request, update them with the throwable + if (context.progressUpdater().isPresent()) { + reportFailureToProgressListeners(context.progressUpdater().get(), toReport); + } throw CompletableFutureUtils.errorAsCompletionException(ThrowableUtils.asSdkException(toReport)); - } else { - return o; } + return o; }); return CompletableFutureUtils.forwardExceptionTo(executeFuture, wrappedExecute); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStage.java similarity index 60% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStage.java index b402e866a562..1f86cc0d3860 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/PreExecutionUpdateProgressStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStage.java @@ -16,40 +16,28 @@ package software.amazon.awssdk.core.internal.http.pipeline.stages; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.RequestToRequestPipeline; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.internal.util.ProgressListenerUtils; import software.amazon.awssdk.http.SdkHttpFullRequest; @SdkInternalApi -public class PreExecutionUpdateProgressStage implements RequestToRequestPipeline { +public class BeforeExecutionProgressReportingStage implements RequestToRequestPipeline { @Override public SdkHttpFullRequest execute(SdkHttpFullRequest input, RequestExecutionContext context) throws Exception { - if (progressListenerAttached(context.originalRequest())) { + if (ProgressListenerUtils.progressListenerAttached(context.originalRequest())) { Long requestContentLength = context.requestProvider() != null && context.requestProvider().contentLength().isPresent() ? context.requestProvider().contentLength().get() : null; - if (context.executionContext().progressUpdater().isPresent()) { - context.executionContext().progressUpdater().get().requestPrepared(input); - } else { - ProgressUpdater progressUpdater = new ProgressUpdater(context.originalRequest(), requestContentLength); - progressUpdater.requestPrepared(input); - context.executionContext().toBuilder().progressUpdater(progressUpdater); - } + ProgressUpdater progressUpdater = new ProgressUpdater(context.originalRequest(), requestContentLength); + progressUpdater.requestPrepared(input); + context.progressUpdater(progressUpdater); } return input; } - - public boolean progressListenerAttached(SdkRequest request) { - if (request.overrideConfiguration().isPresent() && - !request.overrideConfiguration().get().progressListeners().isEmpty()) { - return true; - } - return false; - } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java index 8929d6799170..6b1086ed7bf8 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.core.internal.http.pipeline.stages; import static software.amazon.awssdk.core.internal.http.pipeline.stages.utils.ExceptionReportingUtils.reportFailureToInterceptors; +import static software.amazon.awssdk.core.internal.http.pipeline.stages.utils.ExceptionReportingUtils.reportFailureToProgressListeners; import static software.amazon.awssdk.core.internal.util.ThrowableUtils.failure; import software.amazon.awssdk.annotations.SdkInternalApi; @@ -38,8 +39,8 @@ public OutputT execute(SdkHttpFullRequest input, RequestExecutionContext context } catch (Exception e) { Throwable throwable = reportFailureToInterceptors(context, e); - context.executionContext().progressUpdater().ifPresent(progressUpdater -> { - progressUpdater.attemptFailure(e); + context.progressUpdater().ifPresent(progressUpdater -> { + reportFailureToProgressListeners(progressUpdater, throwable); }); throw failure(throwable); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index 20ecde1c8347..c7c8e706b417 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -47,9 +47,8 @@ import software.amazon.awssdk.core.internal.http.timers.TimeoutTracker; import software.amazon.awssdk.core.internal.http.timers.TimerUtils; import software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher; -import software.amazon.awssdk.core.internal.metrics.BytesSentTrackingPublisher; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.internal.util.MetricUtils; +import software.amazon.awssdk.core.internal.util.ProgressListenerUtils; import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; @@ -60,7 +59,6 @@ import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Logger; -import software.amazon.awssdk.utils.StringUtils; /** * Delegate to the HTTP implementation to make an HTTP request and receive the response. @@ -141,18 +139,11 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque MetricCollector httpMetricCollector = MetricUtils.createHttpMetricsCollector(context); //If Progress Listening is enabled, wrap around BytesSentTrackingPublisher to track progress on bytes sent - if (context.executionContext().progressUpdater().isPresent()) { - - ProgressUpdater progressUpdater = context.executionContext().progressUpdater().get(); - - if (shouldSetContentLength(request, requestProvider)) { - progressUpdater.updateRequestContentLength(requestProvider.contentLength().get()); - } - - requestProvider = new BytesSentTrackingPublisher(requestProvider, - progressUpdater, - requestProvider.contentLength()); - + if (context.progressUpdater().isPresent()) { + boolean shouldSetContentLength = shouldSetContentLength(request, requestProvider); + requestProvider = ProgressListenerUtils.updateProgressListenersWithRequestStatus(context.progressUpdater().get(), + requestProvider, + shouldSetContentLength); } AsyncExecuteRequest.Builder executeRequestBuilder = AsyncExecuteRequest.builder() @@ -322,25 +313,19 @@ public void onHeaders(SdkHttpResponse headers) { context.attemptMetricCollector().reportMetric(CoreMetric.TIME_TO_FIRST_BYTE, Duration.ofNanos(d)); super.onHeaders(headers); - context.executionContext().progressUpdater().ifPresent(progressUpdater -> { - progressUpdater.responseHeaderReceived(); - headers.firstMatchingHeader(CONTENT_LENGTH).ifPresent(value -> { - if (!StringUtils.isNotBlank(value)) { - progressUpdater.updateResponseContentLength(Long.parseLong(value)); - } - }); + context.progressUpdater().ifPresent(progressUpdater -> { + ProgressListenerUtils.updateProgressListenersWithResponseStatus(progressUpdater, headers); }); } @Override public void onStream(Publisher stream) { - BytesReadTrackingPublisher bytesReadTrackingPublisher; AtomicLong bytesReadCounter = context.executionAttributes() .getAttribute(SdkInternalExecutionAttribute.RESPONSE_BYTES_READ); - bytesReadTrackingPublisher = new BytesReadTrackingPublisher(stream, + BytesReadTrackingPublisher bytesReadTrackingPublisher = new BytesReadTrackingPublisher(stream, bytesReadCounter, - context.executionContext().progressUpdater()); + context.progressUpdater()); super.onStream(bytesReadTrackingPublisher); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java index 0abb9915f18b..be713469621a 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java @@ -32,9 +32,6 @@ public class UnwrapResponseContainer implements RequestPipeline, OutputT> { @Override public OutputT execute(Response input, RequestExecutionContext context) throws Exception { - context.executionContext().progressUpdater().ifPresent(progressUpdater -> { - progressUpdater.executionSuccess((SdkResponse) input.response()); - }); return input.response(); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/utils/ExceptionReportingUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/utils/ExceptionReportingUtils.java index 06a4b2544fc7..d16b19e05e8f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/utils/ExceptionReportingUtils.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/utils/ExceptionReportingUtils.java @@ -18,6 +18,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.interceptor.DefaultFailedExecutionContext; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.utils.Logger; @SdkInternalApi @@ -28,7 +29,8 @@ private ExceptionReportingUtils() { } /** - * Report the failure to the execution interceptors. Swallow any exceptions thrown from the interceptor since + * Report the failure to the execution interceptors and progress listeners if present. + * Swallow any exceptions thrown from the interceptor since * we don't want to replace the execution failure. * * @param context The execution context. @@ -46,6 +48,21 @@ public static Throwable reportFailureToInterceptors(RequestExecutionContext cont return modifiedContext.exception(); } + /** + * Report the failure to the Progress Listeners if they are present + * + * @param context The execution context. + * @param failure The execution failure. + */ + public static void reportFailureToProgressListeners(ProgressUpdater progressUpdater, Throwable failure) { + + try { + progressUpdater.attemptFailure(failure); + } catch (Exception exception) { + log.warn(() -> "Progess Listener update threw an error while invoking attemptFailure().", exception); + } + } + private static DefaultFailedExecutionContext runModifyException(RequestExecutionContext context, Throwable e) { DefaultFailedExecutionContext failedContext = DefaultFailedExecutionContext.builder() diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java new file mode 100644 index 000000000000..04d6f0f660e2 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java @@ -0,0 +1,76 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.util; + +import static software.amazon.awssdk.http.Header.CONTENT_LENGTH; + +import java.util.List; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.RequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.metrics.BytesSentTrackingPublisher; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.http.async.SdkHttpContentPublisher; +import software.amazon.awssdk.utils.StringUtils; + +@SdkInternalApi +public final class ProgressListenerUtils { + + private ProgressListenerUtils() { + } + + public static SdkHttpContentPublisher updateProgressListenersWithRequestStatus(ProgressUpdater progressUpdater, + SdkHttpContentPublisher requestProvider, + boolean shouldSetContentLength) { + + if (shouldSetContentLength) { + progressUpdater.updateRequestContentLength(requestProvider.contentLength().get()); + } + + requestProvider = new BytesSentTrackingPublisher(requestProvider, + progressUpdater, + requestProvider.contentLength()); + + return requestProvider; + } + + public static void updateProgressListenersWithResponseStatus(ProgressUpdater progressUpdater, + SdkHttpResponse headers) { + progressUpdater.responseHeaderReceived(); + headers.firstMatchingHeader(CONTENT_LENGTH).ifPresent(value -> { + if (!StringUtils.isNotBlank(value)) { + progressUpdater.updateResponseContentLength(Long.parseLong(value)); + } + }); + } + + public static void updateProgressListenersWithSuccessResponse(SdkResponse response, + RequestExecutionContext context) { + + context.progressUpdater().ifPresent(progressUpdater -> { + progressUpdater.executionSuccess(response); + }); + } + + public static boolean progressListenerAttached(SdkRequest request) { + return !request.overrideConfiguration() + .map(RequestOverrideConfiguration::progressListeners) + .filter(List::isEmpty).isPresent(); + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/client/handler/AsyncClientHandlerTransformerVerificationTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/client/handler/AsyncClientHandlerTransformerVerificationTest.java index ce7d7397f10b..249a454a3838 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/client/handler/AsyncClientHandlerTransformerVerificationTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/client/handler/AsyncClientHandlerTransformerVerificationTest.java @@ -46,7 +46,6 @@ import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkServiceException; import software.amazon.awssdk.core.http.HttpResponseHandler; -import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.protocol.VoidSdkResponse; import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.core.runtime.transform.Marshaller; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStageTest.java new file mode 100644 index 000000000000..582a1762c9ec --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStageTest.java @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.REQUEST_BODY; +import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.createSdkHttpRequest; +import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.createSdkResponseBuilder; +import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.progressListenerContext; + +import java.net.URI; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; + +public class AfterExecutionProgressReportingStageTest { + + @Test + public void afterExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { + ProgressListener progressListener = Mockito.mock(ProgressListener.class); + + SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() + .addProgressListener(progressListener) + .build(); + + SdkRequest request = createSdkHttpRequest(config).build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(request, null); + + RequestExecutionContext requestExecutionContext = progressListenerContext(false, request, + progressUpdater); + + SdkResponse response = createSdkResponseBuilder().build(); + + AfterExecutionProgressReportingStage afterExecutionUpdateProgressStage = new AfterExecutionProgressReportingStage(); + afterExecutionUpdateProgressStage.execute(response, requestExecutionContext); + + Mockito.verify(progressListener, Mockito.times(0)).requestPrepared(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).requestBytesSent(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).responseHeaderReceived(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).responseBytesReceived(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(1)).executionSuccess(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).executionFailure(Mockito.any()); + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStageTest.java new file mode 100644 index 000000000000..38e0554b254c --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStageTest.java @@ -0,0 +1,61 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.createHttpRequestBuilder; +import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.createSdkHttpRequest; +import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.progressListenerContext; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.http.SdkHttpFullRequest; + +public class BeforeExecutionProgressReportingStageTest { + @Test + public void beforeExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { + ProgressListener progressListener = Mockito.mock(ProgressListener.class); + + SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() + .addProgressListener(progressListener) + .build(); + + SdkHttpFullRequest requestBuilder = createHttpRequestBuilder().build(); + + SdkRequest request = createSdkHttpRequest(config).build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(request, null); + + RequestExecutionContext requestExecutionContext = progressListenerContext(false, request, + progressUpdater); + + BeforeExecutionProgressReportingStage beforeExecutionUpdateProgressStage = new BeforeExecutionProgressReportingStage(); + beforeExecutionUpdateProgressStage.execute(requestBuilder, requestExecutionContext); + + Mockito.verify(progressListener, Mockito.times(1)).requestPrepared(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).requestBytesSent(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).responseHeaderReceived(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).responseBytesReceived(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).executionSuccess(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).executionFailure(Mockito.any()); + Mockito.verify(progressListener, Mockito.times(0)).attemptFailure(Mockito.any()); + + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java deleted file mode 100644 index 0c6294324c10..000000000000 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ProgressUpdateStageTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.internal.http.pipeline.stages; - -import java.net.URI; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; -import software.amazon.awssdk.core.SdkRequest; -import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; -import software.amazon.awssdk.core.SdkResponse; -import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.core.http.ExecutionContext; -import software.amazon.awssdk.core.http.NoopTestRequest; -import software.amazon.awssdk.core.internal.http.RequestExecutionContext; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; -import software.amazon.awssdk.core.progress.listener.ProgressListener; -import software.amazon.awssdk.core.protocol.VoidSdkResponse; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.http.SdkHttpFullRequest; -import software.amazon.awssdk.http.SdkHttpMethod; - -@RunWith(MockitoJUnitRunner.class) -public class ProgressUpdateStageTest { - - private static final AsyncRequestBody ASYNC_REQUEST_BODY = AsyncRequestBody.fromString("TestBody"); - private static final RequestBody REQUEST_BODY = RequestBody.fromString("TestBody"); - - @Test - public void sync_preExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { - ProgressListener progressListener = Mockito.mock(ProgressListener.class); - - SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() - .addProgressListener(progressListener) - .build(); - - SdkHttpFullRequest requestBuilder = createHttpRequestBuilder().build(); - - SdkRequest request = createSdkHttpRequest(config).build(); - - ProgressUpdater progressUpdater = new ProgressUpdater(request, null); - ExecutionContext executionContext = ExecutionContext.builder() - .progressUpdater(progressUpdater) - .build(); - - RequestExecutionContext requestExecutionContext = progressListenerContext(false, request, - executionContext).build(); - - PreExecutionUpdateProgressStage preExecutionUpdateProgressStage = new PreExecutionUpdateProgressStage(); - preExecutionUpdateProgressStage.execute(requestBuilder, requestExecutionContext); - - Mockito.verify(progressListener, Mockito.times(1)).requestPrepared(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).requestBytesSent(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).responseHeaderReceived(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).responseBytesReceived(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).executionSuccess(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).executionFailure(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).attemptFailure(Mockito.any()); - - } - - @Test - public void sync_postExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { - ProgressListener progressListener = Mockito.mock(ProgressListener.class); - - SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() - .addProgressListener(progressListener) - .build(); - - SdkRequest request = createSdkHttpRequest(config).build(); - - ProgressUpdater progressUpdater = new ProgressUpdater(request, null); - ExecutionContext executionContext = ExecutionContext.builder() - .progressUpdater(progressUpdater) - .build(); - - RequestExecutionContext requestExecutionContext = progressListenerContext(false, request, - executionContext).build(); - - SdkResponse response = createSdkResponseBuilder().build(); - - PostExecutionUpdateProgressStage postExecutionUpdateProgressStage = new PostExecutionUpdateProgressStage(); - postExecutionUpdateProgressStage.execute(response, requestExecutionContext); - - Mockito.verify(progressListener, Mockito.times(0)).requestPrepared(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).requestBytesSent(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).responseHeaderReceived(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).responseBytesReceived(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(1)).executionSuccess(Mockito.any()); - Mockito.verify(progressListener, Mockito.times(0)).executionFailure(Mockito.any()); - } - - - private RequestExecutionContext.Builder progressListenerContext(boolean isAsyncStreaming, SdkRequest sdkRequest, - ExecutionContext executionContext) { - - RequestExecutionContext.Builder builder = - RequestExecutionContext.builder().executionContext(executionContext). - originalRequest(sdkRequest); - if (isAsyncStreaming) { - builder.requestProvider(ASYNC_REQUEST_BODY); - } - - return builder; - } - - private SdkHttpFullRequest.Builder createHttpRequestBuilder() { - return SdkHttpFullRequest.builder().uri(URI.create("https://endpoint.host")) - .method(SdkHttpMethod.GET) - .contentStreamProvider(REQUEST_BODY.contentStreamProvider()); - } - - private SdkResponse.Builder createSdkResponseBuilder() { - return VoidSdkResponse.builder(); - } - - private SdkRequest.Builder createSdkHttpRequest(SdkRequestOverrideConfiguration config) { - return NoopTestRequest.builder() - .overrideConfiguration(config); - } -} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java index 96f6aced2ba0..faf4691b67fd 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java @@ -122,22 +122,6 @@ public static ExecutionContext executionContext(SdkHttpFullRequest request) { .build(); } - public static ExecutionContext executionContextWithProgressUpdater(SdkHttpFullRequest request, ProgressUpdater progressUpdater) { - InterceptorContext incerceptorContext = - InterceptorContext.builder() - .request(NoopTestRequest.builder().build()) - .httpRequest(request) - .build(); - return ExecutionContext.builder() - .signer(new NoOpSigner()) - .interceptorChain(new ExecutionInterceptorChain(Collections.emptyList())) - .executionAttributes(new ExecutionAttributes()) - .progressUpdater(progressUpdater) - .interceptorContext(incerceptorContext) - .metricCollector(MetricCollector.create("ApiCall")) - .build(); - } - private static void waitBeforeAssertOnExecutor() { try { Thread.sleep(WAIT_BEFORE_ASSERT_ON_EXECUTOR); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java index 4becab26f6f4..eccfd26bd227 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java @@ -40,7 +40,7 @@ public class BytesReadTrackingPublisherTest { @Test - public void test_requestAll_calculatesCorrectTotal() { + public void requestAll_calculatesCorrectTotal() { long nElements = 1024; int elementSize = 4; Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); @@ -51,7 +51,7 @@ public void test_requestAll_calculatesCorrectTotal() { } @Test - public void test_requestAll_updatesInputCount() { + public void requestAll_updatesInputCount() { long nElements = 8; int elementSize = 2; @@ -68,7 +68,7 @@ public void test_requestAll_updatesInputCount() { } @Test - public void test_progressUpdater_invokes_incrementBytesReceived() { + public void progressUpdater_invokes_incrementBytesReceived() { int nElements = 8; int elementSize = 2; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java new file mode 100644 index 000000000000..349ea332bb57 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java @@ -0,0 +1,70 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.util; + +import java.net.URI; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.http.NoopTestRequest; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.protocol.VoidSdkResponse; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; + +public final class ProgressListenerTestUtils { + + public static final AsyncRequestBody ASYNC_REQUEST_BODY = AsyncRequestBody.fromString("TestBody"); + public static final RequestBody REQUEST_BODY = RequestBody.fromString("TestBody"); + + private ProgressListenerTestUtils() { + } + + public static SdkResponse.Builder createSdkResponseBuilder() { + return VoidSdkResponse.builder(); + } + + public static SdkRequest.Builder createSdkHttpRequest(SdkRequestOverrideConfiguration config) { + return NoopTestRequest.builder() + .overrideConfiguration(config); + } + + public static RequestExecutionContext progressListenerContext(boolean isAsyncStreaming, SdkRequest sdkRequest, + ProgressUpdater progressUpdater) { + + RequestExecutionContext.Builder builder = + RequestExecutionContext.builder(). + executionContext(ExecutionContext.builder().build()). + originalRequest(sdkRequest); + if (isAsyncStreaming) { + builder.requestProvider(ASYNC_REQUEST_BODY); + } + + RequestExecutionContext context = builder.build(); + context.progressUpdater(progressUpdater); + return context; + } + + public static SdkHttpFullRequest.Builder createHttpRequestBuilder() { + return SdkHttpFullRequest.builder().uri(URI.create("https://endpoint.host")) + .method(SdkHttpMethod.GET) + .contentStreamProvider(REQUEST_BODY.contentStreamProvider()); + } +} From 87565703bf3f9878f7c1c4978ef9a18d09bf2806 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Fri, 7 Jun 2024 10:48:30 -0700 Subject: [PATCH 41/50] Fix PR comments, added an interface and abstract class to denote Upload and Download operations in a wya in which BytesReadTrackingPublisher can be reused for upload and download --- .../imds/internal/AsyncHttpRequestHelper.java | 2 +- .../internal/http/AmazonAsyncHttpClient.java | 2 +- .../async/SimpleHttpContentPublisher.java | 12 ++- .../AfterExecutionProgressReportingStage.java | 5 +- .../stages/MakeAsyncHttpRequestStage.java | 36 ++++--- .../pipeline/stages/MakeHttpRequestStage.java | 1 + .../stages/UnwrapResponseContainer.java | 1 - .../metrics/BytesReadTrackingPublisher.java | 30 +++--- .../metrics/BytesSentTrackingPublisher.java | 102 ------------------ .../progress/ProgressListenerContext.java | 5 +- .../DownloadProgressUpdaterInvocation.java | 35 ++++++ .../internal/util/ProgressListenerUtils.java | 34 ++---- .../internal/util/ProgressUpdaterInvoker.java | 21 ++++ .../util/UploadProgressUpdaterInvocation.java | 35 ++++++ .../progress/listener/ProgressListener.java | 3 +- .../async/SimpleRequestProviderTckTest.java | 2 +- .../BytesReadTrackingPublisherTckTest.java | 2 +- .../BytesReadTrackingPublisherTest.java | 8 +- .../BytesSentTrackingPublisherTckTest.java | 58 ---------- .../BytesSentTrackingPublisherTest.java | 12 ++- .../http/SimpleHttpContentPublisher.java | 1 + 21 files changed, 174 insertions(+), 233 deletions(-) delete mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisher.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/DownloadProgressUpdaterInvocation.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressUpdaterInvoker.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UploadProgressUpdaterInvocation.java delete mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTckTest.java diff --git a/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java b/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java index 73a70ab4ea02..b4e0dcc6622b 100644 --- a/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java +++ b/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java @@ -61,7 +61,7 @@ private static CompletableFuture sendAsync(SdkAsyncHttpClient client, SdkHttpFullRequest request, HttpResponseHandler handler, CompletableFuture parentFuture) { - SdkHttpContentPublisher requestContentPublisher = new SimpleHttpContentPublisher(request); + SdkHttpContentPublisher requestContentPublisher = new SimpleHttpContentPublisher(request, null); TransformingAsyncResponseHandler responseHandler = new AsyncResponseHandler<>(handler, Function.identity(), new ExecutionAttributes()); CompletableFuture responseHandlerFuture = responseHandler.prepare(); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonAsyncHttpClient.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonAsyncHttpClient.java index 2db772d7f696..2111196fcbac 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonAsyncHttpClient.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonAsyncHttpClient.java @@ -210,7 +210,7 @@ public CompletableFuture execute( .wrappedWith(AsyncApiCallAttemptMetricCollectionStage::new) .wrappedWith((deps, wrapped) -> new AsyncRetryableStage<>(responseHandler, deps, wrapped)) .then(async(() -> new UnwrapResponseContainer<>())) - .then(() -> new AfterExecutionProgressReportingStage<>()) + .then(async(() -> new AfterExecutionProgressReportingStage<>())) .then(async(() -> new AfterExecutionInterceptorsStage<>())) .wrappedWith(AsyncExecutionFailureExceptionReportingStage::new) .wrappedWith(AsyncApiCallTimeoutTrackingStage::new) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java index 0471c6199dc1..c7a1b9a0f941 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java @@ -22,6 +22,7 @@ import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.utils.IoUtils; @@ -35,11 +36,13 @@ public final class SimpleHttpContentPublisher implements SdkHttpContentPublisher private final byte[] content; private final int length; + private final ProgressUpdater progressUpdater; - public SimpleHttpContentPublisher(SdkHttpFullRequest request) { + public SimpleHttpContentPublisher(SdkHttpFullRequest request, ProgressUpdater progressUpdater) { this.content = request.contentStreamProvider().map(p -> invokeSafely(() -> IoUtils.toByteArray(p.newStream()))) .orElseGet(() -> new byte[0]); this.length = content.length; + this.progressUpdater = progressUpdater; } @Override @@ -52,7 +55,7 @@ public void subscribe(Subscriber s) { s.onSubscribe(new SubscriptionImpl(s)); } - private class SubscriptionImpl implements Subscription { + private final class SubscriptionImpl implements Subscription { private boolean running = true; private final Subscriber s; @@ -68,6 +71,11 @@ public void request(long n) { s.onError(new IllegalArgumentException("Demand must be positive")); } else { s.onNext(ByteBuffer.wrap(content)); + + if (progressUpdater != null) { + progressUpdater.incrementBytesSent(length); + } + s.onComplete(); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStage.java index 5c53475103c9..cece9108d49f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStage.java @@ -25,7 +25,10 @@ public class AfterExecutionProgressReportingStage implements RequestPipeline { @Override public OutputT execute(OutputT input, RequestExecutionContext context) throws Exception { - ProgressListenerUtils.updateProgressListenersWithSuccessResponse((SdkResponse) input, context); + if (input instanceof SdkResponse) { + ProgressListenerUtils.updateProgressListenersWithSuccessResponse((SdkResponse) input, context); + } + return input; } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index c7c8e706b417..e8d2099af4aa 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -47,8 +47,11 @@ import software.amazon.awssdk.core.internal.http.timers.TimeoutTracker; import software.amazon.awssdk.core.internal.http.timers.TimerUtils; import software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.internal.util.DownloadProgressUpdaterInvocation; import software.amazon.awssdk.core.internal.util.MetricUtils; import software.amazon.awssdk.core.internal.util.ProgressListenerUtils; +import software.amazon.awssdk.core.internal.util.UploadProgressUpdaterInvocation; import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; @@ -130,22 +133,16 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque CompletableFuture> responseHandlerFuture = responseHandler.prepare(); + ProgressUpdater progressUpdater = ProgressListenerUtils.getProgressUpdaterIfAttached(context); + SdkHttpContentPublisher requestProvider = context.requestProvider() == null - ? new SimpleHttpContentPublisher(request) - : new SdkHttpContentPublisherAdapter(context.requestProvider()); + ? new SimpleHttpContentPublisher(request, progressUpdater) + : new SdkHttpContentPublisherAdapter(context.requestProvider(), progressUpdater); // Set content length if it hasn't been set already. SdkHttpFullRequest requestWithContentLength = getRequestWithContentLength(request, requestProvider); MetricCollector httpMetricCollector = MetricUtils.createHttpMetricsCollector(context); - //If Progress Listening is enabled, wrap around BytesSentTrackingPublisher to track progress on bytes sent - if (context.progressUpdater().isPresent()) { - boolean shouldSetContentLength = shouldSetContentLength(request, requestProvider); - requestProvider = ProgressListenerUtils.updateProgressListenersWithRequestStatus(context.progressUpdater().get(), - requestProvider, - shouldSetContentLength); - } - AsyncExecuteRequest.Builder executeRequestBuilder = AsyncExecuteRequest.builder() .request(requestWithContentLength) .requestContentPublisher(requestProvider) @@ -273,9 +270,11 @@ private void completeResponseFuture(CompletableFuture> respons private static final class SdkHttpContentPublisherAdapter implements SdkHttpContentPublisher { private final AsyncRequestBody asyncRequestBody; + private final ProgressUpdater progressUpdater; - private SdkHttpContentPublisherAdapter(AsyncRequestBody asyncRequestBody) { + private SdkHttpContentPublisherAdapter(AsyncRequestBody asyncRequestBody, ProgressUpdater progressUpdater) { this.asyncRequestBody = asyncRequestBody; + this.progressUpdater = progressUpdater; } @Override @@ -285,7 +284,12 @@ public Optional contentLength() { @Override public void subscribe(Subscriber s) { - asyncRequestBody.subscribe(s); + if (progressUpdater != null) { + Publisher readTrackingPublisher = new BytesReadTrackingPublisher(asyncRequestBody, new AtomicLong(0L), new UploadProgressUpdaterInvocation(progressUpdater)); + readTrackingPublisher.subscribe(s); + } else { + asyncRequestBody.subscribe(s); + } } } @@ -323,9 +327,11 @@ public void onStream(Publisher stream) { AtomicLong bytesReadCounter = context.executionAttributes() .getAttribute(SdkInternalExecutionAttribute.RESPONSE_BYTES_READ); - BytesReadTrackingPublisher bytesReadTrackingPublisher = new BytesReadTrackingPublisher(stream, - bytesReadCounter, - context.progressUpdater()); + ProgressUpdater progressUpdater = context.progressUpdater().isPresent() ? + context.progressUpdater().get() : null; + Publisher bytesReadTrackingPublisher = new BytesReadTrackingPublisher(stream, + bytesReadCounter, + new DownloadProgressUpdaterInvocation(progressUpdater)); super.onStream(bytesReadTrackingPublisher); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java index 32cd094a57de..92c3003e377a 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java @@ -55,6 +55,7 @@ public Pair execute(SdkHttpFullRequest RequestExecutionContext context) throws Exception { InterruptMonitor.checkInterrupted(); HttpExecuteResponse executeResponse = executeHttpRequest(request, context); + // TODO: Plumb through ExecuteResponse instead SdkHttpFullResponse httpResponse = (SdkHttpFullResponse) executeResponse.httpResponse(); return Pair.of(request, httpResponse.toBuilder().content(executeResponse.responseBody().orElse(null)).build()); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java index be713469621a..d7290a29c1cb 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/UnwrapResponseContainer.java @@ -17,7 +17,6 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.Response; -import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java index 8b6364a4326d..b9590e5fc456 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java @@ -16,13 +16,12 @@ package software.amazon.awssdk.core.internal.metrics; import java.nio.ByteBuffer; -import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.internal.util.ProgressUpdaterInvoker; /** * Publisher that tracks how many bytes are published from the wrapped publisher to the downstream subscriber. @@ -31,20 +30,18 @@ public final class BytesReadTrackingPublisher implements Publisher { private final Publisher upstream; private final AtomicLong bytesRead; - private ProgressUpdater progressUpdater; + private final ProgressUpdaterInvoker progressUpdaterInvoker; public BytesReadTrackingPublisher(Publisher upstream, AtomicLong bytesRead, - Optional progressUpdater) { + ProgressUpdaterInvoker progressUpdaterInvoker) { this.upstream = upstream; this.bytesRead = bytesRead; - progressUpdater.ifPresent(value -> { - this.progressUpdater = value; - }); + this.progressUpdaterInvoker = progressUpdaterInvoker; } @Override public void subscribe(Subscriber subscriber) { - upstream.subscribe(new BytesReadTracker(subscriber, bytesRead, progressUpdater)); + upstream.subscribe(new BytesReadTracker(subscriber, bytesRead, progressUpdaterInvoker)); } public long bytesRead() { @@ -54,29 +51,30 @@ public long bytesRead() { private static final class BytesReadTracker implements Subscriber { private final Subscriber downstream; private final AtomicLong bytesRead; - private final ProgressUpdater progressUpdater; + private final ProgressUpdaterInvoker progressUpdaterInvoker; private BytesReadTracker(Subscriber downstream, - AtomicLong bytesRead, ProgressUpdater progressUpdater) { + AtomicLong bytesRead, ProgressUpdaterInvoker progressUpdaterInvoker) { this.downstream = downstream; this.bytesRead = bytesRead; - this.progressUpdater = progressUpdater; + this.progressUpdaterInvoker = progressUpdaterInvoker; } @Override public void onSubscribe(Subscription subscription) { downstream.onSubscribe(subscription); - if (progressUpdater != null) { - progressUpdater.resetBytesReceived(); + if (progressUpdaterInvoker != null) { + progressUpdaterInvoker.resetBytes(); } } @Override public void onNext(ByteBuffer byteBuffer) { - bytesRead.addAndGet(byteBuffer.remaining()); + long byteBufferSize = byteBuffer.remaining(); + bytesRead.addAndGet(byteBufferSize); downstream.onNext(byteBuffer); - if (progressUpdater != null) { - progressUpdater.incrementBytesReceived(bytesRead.get()); + if (progressUpdaterInvoker != null) { + progressUpdaterInvoker.updateBytesTransferred(byteBufferSize); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisher.java deleted file mode 100644 index 475469b88926..000000000000 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisher.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.internal.metrics; - -import java.nio.ByteBuffer; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; -import org.reactivestreams.Publisher; -import org.reactivestreams.Subscriber; -import org.reactivestreams.Subscription; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; -import software.amazon.awssdk.http.async.SdkHttpContentPublisher; - -@SdkInternalApi -public class BytesSentTrackingPublisher implements SdkHttpContentPublisher { - - private final Publisher upstream; - private final AtomicLong bytesSent; - private ProgressUpdater progressUpdater; - private Long contentLength; - - public BytesSentTrackingPublisher(Publisher upstream, - ProgressUpdater progressUpdater, - Optional contentLength) { - this.upstream = upstream; - this.bytesSent = new AtomicLong(0L); - this.progressUpdater = progressUpdater; - contentLength.ifPresent(value -> { - this.contentLength = value; - }); - } - - @Override - public void subscribe(Subscriber subscriber) { - upstream.subscribe(new BytesSentTrackingPublisher.BytesSentTracker(subscriber, - bytesSent, - progressUpdater)); - } - - public long bytesSent() { - return bytesSent.get(); - } - - @Override - public Optional contentLength() { - return Optional.ofNullable(contentLength); - } - - private static final class BytesSentTracker implements Subscriber { - private final Subscriber downstream; - private final AtomicLong bytesSent; - private final ProgressUpdater progressUpdater; - - private BytesSentTracker(Subscriber downstream, AtomicLong bytesSent, - ProgressUpdater progressUpdater) { - this.downstream = downstream; - this.bytesSent = bytesSent; - this.progressUpdater = progressUpdater; - } - - @Override - public void onSubscribe(Subscription subscription) { - downstream.onSubscribe(subscription); - if (progressUpdater != null) { - progressUpdater.resetBytesSent(); - } - } - - @Override - public void onNext(ByteBuffer byteBuffer) { - downstream.onNext(byteBuffer); - bytesSent.addAndGet(byteBuffer.remaining()); - if (progressUpdater != null) { - progressUpdater.incrementBytesSent(bytesSent.get()); - } - } - - @Override - public void onError(Throwable throwable) { - downstream.onError(throwable); - } - - @Override - public void onComplete() { - downstream.onComplete(); - } - } -} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java index 86af01d93395..6a821d1bcd0a 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/ProgressListenerContext.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.core.internal.progress; +import java.util.Optional; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.SdkRequest; @@ -89,8 +90,8 @@ public SdkHttpResponse httpResponse() { } @Override - public SdkResponse response() { - return response; + public Optional response() { + return Optional.of(response); } @Override diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/DownloadProgressUpdaterInvocation.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/DownloadProgressUpdaterInvocation.java new file mode 100644 index 000000000000..90006f1a69cc --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/DownloadProgressUpdaterInvocation.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.util; + +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; + +public class DownloadProgressUpdaterInvocation implements ProgressUpdaterInvoker { + ProgressUpdater progressUpdater; + + public DownloadProgressUpdaterInvocation(ProgressUpdater progressUpdater) { + this.progressUpdater = progressUpdater; + } + @Override + public void updateBytesTransferred(long bytes) { + progressUpdater.incrementBytesReceived(bytes); + } + + @Override + public void resetBytes() { + progressUpdater.resetBytesReceived(); + } +} \ No newline at end of file diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java index 04d6f0f660e2..ce97cc216964 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java @@ -17,16 +17,13 @@ import static software.amazon.awssdk.http.Header.CONTENT_LENGTH; -import java.util.List; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.RequestOverrideConfiguration; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; -import software.amazon.awssdk.core.internal.metrics.BytesSentTrackingPublisher; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.http.SdkHttpResponse; -import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.utils.StringUtils; @SdkInternalApi @@ -35,21 +32,6 @@ public final class ProgressListenerUtils { private ProgressListenerUtils() { } - public static SdkHttpContentPublisher updateProgressListenersWithRequestStatus(ProgressUpdater progressUpdater, - SdkHttpContentPublisher requestProvider, - boolean shouldSetContentLength) { - - if (shouldSetContentLength) { - progressUpdater.updateRequestContentLength(requestProvider.contentLength().get()); - } - - requestProvider = new BytesSentTrackingPublisher(requestProvider, - progressUpdater, - requestProvider.contentLength()); - - return requestProvider; - } - public static void updateProgressListenersWithResponseStatus(ProgressUpdater progressUpdater, SdkHttpResponse headers) { progressUpdater.responseHeaderReceived(); @@ -63,14 +45,18 @@ public static void updateProgressListenersWithResponseStatus(ProgressUpdater pro public static void updateProgressListenersWithSuccessResponse(SdkResponse response, RequestExecutionContext context) { - context.progressUpdater().ifPresent(progressUpdater -> { - progressUpdater.executionSuccess(response); - }); + context.progressUpdater().ifPresent(progressUpdater -> progressUpdater.executionSuccess(response)); } public static boolean progressListenerAttached(SdkRequest request) { - return !request.overrideConfiguration() - .map(RequestOverrideConfiguration::progressListeners) - .filter(List::isEmpty).isPresent(); + return request.overrideConfiguration() + .map(RequestOverrideConfiguration::progressListeners).isPresent(); + } + + public static ProgressUpdater getProgressUpdaterIfAttached(RequestExecutionContext context) { + if (context.progressUpdater().isPresent()) { + return context.progressUpdater().get(); + } + return null; } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressUpdaterInvoker.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressUpdaterInvoker.java new file mode 100644 index 000000000000..7074ad8cd145 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressUpdaterInvoker.java @@ -0,0 +1,21 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.util; + +public interface ProgressUpdaterInvoker { + void updateBytesTransferred(long bytes); + void resetBytes(); +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UploadProgressUpdaterInvocation.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UploadProgressUpdaterInvocation.java new file mode 100644 index 000000000000..cf9350865e5d --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UploadProgressUpdaterInvocation.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.util; + +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; + +public class UploadProgressUpdaterInvocation implements ProgressUpdaterInvoker { + ProgressUpdater progressUpdater; + + public UploadProgressUpdaterInvocation(ProgressUpdater progressUpdater) { + this.progressUpdater = progressUpdater; + } + @Override + public void updateBytesTransferred(long bytes) { + progressUpdater.incrementBytesSent(bytes); + } + + @Override + public void resetBytes() { + progressUpdater.resetBytesSent(); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index 082b6933ba55..d401c88d5942 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.core.progress.listener; +import java.util.Optional; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkProtectedApi; @@ -422,7 +423,7 @@ public interface ExecutionSuccess extends ResponseBytesReceived { /** * The successful completion of a request submitted to the Sdk */ - SdkResponse response(); + Optional response(); } /** diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java index de19d7bdc983..46ca96a354c0 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java @@ -37,7 +37,7 @@ public SimpleRequestProviderTckTest() { @Override public Publisher createPublisher(long l) { - return new SimpleHttpContentPublisher(makeFullRequest()); + return new SimpleHttpContentPublisher(makeFullRequest(), null); } @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java index 813a96ed4dfd..2ba6d58c1dfe 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java @@ -35,7 +35,7 @@ public BytesReadTrackingPublisherTckTest() { @Override public Publisher createPublisher(long l) { - return new BytesReadTrackingPublisher(createUpstreamPublisher(l), new AtomicLong(0), Optional.empty()); + return new BytesReadTrackingPublisher(createUpstreamPublisher(l), new AtomicLong(0), null); } @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java index eccfd26bd227..5d7e6246f548 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java @@ -31,6 +31,7 @@ import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.internal.util.DownloadProgressUpdaterInvocation; import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.http.SdkHttpFullRequest; @@ -44,7 +45,7 @@ public void requestAll_calculatesCorrectTotal() { long nElements = 1024; int elementSize = 4; Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0), Optional.empty()); + BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0), null); readFully(trackingPublisher); assertThat(trackingPublisher.bytesRead()).isEqualTo(nElements * elementSize); @@ -59,7 +60,7 @@ public void requestAll_updatesInputCount() { AtomicLong bytesRead = new AtomicLong(baseBytesRead); Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, bytesRead, Optional.empty()); + BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, bytesRead, null); readFully(trackingPublisher); long expectedRead = baseBytesRead + nElements * elementSize; @@ -88,7 +89,8 @@ public void progressUpdater_invokes_incrementBytesReceived() { ProgressUpdater progressUpdater = new ProgressUpdater(request, null); Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - Publisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, bytesRead, Optional.of(progressUpdater)); + Publisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, bytesRead, + new DownloadProgressUpdaterInvocation(progressUpdater)); readFully(trackingPublisher); Mockito.verify(progressListener, Mockito.times(nElements)).responseBytesReceived(ArgumentMatchers.any()); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTckTest.java deleted file mode 100644 index 4c1ca5ec13e5..000000000000 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTckTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.internal.metrics; - -import io.reactivex.Flowable; -import java.nio.ByteBuffer; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.mockito.Mockito; -import org.reactivestreams.Publisher; -import org.reactivestreams.tck.PublisherVerification; -import org.reactivestreams.tck.TestEnvironment; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; - -public class BytesSentTrackingPublisherTckTest extends PublisherVerification { - - ProgressUpdater progressUpdater = Mockito.mock(ProgressUpdater.class); - - public BytesSentTrackingPublisherTckTest() { - super(new TestEnvironment()); - } - - @Override - public Publisher createPublisher(long l) { - return new BytesSentTrackingPublisher(createUpstreamPublisher(l), progressUpdater, Optional.empty()); - } - - @Override - public long maxElementsFromPublisher() { - return 1024; - } - - @Override - public Publisher createFailedPublisher() { - return null; - } - - private Publisher createUpstreamPublisher(long elements) { - return Flowable.fromIterable(Stream.generate(() -> ByteBuffer.wrap(new byte[1])) - .limit(elements) - .collect(Collectors.toList())); - } -} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTest.java index e8a51634bc09..1c56b22fe5bd 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesSentTrackingPublisherTest.java @@ -20,6 +20,7 @@ import io.reactivex.Flowable; import java.nio.ByteBuffer; import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; import org.junit.jupiter.api.Test; @@ -30,6 +31,7 @@ import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.internal.util.UploadProgressUpdaterInvocation; import software.amazon.awssdk.core.progress.listener.ProgressListener; public class BytesSentTrackingPublisherTest { @@ -42,12 +44,13 @@ public void test_updatesBytesSent() { ProgressUpdater progressUpdater = Mockito.mock(ProgressUpdater.class); Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - BytesSentTrackingPublisher trackingPublisher = new BytesSentTrackingPublisher(upstreamPublisher, progressUpdater, Optional.empty()); + BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0), + new UploadProgressUpdaterInvocation(progressUpdater)); readFully(trackingPublisher); long expectedSent = nElements * elementSize; - assertThat(trackingPublisher.bytesSent()).isEqualTo(expectedSent); + assertThat(trackingPublisher.bytesRead()).isEqualTo(expectedSent); } @Test @@ -68,12 +71,13 @@ public void test_progressUpdater_invokes_incrementBytesSent() { ProgressUpdater progressUpdater = new ProgressUpdater(request, null); Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - BytesSentTrackingPublisher trackingPublisher = new BytesSentTrackingPublisher(upstreamPublisher, progressUpdater, Optional.empty()); + BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0L), + new UploadProgressUpdaterInvocation(progressUpdater)); readFully(trackingPublisher); long expectedSent = nElements * elementSize; - assertThat(trackingPublisher.bytesSent()).isEqualTo(expectedSent); + assertThat(trackingPublisher.bytesRead()).isEqualTo(expectedSent); Mockito.verify(progressListener, Mockito.times(nElements)).requestBytesSent(ArgumentMatchers.any()); } diff --git a/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SimpleHttpContentPublisher.java b/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SimpleHttpContentPublisher.java index 5d00e5a2140f..0d3197066abb 100644 --- a/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SimpleHttpContentPublisher.java +++ b/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SimpleHttpContentPublisher.java @@ -76,6 +76,7 @@ public void request(long n) { if (n <= 0) { s.onError(new IllegalArgumentException("Demand must be positive")); } else { + ProgressListenerU s.onNext(content); s.onComplete(); } From a604bd31717c2c9e9edcfc9e9c0b8a1cc1765958 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Fri, 7 Jun 2024 10:57:54 -0700 Subject: [PATCH 42/50] Remove incorrect modification of test class --- .../software/amazon/awssdk/http/SimpleHttpContentPublisher.java | 1 - 1 file changed, 1 deletion(-) diff --git a/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SimpleHttpContentPublisher.java b/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SimpleHttpContentPublisher.java index 0d3197066abb..5d00e5a2140f 100644 --- a/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SimpleHttpContentPublisher.java +++ b/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SimpleHttpContentPublisher.java @@ -76,7 +76,6 @@ public void request(long n) { if (n <= 0) { s.onError(new IllegalArgumentException("Demand must be positive")); } else { - ProgressListenerU s.onNext(content); s.onComplete(); } From be2f663bb1bdc3acaf0822d5865403567b6f43c5 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Mon, 10 Jun 2024 14:18:42 -0700 Subject: [PATCH 43/50] Repurposed BytesReadTrackingPublisher --- .../pipeline/stages/MakeAsyncHttpRequestStage.java | 13 +++++++++---- .../metrics/BytesReadTrackingPublisher.java | 4 ++-- .../util/DownloadProgressUpdaterInvocation.java | 8 ++++++++ .../core/internal/util/ProgressUpdaterInvoker.java | 7 +++++++ .../util/UploadProgressUpdaterInvocation.java | 8 ++++++++ .../metrics/BytesReadTrackingPublisherTckTest.java | 4 +++- .../metrics/BytesReadTrackingPublisherTest.java | 6 ++++-- 7 files changed, 41 insertions(+), 9 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index e8d2099af4aa..aef99d6f571c 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -137,7 +137,8 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque SdkHttpContentPublisher requestProvider = context.requestProvider() == null ? new SimpleHttpContentPublisher(request, progressUpdater) - : new SdkHttpContentPublisherAdapter(context.requestProvider(), progressUpdater); + : new SdkHttpContentPublisherAdapter(context.requestProvider(), + progressUpdater); // Set content length if it hasn't been set already. SdkHttpFullRequest requestWithContentLength = getRequestWithContentLength(request, requestProvider); @@ -285,7 +286,11 @@ public Optional contentLength() { @Override public void subscribe(Subscriber s) { if (progressUpdater != null) { - Publisher readTrackingPublisher = new BytesReadTrackingPublisher(asyncRequestBody, new AtomicLong(0L), new UploadProgressUpdaterInvocation(progressUpdater)); + Publisher readTrackingPublisher = new BytesReadTrackingPublisher(asyncRequestBody, + new AtomicLong(0L), + new UploadProgressUpdaterInvocation( + progressUpdater + )); readTrackingPublisher.subscribe(s); } else { asyncRequestBody.subscribe(s); @@ -330,8 +335,8 @@ public void onStream(Publisher stream) { ProgressUpdater progressUpdater = context.progressUpdater().isPresent() ? context.progressUpdater().get() : null; Publisher bytesReadTrackingPublisher = new BytesReadTrackingPublisher(stream, - bytesReadCounter, - new DownloadProgressUpdaterInvocation(progressUpdater)); + bytesReadCounter, + new DownloadProgressUpdaterInvocation(progressUpdater)); super.onStream(bytesReadTrackingPublisher); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java index b9590e5fc456..7e6bdadb29f8 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java @@ -63,7 +63,7 @@ private BytesReadTracker(Subscriber downstream, @Override public void onSubscribe(Subscription subscription) { downstream.onSubscribe(subscription); - if (progressUpdaterInvoker != null) { + if (progressUpdaterInvoker.progressUpdater() != null) { progressUpdaterInvoker.resetBytes(); } } @@ -73,7 +73,7 @@ public void onNext(ByteBuffer byteBuffer) { long byteBufferSize = byteBuffer.remaining(); bytesRead.addAndGet(byteBufferSize); downstream.onNext(byteBuffer); - if (progressUpdaterInvoker != null) { + if (progressUpdaterInvoker != null && progressUpdaterInvoker.progressUpdater() != null) { progressUpdaterInvoker.updateBytesTransferred(byteBufferSize); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/DownloadProgressUpdaterInvocation.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/DownloadProgressUpdaterInvocation.java index 90006f1a69cc..d647a36c3227 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/DownloadProgressUpdaterInvocation.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/DownloadProgressUpdaterInvocation.java @@ -15,14 +15,17 @@ package software.amazon.awssdk.core.internal.util; +import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +@SdkInternalApi public class DownloadProgressUpdaterInvocation implements ProgressUpdaterInvoker { ProgressUpdater progressUpdater; public DownloadProgressUpdaterInvocation(ProgressUpdater progressUpdater) { this.progressUpdater = progressUpdater; } + @Override public void updateBytesTransferred(long bytes) { progressUpdater.incrementBytesReceived(bytes); @@ -32,4 +35,9 @@ public void updateBytesTransferred(long bytes) { public void resetBytes() { progressUpdater.resetBytesReceived(); } + + @Override + public ProgressUpdater progressUpdater() { + return progressUpdater; + } } \ No newline at end of file diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressUpdaterInvoker.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressUpdaterInvoker.java index 7074ad8cd145..0ec0db39ec8f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressUpdaterInvoker.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressUpdaterInvoker.java @@ -15,7 +15,14 @@ package software.amazon.awssdk.core.internal.util; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; + +@SdkInternalApi public interface ProgressUpdaterInvoker { void updateBytesTransferred(long bytes); + void resetBytes(); + + ProgressUpdater progressUpdater(); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UploadProgressUpdaterInvocation.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UploadProgressUpdaterInvocation.java index cf9350865e5d..ce829afd9925 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UploadProgressUpdaterInvocation.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UploadProgressUpdaterInvocation.java @@ -15,14 +15,17 @@ package software.amazon.awssdk.core.internal.util; +import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +@SdkInternalApi public class UploadProgressUpdaterInvocation implements ProgressUpdaterInvoker { ProgressUpdater progressUpdater; public UploadProgressUpdaterInvocation(ProgressUpdater progressUpdater) { this.progressUpdater = progressUpdater; } + @Override public void updateBytesTransferred(long bytes) { progressUpdater.incrementBytesSent(bytes); @@ -32,4 +35,9 @@ public void updateBytesTransferred(long bytes) { public void resetBytes() { progressUpdater.resetBytesSent(); } + + @Override + public ProgressUpdater progressUpdater() { + return progressUpdater; + } } diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java index 2ba6d58c1dfe..39907553bced 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java @@ -24,6 +24,8 @@ import org.reactivestreams.Publisher; import org.reactivestreams.tck.PublisherVerification; import org.reactivestreams.tck.TestEnvironment; +import software.amazon.awssdk.core.internal.util.DownloadProgressUpdaterInvocation; +import software.amazon.awssdk.core.internal.util.UploadProgressUpdaterInvocation; /** * TCK verification class for {@link BytesReadTrackingPublisher}. @@ -35,7 +37,7 @@ public BytesReadTrackingPublisherTckTest() { @Override public Publisher createPublisher(long l) { - return new BytesReadTrackingPublisher(createUpstreamPublisher(l), new AtomicLong(0), null); + return new BytesReadTrackingPublisher(createUpstreamPublisher(l), new AtomicLong(0), new UploadProgressUpdaterInvocation(null)); } @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java index 5d7e6246f548..8250b6351c4f 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java @@ -32,6 +32,8 @@ import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.internal.util.DownloadProgressUpdaterInvocation; +import software.amazon.awssdk.core.internal.util.ProgressUpdaterInvoker; +import software.amazon.awssdk.core.internal.util.UploadProgressUpdaterInvocation; import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.http.SdkHttpFullRequest; @@ -45,7 +47,7 @@ public void requestAll_calculatesCorrectTotal() { long nElements = 1024; int elementSize = 4; Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0), null); + BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, new AtomicLong(0), new UploadProgressUpdaterInvocation(null)); readFully(trackingPublisher); assertThat(trackingPublisher.bytesRead()).isEqualTo(nElements * elementSize); @@ -60,7 +62,7 @@ public void requestAll_updatesInputCount() { AtomicLong bytesRead = new AtomicLong(baseBytesRead); Publisher upstreamPublisher = createUpstreamPublisher(nElements, elementSize); - BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, bytesRead, null); + BytesReadTrackingPublisher trackingPublisher = new BytesReadTrackingPublisher(upstreamPublisher, bytesRead, new UploadProgressUpdaterInvocation(null)); readFully(trackingPublisher); long expectedRead = baseBytesRead + nElements * elementSize; From dea870b4fd912d277305092fa2752822f2fb4966 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Fri, 5 Jul 2024 05:03:10 -0700 Subject: [PATCH 44/50] Added testing for catching attempt failures --- .../async/SimpleHttpContentPublisher.java | 12 +-- ...ecutionFailureExceptionReportingStage.java | 4 +- .../stages/MakeAsyncHttpRequestStage.java | 23 ++---- .../metrics/BytesReadTrackingPublisher.java | 9 ++- .../internal/util/ProgressListenerUtils.java | 14 ++++ .../awssdk/core/http/NoopHttpFullRequest.java | 78 +++++++++++++++++++ .../async/SimpleRequestProviderTckTest.java | 2 +- ...ionFailureExceptionReportingStageTest.java | 66 ++++++++++++++++ .../util/ProgressListenerTestUtils.java | 13 +++- 9 files changed, 186 insertions(+), 35 deletions(-) create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopHttpFullRequest.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java index c7a1b9a0f941..0471c6199dc1 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/SimpleHttpContentPublisher.java @@ -22,7 +22,6 @@ import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.utils.IoUtils; @@ -36,13 +35,11 @@ public final class SimpleHttpContentPublisher implements SdkHttpContentPublisher private final byte[] content; private final int length; - private final ProgressUpdater progressUpdater; - public SimpleHttpContentPublisher(SdkHttpFullRequest request, ProgressUpdater progressUpdater) { + public SimpleHttpContentPublisher(SdkHttpFullRequest request) { this.content = request.contentStreamProvider().map(p -> invokeSafely(() -> IoUtils.toByteArray(p.newStream()))) .orElseGet(() -> new byte[0]); this.length = content.length; - this.progressUpdater = progressUpdater; } @Override @@ -55,7 +52,7 @@ public void subscribe(Subscriber s) { s.onSubscribe(new SubscriptionImpl(s)); } - private final class SubscriptionImpl implements Subscription { + private class SubscriptionImpl implements Subscription { private boolean running = true; private final Subscriber s; @@ -71,11 +68,6 @@ public void request(long n) { s.onError(new IllegalArgumentException("Demand must be positive")); } else { s.onNext(ByteBuffer.wrap(content)); - - if (progressUpdater != null) { - progressUpdater.incrementBytesSent(length); - } - s.onComplete(); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java index 6b1086ed7bf8..3bc5eac484c7 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStage.java @@ -39,9 +39,7 @@ public OutputT execute(SdkHttpFullRequest input, RequestExecutionContext context } catch (Exception e) { Throwable throwable = reportFailureToInterceptors(context, e); - context.progressUpdater().ifPresent(progressUpdater -> { - reportFailureToProgressListeners(progressUpdater, throwable); - }); + context.progressUpdater().ifPresent(progressUpdater -> reportFailureToProgressListeners(progressUpdater, throwable)); throw failure(throwable); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index aef99d6f571c..7ebc4a358cbf 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -133,12 +133,10 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque CompletableFuture> responseHandlerFuture = responseHandler.prepare(); - ProgressUpdater progressUpdater = ProgressListenerUtils.getProgressUpdaterIfAttached(context); - SdkHttpContentPublisher requestProvider = context.requestProvider() == null - ? new SimpleHttpContentPublisher(request, progressUpdater) - : new SdkHttpContentPublisherAdapter(context.requestProvider(), - progressUpdater); + ? new SimpleHttpContentPublisher(request) + : new SdkHttpContentPublisherAdapter(context.requestProvider()); + requestProvider = ProgressListenerUtils.wrapRequestProviderWithByteTrackingIfProgressListenerAttached(requestProvider, context); // Set content length if it hasn't been set already. SdkHttpFullRequest requestWithContentLength = getRequestWithContentLength(request, requestProvider); @@ -271,11 +269,9 @@ private void completeResponseFuture(CompletableFuture> respons private static final class SdkHttpContentPublisherAdapter implements SdkHttpContentPublisher { private final AsyncRequestBody asyncRequestBody; - private final ProgressUpdater progressUpdater; - private SdkHttpContentPublisherAdapter(AsyncRequestBody asyncRequestBody, ProgressUpdater progressUpdater) { + private SdkHttpContentPublisherAdapter(AsyncRequestBody asyncRequestBody) { this.asyncRequestBody = asyncRequestBody; - this.progressUpdater = progressUpdater; } @Override @@ -285,16 +281,7 @@ public Optional contentLength() { @Override public void subscribe(Subscriber s) { - if (progressUpdater != null) { - Publisher readTrackingPublisher = new BytesReadTrackingPublisher(asyncRequestBody, - new AtomicLong(0L), - new UploadProgressUpdaterInvocation( - progressUpdater - )); - readTrackingPublisher.subscribe(s); - } else { - asyncRequestBody.subscribe(s); - } + asyncRequestBody.subscribe(s); } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java index 7e6bdadb29f8..09ee3063ffba 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java @@ -16,18 +16,20 @@ package software.amazon.awssdk.core.internal.metrics; import java.nio.ByteBuffer; +import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.internal.util.ProgressUpdaterInvoker; +import software.amazon.awssdk.http.async.SdkHttpContentPublisher; /** * Publisher that tracks how many bytes are published from the wrapped publisher to the downstream subscriber. */ @SdkInternalApi -public final class BytesReadTrackingPublisher implements Publisher { +public final class BytesReadTrackingPublisher implements SdkHttpContentPublisher { private final Publisher upstream; private final AtomicLong bytesRead; private final ProgressUpdaterInvoker progressUpdaterInvoker; @@ -48,6 +50,11 @@ public long bytesRead() { return bytesRead.get(); } + @Override + public Optional contentLength() { + return Optional.empty(); + } + private static final class BytesReadTracker implements Subscriber { private final Subscriber downstream; private final AtomicLong bytesRead; diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java index ce97cc216964..c769c5929f74 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java @@ -17,13 +17,18 @@ import static software.amazon.awssdk.http.Header.CONTENT_LENGTH; +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicLong; +import org.reactivestreams.Publisher; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.RequestOverrideConfiguration; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.utils.StringUtils; @SdkInternalApi @@ -32,6 +37,15 @@ public final class ProgressListenerUtils { private ProgressListenerUtils() { } + public static SdkHttpContentPublisher wrapRequestProviderWithByteTrackingIfProgressListenerAttached(SdkHttpContentPublisher requestProvider, RequestExecutionContext context) { + ProgressUpdater progressUpdater = getProgressUpdaterIfAttached(context); + SdkHttpContentPublisher wrappedHttpContentPublisher = requestProvider; + if (progressUpdater != null) { + wrappedHttpContentPublisher = new BytesReadTrackingPublisher(requestProvider, new AtomicLong(0L), new UploadProgressUpdaterInvocation(progressUpdater)); + } + return wrappedHttpContentPublisher; + } + public static void updateProgressListenersWithResponseStatus(ProgressUpdater progressUpdater, SdkHttpResponse headers) { progressUpdater.responseHeaderReceived(); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopHttpFullRequest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopHttpFullRequest.java new file mode 100644 index 000000000000..bee4255ab969 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopHttpFullRequest.java @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.http; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; + +@SdkInternalApi +@Immutable +public class NoopHttpFullRequest implements SdkHttpFullRequest { + + public void NoopHttpFullRequest() { + + } + @Override + public Builder toBuilder() { + return null; + } + + @Override + public Optional contentStreamProvider() { + return Optional.empty(); + } + + @Override + public Map> headers() { + return null; + } + + @Override + public String protocol() { + return null; + } + + @Override + public String host() { + return null; + } + + @Override + public int port() { + return 0; + } + + @Override + public String encodedPath() { + return null; + } + + @Override + public Map> rawQueryParameters() { + return null; + } + + @Override + public SdkHttpMethod method() { + return null; + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java index 46ca96a354c0..de19d7bdc983 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/SimpleRequestProviderTckTest.java @@ -37,7 +37,7 @@ public SimpleRequestProviderTckTest() { @Override public Publisher createPublisher(long l) { - return new SimpleHttpContentPublisher(makeFullRequest(), null); + return new SimpleHttpContentPublisher(makeFullRequest()); } @Override diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java new file mode 100644 index 000000000000..68b81f47a910 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertThrows; +import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.createSdkHttpRequest; +import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.progressListenerContext; + +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import software.amazon.awssdk.core.Response; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; +import software.amazon.awssdk.core.http.NoopHttpFullRequest; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; +import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; +import software.amazon.awssdk.core.progress.listener.ProgressListener; +import software.amazon.awssdk.http.SdkHttpFullRequest; + +public class ExecutionFailureExceptionReportingStageTest { + + + @Test + public void afterExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { + + RequestPipeline> requestPipeline = Mockito.mock(RequestPipeline.class); + ProgressListener progressListener = Mockito.mock(ProgressListener.class); + + SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() + .addProgressListener(progressListener) + .build(); + + SdkRequest request = createSdkHttpRequest(config).build(); + ProgressUpdater progressUpdater = new ProgressUpdater(request, null); + RequestExecutionContext requestExecutionContext = progressListenerContext(false, request, + progressUpdater); + + ExecutionFailureExceptionReportingStage executionFailureExceptionReportingStage = new ExecutionFailureExceptionReportingStage(requestPipeline); + when(requestPipeline.execute(any(), any())).thenThrow(new RuntimeException()); + assertThrows(RuntimeException.class, () -> executionFailureExceptionReportingStage.execute(new NoopHttpFullRequest(), requestExecutionContext)); + + Mockito.verify(progressListener, Mockito.times(0)).requestPrepared(any()); + Mockito.verify(progressListener, Mockito.times(0)).requestBytesSent(any()); + Mockito.verify(progressListener, Mockito.times(0)).responseHeaderReceived(any()); + Mockito.verify(progressListener, Mockito.times(0)).responseBytesReceived(any()); + Mockito.verify(progressListener, Mockito.times(0)).executionSuccess(any()); + Mockito.verify(progressListener, Mockito.times(1)).attemptFailure(any()); + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java index 349ea332bb57..ca9408b8f0b9 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java @@ -16,12 +16,16 @@ package software.amazon.awssdk.core.internal.util; import java.net.URI; +import java.util.ArrayList; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.http.NoopHttpFullRequest; import software.amazon.awssdk.core.http.NoopTestRequest; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain; +import software.amazon.awssdk.core.interceptor.InterceptorContext; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.protocol.VoidSdkResponse; @@ -51,8 +55,13 @@ public static RequestExecutionContext progressListenerContext(boolean isAsyncStr RequestExecutionContext.Builder builder = RequestExecutionContext.builder(). - executionContext(ExecutionContext.builder().build()). - originalRequest(sdkRequest); + executionContext(ExecutionContext.builder() + .interceptorContext(InterceptorContext.builder() + .request(sdkRequest) + .build()) + .interceptorChain(new ExecutionInterceptorChain(new ArrayList<>())) + .build()) + .originalRequest(sdkRequest); if (isAsyncStreaming) { builder.requestProvider(ASYNC_REQUEST_BODY); } From e0e5e23a3d583687376357f35c1c9a206e96f32c Mon Sep 17 00:00:00 2001 From: Krishnan Date: Fri, 5 Jul 2024 05:48:44 -0700 Subject: [PATCH 45/50] Added Tests for async exception reporting --- .../awssdk/core/http/NoopHttpFullRequest.java | 78 ------------------- ...ionFailureExceptionReportingStageTest.java | 39 +++++++++- .../util/ProgressListenerTestUtils.java | 1 - 3 files changed, 35 insertions(+), 83 deletions(-) delete mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopHttpFullRequest.java diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopHttpFullRequest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopHttpFullRequest.java deleted file mode 100644 index bee4255ab969..000000000000 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/http/NoopHttpFullRequest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.core.http; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import software.amazon.awssdk.annotations.Immutable; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.http.ContentStreamProvider; -import software.amazon.awssdk.http.SdkHttpFullRequest; -import software.amazon.awssdk.http.SdkHttpMethod; - -@SdkInternalApi -@Immutable -public class NoopHttpFullRequest implements SdkHttpFullRequest { - - public void NoopHttpFullRequest() { - - } - @Override - public Builder toBuilder() { - return null; - } - - @Override - public Optional contentStreamProvider() { - return Optional.empty(); - } - - @Override - public Map> headers() { - return null; - } - - @Override - public String protocol() { - return null; - } - - @Override - public String host() { - return null; - } - - @Override - public int port() { - return 0; - } - - @Override - public String encodedPath() { - return null; - } - - @Override - public Map> rawQueryParameters() { - return null; - } - - @Override - public SdkHttpMethod method() { - return null; - } -} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java index 68b81f47a910..5acdfe5f3756 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java @@ -21,24 +21,25 @@ import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.createSdkHttpRequest; import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.progressListenerContext; +import java.util.concurrent.CompletableFuture; import org.junit.jupiter.api.Test; -import org.mockito.Mock; import org.mockito.Mockito; import software.amazon.awssdk.core.Response; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.SdkRequestOverrideConfiguration; -import software.amazon.awssdk.core.http.NoopHttpFullRequest; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import utils.ValidSdkObjects; public class ExecutionFailureExceptionReportingStageTest { @Test - public void afterExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { + public void when_sync_executeThrowsException_attemptFailureInvoked() throws Exception { RequestPipeline> requestPipeline = Mockito.mock(RequestPipeline.class); ProgressListener progressListener = Mockito.mock(ProgressListener.class); @@ -54,7 +55,37 @@ public void afterExecutionProgressListener_calledFrom_ExecutionPipeline() throws ExecutionFailureExceptionReportingStage executionFailureExceptionReportingStage = new ExecutionFailureExceptionReportingStage(requestPipeline); when(requestPipeline.execute(any(), any())).thenThrow(new RuntimeException()); - assertThrows(RuntimeException.class, () -> executionFailureExceptionReportingStage.execute(new NoopHttpFullRequest(), requestExecutionContext)); + assertThrows(RuntimeException.class, () -> executionFailureExceptionReportingStage.execute(ValidSdkObjects.sdkHttpFullRequest().build(), requestExecutionContext)); + + Mockito.verify(progressListener, Mockito.times(0)).requestPrepared(any()); + Mockito.verify(progressListener, Mockito.times(0)).requestBytesSent(any()); + Mockito.verify(progressListener, Mockito.times(0)).responseHeaderReceived(any()); + Mockito.verify(progressListener, Mockito.times(0)).responseBytesReceived(any()); + Mockito.verify(progressListener, Mockito.times(0)).executionSuccess(any()); + Mockito.verify(progressListener, Mockito.times(1)).attemptFailure(any()); + } + + @Test + public void when_async_executeThrowsException_attemptFailureInvoked() throws Exception { + + RequestPipeline requestPipeline = Mockito.mock(RequestPipeline.class); + ProgressListener progressListener = Mockito.mock(ProgressListener.class); + CompletableFuture future = new CompletableFuture<>(); + + SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() + .addProgressListener(progressListener) + .build(); + + SdkRequest request = createSdkHttpRequest(config).build(); + ProgressUpdater progressUpdater = new ProgressUpdater(request, null); + RequestExecutionContext requestExecutionContext = progressListenerContext(false, request, + progressUpdater); + + AsyncExecutionFailureExceptionReportingStage executionFailureExceptionReportingStage = new AsyncExecutionFailureExceptionReportingStage(requestPipeline); + when(requestPipeline.execute(any(), any())).thenReturn(future); + future.completeExceptionally(new RuntimeException()); + + executionFailureExceptionReportingStage.execute(ValidSdkObjects.sdkHttpFullRequest().build(), requestExecutionContext); Mockito.verify(progressListener, Mockito.times(0)).requestPrepared(any()); Mockito.verify(progressListener, Mockito.times(0)).requestBytesSent(any()); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java index ca9408b8f0b9..e3a2796244b3 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/ProgressListenerTestUtils.java @@ -22,7 +22,6 @@ import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.http.ExecutionContext; -import software.amazon.awssdk.core.http.NoopHttpFullRequest; import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain; import software.amazon.awssdk.core.interceptor.InterceptorContext; From 89b9f0ac25c124bcc50efad6195afd17d3417016 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Fri, 5 Jul 2024 06:29:02 -0700 Subject: [PATCH 46/50] Fixed checkstyle --- .../http/pipeline/stages/MakeAsyncHttpRequestStage.java | 4 ++-- .../http/pipeline/stages/MakeHttpRequestStage.java | 1 - .../awssdk/core/internal/util/ProgressListenerUtils.java | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index 7ebc4a358cbf..507df2572d3d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -51,7 +51,6 @@ import software.amazon.awssdk.core.internal.util.DownloadProgressUpdaterInvocation; import software.amazon.awssdk.core.internal.util.MetricUtils; import software.amazon.awssdk.core.internal.util.ProgressListenerUtils; -import software.amazon.awssdk.core.internal.util.UploadProgressUpdaterInvocation; import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; @@ -136,7 +135,8 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque SdkHttpContentPublisher requestProvider = context.requestProvider() == null ? new SimpleHttpContentPublisher(request) : new SdkHttpContentPublisherAdapter(context.requestProvider()); - requestProvider = ProgressListenerUtils.wrapRequestProviderWithByteTrackingIfProgressListenerAttached(requestProvider, context); + requestProvider = ProgressListenerUtils.wrapRequestProviderWithByteTrackingIfProgressListenerAttached(requestProvider, + context); // Set content length if it hasn't been set already. SdkHttpFullRequest requestWithContentLength = getRequestWithContentLength(request, requestProvider); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java index 92c3003e377a..32cd094a57de 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java @@ -55,7 +55,6 @@ public Pair execute(SdkHttpFullRequest RequestExecutionContext context) throws Exception { InterruptMonitor.checkInterrupted(); HttpExecuteResponse executeResponse = executeHttpRequest(request, context); - // TODO: Plumb through ExecuteResponse instead SdkHttpFullResponse httpResponse = (SdkHttpFullResponse) executeResponse.httpResponse(); return Pair.of(request, httpResponse.toBuilder().content(executeResponse.responseBody().orElse(null)).build()); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java index c769c5929f74..99f71a096b57 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/ProgressListenerUtils.java @@ -17,9 +17,7 @@ import static software.amazon.awssdk.http.Header.CONTENT_LENGTH; -import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicLong; -import org.reactivestreams.Publisher; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.RequestOverrideConfiguration; import software.amazon.awssdk.core.SdkRequest; @@ -37,11 +35,13 @@ public final class ProgressListenerUtils { private ProgressListenerUtils() { } - public static SdkHttpContentPublisher wrapRequestProviderWithByteTrackingIfProgressListenerAttached(SdkHttpContentPublisher requestProvider, RequestExecutionContext context) { + public static SdkHttpContentPublisher wrapRequestProviderWithByteTrackingIfProgressListenerAttached( + SdkHttpContentPublisher requestProvider, RequestExecutionContext context) { ProgressUpdater progressUpdater = getProgressUpdaterIfAttached(context); SdkHttpContentPublisher wrappedHttpContentPublisher = requestProvider; if (progressUpdater != null) { - wrappedHttpContentPublisher = new BytesReadTrackingPublisher(requestProvider, new AtomicLong(0L), new UploadProgressUpdaterInvocation(progressUpdater)); + wrappedHttpContentPublisher = new BytesReadTrackingPublisher(requestProvider, new AtomicLong(0L), + new UploadProgressUpdaterInvocation(progressUpdater)); } return wrappedHttpContentPublisher; } From 13a846132d2cc1d30a22ce38a3595cff3ed68cc4 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Fri, 5 Jul 2024 10:56:58 -0700 Subject: [PATCH 47/50] Modified method call signatures --- .../amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java b/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java index b4e0dcc6622b..73a70ab4ea02 100644 --- a/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java +++ b/core/imds/src/main/java/software/amazon/awssdk/imds/internal/AsyncHttpRequestHelper.java @@ -61,7 +61,7 @@ private static CompletableFuture sendAsync(SdkAsyncHttpClient client, SdkHttpFullRequest request, HttpResponseHandler handler, CompletableFuture parentFuture) { - SdkHttpContentPublisher requestContentPublisher = new SimpleHttpContentPublisher(request, null); + SdkHttpContentPublisher requestContentPublisher = new SimpleHttpContentPublisher(request); TransformingAsyncResponseHandler responseHandler = new AsyncResponseHandler<>(handler, Function.identity(), new ExecutionAttributes()); CompletableFuture responseHandlerFuture = responseHandler.prepare(); From fa580e420a5dfd8622ade358900f0506524454ae Mon Sep 17 00:00:00 2001 From: Krishnan Date: Fri, 5 Jul 2024 11:08:43 -0700 Subject: [PATCH 48/50] Added comments to update BytesReadTrackingPublishee class --- .../core/internal/metrics/BytesReadTrackingPublisher.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java index 09ee3063ffba..47df209c9b57 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisher.java @@ -27,6 +27,8 @@ /** * Publisher that tracks how many bytes are published from the wrapped publisher to the downstream subscriber. + * If request contains Progress Listeners attached, the callbacks invoke methods to update and track request status + * by invoking progress updater methods with the bytes being transacted */ @SdkInternalApi public final class BytesReadTrackingPublisher implements SdkHttpContentPublisher { From b40b3e152e561d9078e4fd59ec241d891873065e Mon Sep 17 00:00:00 2001 From: Krishnan Date: Fri, 5 Jul 2024 11:27:11 -0700 Subject: [PATCH 49/50] Revert "Merge branch 'feature/anirudkr-progress-listener' into dev/progress-listener" This reverts commit 2d9cfe70193af6a590b84b97c39ad8083e65983b, reversing changes made to fa580e420a5dfd8622ade358900f0506524454ae. --- .../awssdk/core/http/ExecutionContext.java | 15 ------- .../progress/ProgressListenerContext.java | 8 ++-- .../progress/listener/ProgressUpdater.java | 11 +++-- .../progress/listener/ProgressListener.java | 6 +-- .../listener/ProgressUpdaterTest.java | 45 ++++++++++++++++--- 5 files changed, 55 insertions(+), 30 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java index 01cc3e0c5166..141ba473986f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/ExecutionContext.java @@ -15,13 +15,11 @@ package software.amazon.awssdk.core.http; -import java.util.Optional; import software.amazon.awssdk.annotations.NotThreadSafe; import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain; import software.amazon.awssdk.core.interceptor.InterceptorContext; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.signer.Signer; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.builder.CopyableBuilder; @@ -38,7 +36,6 @@ public final class ExecutionContext implements ToCopyableBuilder progressUpdater() { - return progressUpdater != null ? Optional.of(progressUpdater) : Optional.empty(); - } - @Override public Builder toBuilder() { return new Builder(this); @@ -95,7 +87,6 @@ public static class Builder implements CopyableBuilder response() { + return Optional.of(response); } @Override @@ -158,4 +159,5 @@ public ProgressListenerContext build() { return new ProgressListenerContext(this); } } -} \ No newline at end of file +} + diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java index d72389f05ab2..ef6729b374f2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdater.java @@ -26,6 +26,7 @@ import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; import software.amazon.awssdk.core.progress.listener.SdkExchangeProgress; import software.amazon.awssdk.core.progress.snapshot.ProgressSnapshot; +import software.amazon.awssdk.http.SdkHttpRequest; /** * ProgressUpdater exposes methods that invokes listener methods to update and store request progress state @@ -62,6 +63,10 @@ public ProgressUpdater(SdkRequest sdkRequest, .orElse(Collections.emptyList())); } + public void updateRequestContentLength(Long requestContentLength) { + requestBodyProgress.updateAndGet(b -> b.totalBytes(requestContentLength)); + } + public void updateResponseContentLength(Long responseContentLength) { responseBodyProgress.updateAndGet(b -> b.totalBytes(responseContentLength)); } @@ -74,8 +79,8 @@ public SdkExchangeProgress responseBodyProgress() { return responseBodyProgress; } - public void requestPrepared() { - listenerInvoker.requestPrepared(context); + public void requestPrepared(SdkHttpRequest httpRequest) { + listenerInvoker.requestPrepared(context.copy(b -> b.httpRequest(httpRequest))); } public void requestHeaderSent() { @@ -154,4 +159,4 @@ public void attemptFailureResponseBytesReceived(Throwable t) { .exception(t) .build()); } -} \ No newline at end of file +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java index 4a2e2a4df24d..d401c88d5942 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/progress/listener/ProgressListener.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.core.progress.listener; +import java.util.Optional; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkProtectedApi; @@ -189,8 +190,7 @@ default void responseBytesReceived(Context.ResponseBytesReceived context) { /** * For Expect: 100-continue embedded requests, the service returning anything other than 100 continue * indicates a request failure. This method captures the error in the payload - * After this, either executionFailure or requestHeaderSent will always be invoked depending on - * whether the error type is retryable or not + * After this it will either be an executionFailure or a request retry *

      * Available context attributes: *

        @@ -423,7 +423,7 @@ public interface ExecutionSuccess extends ResponseBytesReceived { /** * The successful completion of a request submitted to the Sdk */ - SdkResponse response(); + Optional response(); } /** diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java index 0e2f77380ab9..f4d08c19872f 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; +import java.net.URI; import java.util.Arrays; import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; @@ -36,6 +37,8 @@ import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.core.protocol.VoidSdkResponse; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; public class ProgressUpdaterTest { private CaptureProgressListener captureProgressListener; @@ -74,7 +77,7 @@ public void requestPrepared_transferredBytes_equals_zero() { .build(); ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); - progressUpdater.requestPrepared(); + progressUpdater.requestPrepared(createHttpRequest()); assertEquals(0.0, progressUpdater.requestBodyProgress().progressSnapshot().transferredBytes(), 0.0); assertTrue(captureProgressListener.requestPrepared()); @@ -206,13 +209,37 @@ public void ratioTransferred_upload_transferredBytes(long contentLength) { .overrideConfiguration(overrideConfig) .build(); - ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, contentLength); + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.updateRequestContentLength(contentLength); progressUpdater.incrementBytesSent(BYTES_TRANSFERRED); assertEquals((double) BYTES_TRANSFERRED / contentLength, progressUpdater.requestBodyProgress().progressSnapshot().ratioTransferred().getAsDouble(), 0.0); } + @ParameterizedTest + @MethodSource("contentLength") + public void ratioTransferred_download_transferredBytes(long contentLength) { + + CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); + + SdkRequestOverrideConfiguration.Builder builder = SdkRequestOverrideConfiguration.builder(); + builder.progressListeners(Arrays.asList(mockListener, captureProgressListener)); + + SdkRequestOverrideConfiguration overrideConfig = builder.build(); + + SdkRequest sdkRequest = NoopTestRequest.builder() + .overrideConfiguration(overrideConfig) + .build(); + + ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); + progressUpdater.updateResponseContentLength(contentLength); + progressUpdater.incrementBytesReceived(BYTES_TRANSFERRED); + assertEquals((double) BYTES_TRANSFERRED / contentLength, + progressUpdater.responseBodyProgress().progressSnapshot().ratioTransferred().getAsDouble(), 0.0); + + } + @Test public void responseHeaderReceived_transferredBytes_equals_zero() { @@ -286,7 +313,7 @@ public void attemptFailureResponseBytesReceived() { .build(); ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); - progressUpdater.requestPrepared(); + progressUpdater.requestPrepared(createHttpRequest()); progressUpdater.responseHeaderReceived(); progressUpdater.attemptFailureResponseBytesReceived(attemptFailureResponseBytesReceived); @@ -314,7 +341,7 @@ public void attemptFailure() { .build(); ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); - progressUpdater.requestPrepared(); + progressUpdater.requestPrepared(createHttpRequest()); progressUpdater.attemptFailure(attemptFailure); Mockito.verify(mockListener, times(1)).requestPrepared(ArgumentMatchers.any(ProgressListener.Context.RequestPrepared.class)); @@ -343,7 +370,7 @@ public void executionFailure() { .build(); ProgressUpdater progressUpdater = new ProgressUpdater(sdkRequest, null); - progressUpdater.requestPrepared(); + progressUpdater.requestPrepared(createHttpRequest()); progressUpdater.executionFailure(executionFailure); @@ -357,4 +384,10 @@ public void executionFailure() { Assertions.assertEquals(captureProgressListener.exceptionCaught().getMessage(), executionFailure.getMessage()); } -} \ No newline at end of file + + private SdkHttpFullRequest createHttpRequest() { + return SdkHttpFullRequest.builder().uri(URI.create("https://endpoint.host")) + .method(SdkHttpMethod.GET) + .build(); + } +} From ab9445d546e5b50af702da94d2e01ebd4de07903 Mon Sep 17 00:00:00 2001 From: Krishnan Date: Tue, 9 Jul 2024 13:36:29 -0700 Subject: [PATCH 50/50] Improve code quality --- ...BeforeExecutionProgressReportingStage.java | 2 +- ...erExecutionProgressReportingStageTest.java | 8 ++---- ...reExecutionProgressReportingStageTest.java | 4 +-- ...ionFailureExceptionReportingStageTest.java | 6 ++--- ...ientExecutionAndRequestTimerTestUtils.java | 1 - .../BytesReadTrackingPublisherTckTest.java | 2 -- .../BytesReadTrackingPublisherTest.java | 5 +--- .../listener/ProgressUpdaterTest.java | 26 +++++++++---------- .../snapshot/DefaultProgressSnapshotTest.java | 17 +++++------- 9 files changed, 29 insertions(+), 42 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStage.java index 1f86cc0d3860..76f833b7459c 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStage.java @@ -30,7 +30,7 @@ public SdkHttpFullRequest execute(SdkHttpFullRequest input, RequestExecutionCont if (ProgressListenerUtils.progressListenerAttached(context.originalRequest())) { Long requestContentLength = - context.requestProvider() != null && context.requestProvider().contentLength().isPresent() ? + (context.requestProvider() != null && context.requestProvider().contentLength().isPresent()) ? context.requestProvider().contentLength().get() : null; ProgressUpdater progressUpdater = new ProgressUpdater(context.originalRequest(), requestContentLength); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStageTest.java index 582a1762c9ec..efca721dac06 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStageTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AfterExecutionProgressReportingStageTest.java @@ -15,12 +15,10 @@ package software.amazon.awssdk.core.internal.http.pipeline.stages; -import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.REQUEST_BODY; import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.createSdkHttpRequest; import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.createSdkResponseBuilder; import static software.amazon.awssdk.core.internal.util.ProgressListenerTestUtils.progressListenerContext; -import java.net.URI; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import software.amazon.awssdk.core.SdkRequest; @@ -29,13 +27,11 @@ import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.progress.listener.ProgressListener; -import software.amazon.awssdk.http.SdkHttpFullRequest; -import software.amazon.awssdk.http.SdkHttpMethod; -public class AfterExecutionProgressReportingStageTest { +class AfterExecutionProgressReportingStageTest { @Test - public void afterExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { + void afterExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { ProgressListener progressListener = Mockito.mock(ProgressListener.class); SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStageTest.java index 38e0554b254c..a311808ca02a 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStageTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/BeforeExecutionProgressReportingStageTest.java @@ -28,9 +28,9 @@ import software.amazon.awssdk.core.progress.listener.ProgressListener; import software.amazon.awssdk.http.SdkHttpFullRequest; -public class BeforeExecutionProgressReportingStageTest { +class BeforeExecutionProgressReportingStageTest { @Test - public void beforeExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { + void beforeExecutionProgressListener_calledFrom_ExecutionPipeline() throws Exception { ProgressListener progressListener = Mockito.mock(ProgressListener.class); SdkRequestOverrideConfiguration config = SdkRequestOverrideConfiguration.builder() diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java index 5acdfe5f3756..511aeaa9c9a0 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ExecutionFailureExceptionReportingStageTest.java @@ -35,11 +35,11 @@ import software.amazon.awssdk.http.SdkHttpFullResponse; import utils.ValidSdkObjects; -public class ExecutionFailureExceptionReportingStageTest { +class ExecutionFailureExceptionReportingStageTest { @Test - public void when_sync_executeThrowsException_attemptFailureInvoked() throws Exception { + void when_sync_executeThrowsException_attemptFailureInvoked() throws Exception { RequestPipeline> requestPipeline = Mockito.mock(RequestPipeline.class); ProgressListener progressListener = Mockito.mock(ProgressListener.class); @@ -66,7 +66,7 @@ public void when_sync_executeThrowsException_attemptFailureInvoked() throws Exce } @Test - public void when_async_executeThrowsException_attemptFailureInvoked() throws Exception { + void when_async_executeThrowsException_attemptFailureInvoked() throws Exception { RequestPipeline requestPipeline = Mockito.mock(RequestPipeline.class); ProgressListener progressListener = Mockito.mock(ProgressListener.class); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java index faf4691b67fd..b4923e2b363e 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/timers/ClientExecutionAndRequestTimerTestUtils.java @@ -31,7 +31,6 @@ import software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient; import software.amazon.awssdk.core.internal.http.response.ErrorDuringUnmarshallingResponseHandler; import software.amazon.awssdk.core.internal.http.response.NullErrorResponseHandler; -import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.signer.NoOpSigner; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java index 39907553bced..26945aa75daa 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTckTest.java @@ -17,14 +17,12 @@ import io.reactivex.Flowable; import java.nio.ByteBuffer; -import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; import org.reactivestreams.Publisher; import org.reactivestreams.tck.PublisherVerification; import org.reactivestreams.tck.TestEnvironment; -import software.amazon.awssdk.core.internal.util.DownloadProgressUpdaterInvocation; import software.amazon.awssdk.core.internal.util.UploadProgressUpdaterInvocation; /** diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java index 8250b6351c4f..df7349181724 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/metrics/BytesReadTrackingPublisherTest.java @@ -19,7 +19,6 @@ import io.reactivex.Flowable; import java.nio.ByteBuffer; -import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -32,10 +31,8 @@ import software.amazon.awssdk.core.http.NoopTestRequest; import software.amazon.awssdk.core.internal.progress.listener.ProgressUpdater; import software.amazon.awssdk.core.internal.util.DownloadProgressUpdaterInvocation; -import software.amazon.awssdk.core.internal.util.ProgressUpdaterInvoker; import software.amazon.awssdk.core.internal.util.UploadProgressUpdaterInvocation; import software.amazon.awssdk.core.progress.listener.ProgressListener; -import software.amazon.awssdk.http.SdkHttpFullRequest; /** * Functional tests for {@link BytesReadTrackingPublisher}. @@ -71,7 +68,7 @@ public void requestAll_updatesInputCount() { } @Test - public void progressUpdater_invokes_incrementBytesReceived() { + void progressUpdater_invokes_incrementBytesReceived() { int nElements = 8; int elementSize = 2; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java index f4d08c19872f..6842db999002 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/listener/ProgressUpdaterTest.java @@ -40,7 +40,7 @@ import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; -public class ProgressUpdaterTest { +class ProgressUpdaterTest { private CaptureProgressListener captureProgressListener; private static final long BYTES_TRANSFERRED = 5L; private static final Throwable attemptFailure = new Throwable("AttemptFailureException"); @@ -63,7 +63,7 @@ private static Stream contentLength() { } @Test - public void requestPrepared_transferredBytes_equals_zero() { + void requestPrepared_transferredBytes_equals_zero() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -92,7 +92,7 @@ public void requestPrepared_transferredBytes_equals_zero() { } @Test - public void requestHeaderSent_transferredBytes_equals_zero() { + void requestHeaderSent_transferredBytes_equals_zero() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -120,7 +120,7 @@ public void requestHeaderSent_transferredBytes_equals_zero() { } @Test - public void requestBytesSent_transferredBytes() { + void requestBytesSent_transferredBytes() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -149,7 +149,7 @@ public void requestBytesSent_transferredBytes() { } @Test - public void validate_resetBytesSent() { + void validate_resetBytesSent() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -172,7 +172,7 @@ public void validate_resetBytesSent() { } @Test - public void validate_resetBytesReceived() { + void validate_resetBytesReceived() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -196,7 +196,7 @@ public void validate_resetBytesReceived() { @ParameterizedTest @MethodSource("contentLength") - public void ratioTransferred_upload_transferredBytes(long contentLength) { + void ratioTransferred_upload_transferredBytes(long contentLength) { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -219,7 +219,7 @@ public void ratioTransferred_upload_transferredBytes(long contentLength) { @ParameterizedTest @MethodSource("contentLength") - public void ratioTransferred_download_transferredBytes(long contentLength) { + void ratioTransferred_download_transferredBytes(long contentLength) { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -241,7 +241,7 @@ public void ratioTransferred_download_transferredBytes(long contentLength) { } @Test - public void responseHeaderReceived_transferredBytes_equals_zero() { + void responseHeaderReceived_transferredBytes_equals_zero() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -269,7 +269,7 @@ public void responseHeaderReceived_transferredBytes_equals_zero() { } @Test - public void executionSuccess_transferredBytes_valid() { + void executionSuccess_transferredBytes_valid() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -299,7 +299,7 @@ public void executionSuccess_transferredBytes_valid() { } @Test - public void attemptFailureResponseBytesReceived() { + void attemptFailureResponseBytesReceived() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -327,7 +327,7 @@ public void attemptFailureResponseBytesReceived() { } @Test - public void attemptFailure() { + void attemptFailure() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); @@ -356,7 +356,7 @@ public void attemptFailure() { } @Test - public void executionFailure() { + void executionFailure() { CaptureProgressListener mockListener = Mockito.mock(CaptureProgressListener.class); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java index b2d125f656d1..2d4404556f6f 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/progress/snapshot/DefaultProgressSnapshotTest.java @@ -15,10 +15,8 @@ package software.amazon.awssdk.core.internal.progress.snapshot; -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import java.time.Duration; import java.time.Instant; @@ -29,9 +27,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import software.amazon.awssdk.core.internal.progress.snapshot.DefaultProgressSnapshot; -public class DefaultProgressSnapshotTest { +class DefaultProgressSnapshotTest { private static Stream getArgumentsForInvalidParameterValidationTests() { return Stream.of(Arguments.of("transferredBytes (2) must not be greater than totalBytes (1)", @@ -112,7 +109,7 @@ private static Stream getArgumentsForBytesTest() { @ParameterizedTest @MethodSource("getArgumentsForInvalidParameterValidationTests") - public void test_invalid_arguments_shouldThrow(String expectedErrorMsg, DefaultProgressSnapshot.Builder builder, + void test_invalid_arguments_shouldThrow(String expectedErrorMsg, DefaultProgressSnapshot.Builder builder, Exception e) { assertThatThrownBy(builder::build) .isInstanceOf(e.getClass()) @@ -121,12 +118,12 @@ public void test_invalid_arguments_shouldThrow(String expectedErrorMsg, DefaultP @ParameterizedTest @MethodSource("getArgumentsForMissingParameterValidationTests") - public void test_missing_params_shouldReturnEmpty(boolean condition) { + void test_missing_params_shouldReturnEmpty(boolean condition) { Assertions.assertFalse(condition); } @Test - public void ratioTransferred() { + void ratioTransferred() { DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() .transferredBytes(1L) .totalBytes(5L) @@ -137,18 +134,18 @@ public void ratioTransferred() { @ParameterizedTest @MethodSource("getArgumentsForBytesTest") - public void test_estimatedBytesRemaining_and_totalBytes(long expectedBytes, long actualBytes) { + void test_estimatedBytesRemaining_and_totalBytes(long expectedBytes, long actualBytes) { Assertions.assertEquals(expectedBytes, actualBytes); } @ParameterizedTest @MethodSource("getArgumentsForTimeTest") - public void test_elapsedTime_and_estimatedTimeRemaining(long expected, long timeInMillis, long delta) { + void test_elapsedTime_and_estimatedTimeRemaining(long expected, long timeInMillis, long delta) { Assertions.assertEquals(expected, timeInMillis, delta); } @Test - public void averageBytesPer() { + void averageBytesPer() { DefaultProgressSnapshot snapshot = DefaultProgressSnapshot.builder() .transferredBytes(100L) .startTime(Instant.now().minusMillis(100))