From 0209113e09eb113bd55c3372f846f2712be20f1a Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 23 Oct 2025 19:15:33 -0400 Subject: [PATCH] Add DownloadInitiated, Failed and Completed events stack-info: PR: https://github.com/aws/aws-sdk-net/pull/4079, branch: GarrettBeatty/stacked/10 --- .../9d07dc1e-d82d-4f94-8700-c7b57f872123.json | 11 + .../S3/Custom/Model/GetObjectResponse.cs | 6 + .../Transfer/Internal/DownloadCommand.cs | 34 +++ .../Internal/_async/DownloadCommand.async.cs | 19 +- .../TransferUtilityDownloadRequest.cs | 249 ++++++++++++++++++ .../IntegrationTests/TransferUtilityTests.cs | 185 +++++++++++++ 6 files changed, 502 insertions(+), 2 deletions(-) create mode 100644 generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f872123.json diff --git a/generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f872123.json b/generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f872123.json new file mode 100644 index 000000000000..34e896d28f75 --- /dev/null +++ b/generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f872123.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "minor", + "changeLogMessages": [ + "Added DownloadInitiatedEvent, DownloadCompletedEvent, and DownloadFailedEvent for downloads." + ] + } + ] +} \ No newline at end of file diff --git a/sdk/src/Services/S3/Custom/Model/GetObjectResponse.cs b/sdk/src/Services/S3/Custom/Model/GetObjectResponse.cs index bae8fc4147b5..44c3eaddc6fe 100644 --- a/sdk/src/Services/S3/Custom/Model/GetObjectResponse.cs +++ b/sdk/src/Services/S3/Custom/Model/GetObjectResponse.cs @@ -25,6 +25,7 @@ using Amazon.S3.Model.Internal.MarshallTransformations; using Amazon.S3; using Amazon.Runtime.Internal; +using Amazon.S3.Transfer; namespace Amazon.S3.Model { @@ -1042,5 +1043,10 @@ internal WriteObjectProgressArgs(string bucketName, string key, string filePath, /// True if writing is complete /// public bool IsCompleted { get; private set; } + + /// + /// The original TransferUtilityDownloadRequest created by the user. + /// + public TransferUtilityDownloadRequest Request { get; internal set; } } } diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/DownloadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/DownloadCommand.cs index f8e45d7b20fe..bca43c615b05 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/DownloadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/DownloadCommand.cs @@ -62,6 +62,34 @@ static Logger Logger IAmazonS3 _s3Client; TransferUtilityDownloadRequest _request; + long _totalTransferredBytes; + + #region Event Firing Methods + + private void FireTransferInitiatedEvent() + { + var transferInitiatedEventArgs = new DownloadInitiatedEventArgs(_request, _request.FilePath); + _request.OnRaiseTransferInitiatedEvent(transferInitiatedEventArgs); + } + + private void FireTransferCompletedEvent(TransferUtilityDownloadResponse response, string filePath, long transferredBytes, long totalBytes) + { + var transferCompletedEventArgs = new DownloadCompletedEventArgs( + _request, + response, + filePath, + transferredBytes, + totalBytes); + _request.OnRaiseTransferCompletedEvent(transferCompletedEventArgs); + } + + private void FireTransferFailedEvent(string filePath, long transferredBytes, long totalBytes = -1) + { + var eventArgs = new DownloadFailedEventArgs(this._request, filePath, transferredBytes, totalBytes); + this._request.OnRaiseTransferFailedEvent(eventArgs); + } + + #endregion internal DownloadCommand(IAmazonS3 s3Client, TransferUtilityDownloadRequest request) { @@ -89,6 +117,12 @@ private void ValidateRequest() void OnWriteObjectProgressEvent(object sender, WriteObjectProgressArgs e) { + // Keep track of the total transferred bytes so that we can also return this value in case of failure + Interlocked.Add(ref _totalTransferredBytes, e.IncrementTransferred); + + // Set the Request property to enable access to the original download request + e.Request = this._request; + this._request.OnRaiseProgressEvent(e); } diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/DownloadCommand.async.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/DownloadCommand.async.cs index 6baef9262774..057f7705fd0a 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/DownloadCommand.async.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/DownloadCommand.async.cs @@ -33,12 +33,17 @@ internal partial class DownloadCommand : BaseCommand ExecuteAsync(CancellationToken cancellationToken) { ValidateRequest(); + + FireTransferInitiatedEvent(); + GetObjectRequest getRequest = ConvertToGetObjectRequest(this._request); var maxRetries = _s3Client.Config.MaxErrorRetry; var retries = 0; bool shouldRetry = false; string mostRecentETag = null; + TransferUtilityDownloadResponse lastSuccessfulMappedResponse = null; + long? totalBytesFromResponse = null; // Track total bytes once we have response headers do { shouldRetry = false; @@ -54,12 +59,16 @@ public override async Task ExecuteAsync(Cancell using (var response = await this._s3Client.GetObjectAsync(getRequest, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false)) { + // Capture total bytes from response headers as soon as we get them + totalBytesFromResponse = response.ContentLength; + if (!string.IsNullOrEmpty(mostRecentETag) && !string.Equals(mostRecentETag, response.ETag)) { //if the eTag changed, we need to retry from the start of the file mostRecentETag = response.ETag; getRequest.ByteRange = null; retries = 0; + Interlocked.Exchange(ref _totalTransferredBytes, 0); shouldRetry = true; WaitBeforeRetry(retries); continue; @@ -101,6 +110,8 @@ await response.WriteResponseStreamToFileAsync(this._request.FilePath, false, can await response.WriteResponseStreamToFileAsync(this._request.FilePath, true, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); } + + lastSuccessfulMappedResponse = ResponseMapper.MapGetObjectResponse(response); } } catch (Exception exception) @@ -109,6 +120,9 @@ await response.WriteResponseStreamToFileAsync(this._request.FilePath, true, canc shouldRetry = HandleExceptionForHttpClient(exception, retries, maxRetries); if (!shouldRetry) { + // Pass total bytes if we have them from response headers, otherwise -1 for unknown + FireTransferFailedEvent(this._request.FilePath, Interlocked.Read(ref _totalTransferredBytes), totalBytesFromResponse ?? -1); + if (exception is IOException) { throw; @@ -131,8 +145,9 @@ await response.WriteResponseStreamToFileAsync(this._request.FilePath, true, canc WaitBeforeRetry(retries); } while (shouldRetry); - // TODO map and return response - return new TransferUtilityDownloadResponse(); + FireTransferCompletedEvent(lastSuccessfulMappedResponse, this._request.FilePath, Interlocked.Read(ref _totalTransferredBytes), totalBytesFromResponse ?? -1); + + return lastSuccessfulMappedResponse; } private static bool HandleExceptionForHttpClient(Exception exception, int retries, int maxRetries) diff --git a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityDownloadRequest.cs b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityDownloadRequest.cs index d9a4bc5c7119..f7ba5f97b943 100644 --- a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityDownloadRequest.cs +++ b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityDownloadRequest.cs @@ -90,5 +90,254 @@ internal void OnRaiseProgressEvent(WriteObjectProgressArgs progressArgs) { AWSSDKUtils.InvokeInBackground(WriteObjectProgressEvent, progressArgs, this); } + + /// + /// The event for DownloadInitiatedEvent notifications. All + /// subscribers will be notified when a download transfer operation + /// starts. + /// + /// The DownloadInitiatedEvent is fired exactly once when + /// a download transfer operation begins. The delegates attached to the event + /// will be passed information about the download request and + /// file path, but no progress information. + /// + /// + /// + /// Subscribe to this event if you want to receive + /// DownloadInitiatedEvent notifications. Here is how:
+ /// 1. Define a method with a signature similar to this one: + /// + /// private void downloadStarted(object sender, DownloadInitiatedEventArgs args) + /// { + /// Console.WriteLine($"Download started: {args.FilePath}"); + /// Console.WriteLine($"Bucket: {args.Request.BucketName}"); + /// Console.WriteLine($"Key: {args.Request.Key}"); + /// } + /// + /// 2. Add this method to the DownloadInitiatedEvent delegate's invocation list + /// + /// TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest(); + /// request.DownloadInitiatedEvent += downloadStarted; + /// + /// + public event EventHandler DownloadInitiatedEvent; + + /// + /// The event for DownloadCompletedEvent notifications. All + /// subscribers will be notified when a download transfer operation + /// completes successfully. + /// + /// The DownloadCompletedEvent is fired exactly once when + /// a download transfer operation completes successfully. The delegates attached to the event + /// will be passed information about the completed download including + /// the final response from S3 with ETag, VersionId, and other metadata. + /// + /// + /// + /// Subscribe to this event if you want to receive + /// DownloadCompletedEvent notifications. Here is how:
+ /// 1. Define a method with a signature similar to this one: + /// + /// private void downloadCompleted(object sender, DownloadCompletedEventArgs args) + /// { + /// Console.WriteLine($"Download completed: {args.FilePath}"); + /// Console.WriteLine($"Transferred: {args.TransferredBytes} bytes"); + /// Console.WriteLine($"ETag: {args.Response.ETag}"); + /// Console.WriteLine($"S3 Key: {args.Response.Key}"); + /// Console.WriteLine($"Version ID: {args.Response.VersionId}"); + /// } + /// + /// 2. Add this method to the DownloadCompletedEvent delegate's invocation list + /// + /// TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest(); + /// request.DownloadCompletedEvent += downloadCompleted; + /// + /// + public event EventHandler DownloadCompletedEvent; + + /// + /// The event for DownloadFailedEvent notifications. All + /// subscribers will be notified when a download transfer operation + /// fails. + /// + /// The DownloadFailedEvent is fired exactly once when + /// a download transfer operation fails. The delegates attached to the event + /// will be passed information about the failed download including + /// partial progress information, but no response data since the download failed. + /// + /// + /// + /// Subscribe to this event if you want to receive + /// DownloadFailedEvent notifications. Here is how:
+ /// 1. Define a method with a signature similar to this one: + /// + /// private void downloadFailed(object sender, DownloadFailedEventArgs args) + /// { + /// Console.WriteLine($"Download failed: {args.FilePath}"); + /// Console.WriteLine($"Partial progress: {args.TransferredBytes} bytes"); + /// Console.WriteLine($"Bucket: {args.Request.BucketName}"); + /// Console.WriteLine($"Key: {args.Request.Key}"); + /// } + /// + /// 2. Add this method to the DownloadFailedEvent delegate's invocation list + /// + /// TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest(); + /// request.DownloadFailedEvent += downloadFailed; + /// + /// + public event EventHandler DownloadFailedEvent; + + /// + /// Causes the DownloadInitiatedEvent event to be fired. + /// + /// DownloadInitiatedEventArgs args + internal void OnRaiseTransferInitiatedEvent(DownloadInitiatedEventArgs args) + { + AWSSDKUtils.InvokeInBackground(DownloadInitiatedEvent, args, this); + } + + /// + /// Causes the DownloadCompletedEvent event to be fired. + /// + /// DownloadCompletedEventArgs args + internal void OnRaiseTransferCompletedEvent(DownloadCompletedEventArgs args) + { + AWSSDKUtils.InvokeInBackground(DownloadCompletedEvent, args, this); + } + + /// + /// Causes the DownloadFailedEvent event to be fired. + /// + /// DownloadFailedEventArgs args + internal void OnRaiseTransferFailedEvent(DownloadFailedEventArgs args) + { + AWSSDKUtils.InvokeInBackground(DownloadFailedEvent, args, this); + } + } + + /// + /// Encapsulates the information needed when a download transfer operation is initiated. + /// Provides access to the original request without progress or total byte information. + /// + public class DownloadInitiatedEventArgs : EventArgs + { + /// + /// Initializes a new instance of the DownloadInitiatedEventArgs class. + /// + /// The original TransferUtilityDownloadRequest created by the user + /// The file being downloaded + internal DownloadInitiatedEventArgs(TransferUtilityDownloadRequest request, string filePath) + { + Request = request; + FilePath = filePath; + } + + /// + /// The original TransferUtilityDownloadRequest created by the user. + /// Contains all the download parameters and configuration. + /// + public TransferUtilityDownloadRequest Request { get; private set; } + + /// + /// Gets the file being downloaded. + /// + public string FilePath { get; private set; } + } + + /// + /// Encapsulates the information needed when a download transfer operation completes successfully. + /// Provides access to the original request, final response, and completion details. + /// + public class DownloadCompletedEventArgs : EventArgs + { + /// + /// Initializes a new instance of the DownloadCompletedEventArgs class. + /// + /// The original TransferUtilityDownloadRequest created by the user + /// The unified response from Transfer Utility + /// The file being downloaded + /// The total number of bytes transferred + /// The total number of bytes for the complete file + internal DownloadCompletedEventArgs(TransferUtilityDownloadRequest request, TransferUtilityDownloadResponse response, string filePath, long transferredBytes, long totalBytes) + { + Request = request; + Response = response; + FilePath = filePath; + TransferredBytes = transferredBytes; + TotalBytes = totalBytes; + } + + /// + /// The original TransferUtilityDownloadRequest created by the user. + /// Contains all the download parameters and configuration. + /// + public TransferUtilityDownloadRequest Request { get; private set; } + + /// + /// The unified response from Transfer Utility after successful download completion. + /// Contains mapped fields from GetObjectResponse. + /// + public TransferUtilityDownloadResponse Response { get; private set; } + + /// + /// Gets the file being downloaded. + /// + public string FilePath { get; private set; } + + /// + /// Gets the total number of bytes that were successfully transferred. + /// + public long TransferredBytes { get; private set; } + + /// + /// Gets the total number of bytes for the complete file. + /// + public long TotalBytes { get; private set; } + } + + /// + /// Encapsulates the information needed when a download transfer operation fails. + /// Provides access to the original request and partial progress information. + /// + public class DownloadFailedEventArgs : EventArgs + { + /// + /// Initializes a new instance of the DownloadFailedEventArgs class. + /// + /// The original TransferUtilityDownloadRequest created by the user + /// The file being downloaded + /// The number of bytes transferred before failure + /// The total number of bytes for the complete file, or -1 if unknown + internal DownloadFailedEventArgs(TransferUtilityDownloadRequest request, string filePath, long transferredBytes, long totalBytes) + { + Request = request; + FilePath = filePath; + TransferredBytes = transferredBytes; + TotalBytes = totalBytes; + } + + /// + /// The original TransferUtilityDownloadRequest created by the user. + /// Contains all the download parameters and configuration. + /// + public TransferUtilityDownloadRequest Request { get; private set; } + + /// + /// Gets the file being downloaded. + /// + public string FilePath { get; private set; } + + /// + /// Gets the number of bytes that were transferred before the failure occurred. + /// + public long TransferredBytes { get; private set; } + + /// + /// Gets the total number of bytes for the complete file, or -1 if unknown. + /// This will be -1 for failures that occur before receiving the GetObjectResponse + /// (e.g., authentication errors, non-existent objects), and will contain the actual + /// file size for failures that occur after receiving response headers (e.g., disk full). + /// + public long TotalBytes { get; private set; } } } diff --git a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs index 07a39c500430..224ffd70e7a3 100644 --- a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs +++ b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs @@ -1320,6 +1320,112 @@ public void DownloadProgressZeroLengthFileTest() progressValidator.AssertOnCompletion(); } + [TestMethod] + [TestCategory("S3")] + public void SimpleDownloadInitiatedEventTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleDownloadTest\InitiatedEvent"); + var eventValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName + ".download")); + // Note: DownloadInitiatedEventArgs does not have TotalBytes since we don't know the size until GetObjectResponse + } + }; + DownloadWithLifecycleEvents(fileName, 10 * MEG_SIZE, eventValidator, null, null); + eventValidator.AssertEventFired(); + } + + [TestMethod] + [TestCategory("S3")] + public void SimpleDownloadCompletedEventTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleDownloadTest\CompletedEvent"); + var eventValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.IsNotNull(args.Response); + Assert.AreEqual(args.TransferredBytes, args.TotalBytes); + Assert.AreEqual(10 * MEG_SIZE, args.TotalBytes); + Assert.IsTrue(!string.IsNullOrEmpty(args.Response.ETag)); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName + ".download")); + } + }; + DownloadWithLifecycleEvents(fileName, 10 * MEG_SIZE, null, eventValidator, null); + eventValidator.AssertEventFired(); + } + + [TestMethod] + [TestCategory("S3")] + public void SimpleDownloadFailedEventTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleDownloadTest\FailedEvent"); + var eventValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName + ".download")); + + // Non-existent key should always be early failure with unknown total bytes + Assert.AreEqual(-1, args.TotalBytes, "Non-existent key should result in TotalBytes = -1"); + Assert.AreEqual(0, args.TransferredBytes, "No bytes should be transferred for non-existent key"); + } + }; + + // Use non-existent key to force failure + var nonExistentKey = "non-existent-key-" + Guid.NewGuid().ToString(); + + try + { + DownloadWithLifecycleEventsAndKey(fileName, nonExistentKey, null, null, eventValidator); + Assert.Fail("Expected an exception to be thrown for non-existent key"); + } + catch (AmazonS3Exception) + { + // Expected exception - the failed event should have been fired + eventValidator.AssertEventFired(); + } + } + + [TestMethod] + [TestCategory("S3")] + public void SimpleDownloadCompleteLifecycleTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleDownloadTest\CompleteLifecycle"); + + var initiatedValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName + ".download")); + // Note: DownloadInitiatedEventArgs does not have TotalBytes since we don't know the size until GetObjectResponse + } + }; + + var completedValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.IsNotNull(args.Response); + Assert.AreEqual(args.TransferredBytes, args.TotalBytes); + Assert.AreEqual(8 * MEG_SIZE, args.TotalBytes); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName + ".download")); + } + }; + + DownloadWithLifecycleEvents(fileName, 8 * MEG_SIZE, initiatedValidator, completedValidator, null); + + initiatedValidator.AssertEventFired(); + completedValidator.AssertEventFired(); + } + void Download(string fileName, long size, TransferProgressValidator progressValidator) { var key = fileName; @@ -2145,6 +2251,85 @@ void UploadUnseekableStreamWithLifecycleEventsAndBucket(long size, string target transferUtility.Upload(request); } + + void DownloadWithLifecycleEvents(string fileName, long size, + TransferLifecycleEventValidator initiatedValidator, + TransferLifecycleEventValidator completedValidator, + TransferLifecycleEventValidator failedValidator) + { + // First upload the file so we have something to download + var key = fileName; + var originalFilePath = Path.Combine(BasePath, fileName); + UtilityMethods.GenerateFile(originalFilePath, size); + + Client.PutObject(new PutObjectRequest + { + BucketName = bucketName, + Key = key, + FilePath = originalFilePath + }); + + var downloadedFilePath = originalFilePath + ".download"; + + var transferUtility = new TransferUtility(Client); + var request = new TransferUtilityDownloadRequest + { + BucketName = bucketName, + FilePath = downloadedFilePath, + Key = key + }; + + if (initiatedValidator != null) + { + request.DownloadInitiatedEvent += initiatedValidator.OnEventFired; + } + + if (completedValidator != null) + { + request.DownloadCompletedEvent += completedValidator.OnEventFired; + } + + if (failedValidator != null) + { + request.DownloadFailedEvent += failedValidator.OnEventFired; + } + + transferUtility.Download(request); + } + + void DownloadWithLifecycleEventsAndKey(string fileName, string keyToDownload, + TransferLifecycleEventValidator initiatedValidator, + TransferLifecycleEventValidator completedValidator, + TransferLifecycleEventValidator failedValidator) + { + var downloadedFilePath = Path.Combine(BasePath, fileName + ".download"); + + var transferUtility = new TransferUtility(Client); + var request = new TransferUtilityDownloadRequest + { + BucketName = bucketName, + FilePath = downloadedFilePath, + Key = keyToDownload + }; + + if (initiatedValidator != null) + { + request.DownloadInitiatedEvent += initiatedValidator.OnEventFired; + } + + if (completedValidator != null) + { + request.DownloadCompletedEvent += completedValidator.OnEventFired; + } + + if (failedValidator != null) + { + request.DownloadFailedEvent += failedValidator.OnEventFired; + } + + transferUtility.Download(request); + } + private class UnseekableStream : MemoryStream { private readonly bool _setZeroLengthStream;