diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 00000000..d85b83f7 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,8 @@ + + + unix + windows + obj/$(ResgridHostOS)/ + $(DefaultItemExcludes);obj/** + + diff --git a/Tests/Resgrid.Tests/Services/DepartmentSettingsServiceTtsLanguageTests.cs b/Tests/Resgrid.Tests/Services/DepartmentSettingsServiceTtsLanguageTests.cs index 6c283d1b..6055e2e9 100644 --- a/Tests/Resgrid.Tests/Services/DepartmentSettingsServiceTtsLanguageTests.cs +++ b/Tests/Resgrid.Tests/Services/DepartmentSettingsServiceTtsLanguageTests.cs @@ -12,6 +12,7 @@ namespace Resgrid.Tests.Services { [TestFixture] + [NonParallelizable] public class DepartmentSettingsServiceTtsLanguageTests { private Mock _departmentSettingsRepository; diff --git a/Tests/Resgrid.Tests/Web/Tts/S3StorageServiceTests.cs b/Tests/Resgrid.Tests/Web/Tts/S3StorageServiceTests.cs index 3aa45a2f..2b3d5685 100644 --- a/Tests/Resgrid.Tests/Web/Tts/S3StorageServiceTests.cs +++ b/Tests/Resgrid.Tests/Web/Tts/S3StorageServiceTests.cs @@ -55,6 +55,39 @@ public async Task upload_async_should_buffer_non_seekable_stream_for_retries() s3Client.Verify(x => x.PutObjectAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); } + [Test] + public async Task exists_async_should_fall_back_to_presigned_head_when_metadata_response_is_malformed() + { + var s3Client = new Mock(MockBehavior.Strict); + s3Client + .Setup(x => x.GetObjectMetadataAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new FormatException("bad metadata expiration header")); + s3Client + .Setup(x => x.GetPreSignedURL(It.IsAny())) + .Returns(request => + { + request.BucketName.Should().Be("tts-bucket"); + request.Key.Should().Be("tts/audio.wav"); + request.Verb.Should().Be(HttpVerb.HEAD); + return "https://upload.example.com/tts/audio.wav?signature=head"; + }); + + var handler = new RecordingHttpMessageHandler((request, _) => + { + request.Method.Should().Be(HttpMethod.Head); + request.RequestUri.Should().Be(new Uri("https://upload.example.com/tts/audio.wav?signature=head")); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); + }); + var service = CreateService(s3Client.Object, handler); + + var exists = await service.ExistsAsync("tts/audio.wav", CancellationToken.None); + + exists.Should().BeTrue(); + handler.Requests.Should().HaveCount(1); + s3Client.Verify(x => x.GetObjectMetadataAsync(It.Is(request => request.BucketName == "tts-bucket" && request.Key == "tts/audio.wav"), It.IsAny()), Times.Once); + s3Client.Verify(x => x.GetPreSignedURL(It.IsAny()), Times.Once); + } + [Test] public async Task upload_async_should_treat_format_exception_as_success_when_object_exists_after_upload() { @@ -91,15 +124,39 @@ public async Task upload_async_should_fall_back_to_presigned_put_when_metadata_r .ThrowsAsync(new FormatException("bad metadata expiration header")); s3Client .Setup(x => x.GetPreSignedURL(It.IsAny())) - .Returns("https://upload.example.com/tts/audio.wav?signature=metadata-format"); + .Returns(request => + { + request.BucketName.Should().Be("tts-bucket"); + request.Key.Should().Be("tts/audio.wav"); + + return request.Verb switch + { + HttpVerb.HEAD => "https://upload.example.com/tts/audio.wav?signature=metadata-head", + HttpVerb.PUT => "https://upload.example.com/tts/audio.wav?signature=metadata-put", + _ => throw new AssertionException($"Unexpected presigned verb {request.Verb}") + }; + }); + var headRequests = 0; + var putRequests = 0; var handler = new RecordingHttpMessageHandler(async (request, cancellationToken) => { + request.RequestUri.Should().NotBeNull(); + + if (request.Method == HttpMethod.Head) + { + headRequests++; + request.RequestUri.Should().Be(new Uri("https://upload.example.com/tts/audio.wav?signature=metadata-head")); + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + putRequests++; + request.Method.Should().Be(HttpMethod.Put); + request.RequestUri.Should().Be(new Uri("https://upload.example.com/tts/audio.wav?signature=metadata-put")); + var body = await request.Content!.ReadAsByteArrayAsync(cancellationToken); body.Should().Equal(2, 4, 6, 8); - request.Method.Should().Be(HttpMethod.Put); - request.RequestUri.Should().Be(new Uri("https://upload.example.com/tts/audio.wav?signature=metadata-format")); - request.Content!.Headers.ContentType!.MediaType.Should().Be("audio/wav"); + request.Content.Headers.ContentType!.MediaType.Should().Be("audio/wav"); return new HttpResponseMessage(HttpStatusCode.OK); }); @@ -109,9 +166,12 @@ public async Task upload_async_should_fall_back_to_presigned_put_when_metadata_r await service.UploadAsync("tts/audio.wav", content, "audio/wav", CancellationToken.None); - handler.Requests.Should().HaveCount(1); + headRequests.Should().Be(1); + putRequests.Should().Be(1); + handler.Requests.Should().HaveCount(2); s3Client.Verify(x => x.GetObjectMetadataAsync(It.IsAny(), It.IsAny()), Times.Once); - s3Client.Verify(x => x.GetPreSignedURL(It.IsAny()), Times.Once); + s3Client.Verify(x => x.GetPreSignedURL(It.Is(request => request.Verb == HttpVerb.HEAD)), Times.Once); + s3Client.Verify(x => x.GetPreSignedURL(It.Is(request => request.Verb == HttpVerb.PUT)), Times.Once); } [Test] diff --git a/Tests/Resgrid.Tests/Web/Tts/TtsConfigurationRegistrationTests.cs b/Tests/Resgrid.Tests/Web/Tts/TtsConfigurationRegistrationTests.cs index b60395ef..af731cd4 100644 --- a/Tests/Resgrid.Tests/Web/Tts/TtsConfigurationRegistrationTests.cs +++ b/Tests/Resgrid.Tests/Web/Tts/TtsConfigurationRegistrationTests.cs @@ -8,6 +8,7 @@ namespace Resgrid.Tests.Web.Tts { [TestFixture] + [NonParallelizable] public class TtsConfigurationRegistrationTests { private string _originalS3AccessKey; diff --git a/Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs b/Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs index 225388ba..686e4e1d 100644 --- a/Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs +++ b/Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs @@ -17,6 +17,7 @@ namespace Resgrid.Tests.Workers.Console.Tasks { [TestFixture] + [NonParallelizable] public class TtsStaticPromptRefreshTaskTests { private static readonly FieldInfo WorkerBootstrapperContainerField = typeof(Resgrid.Workers.Framework.Bootstrapper) diff --git a/Web/Resgrid.Web.Tts/Services/S3StorageService.cs b/Web/Resgrid.Web.Tts/Services/S3StorageService.cs index b5c9424c..bc425243 100644 --- a/Web/Resgrid.Web.Tts/Services/S3StorageService.cs +++ b/Web/Resgrid.Web.Tts/Services/S3StorageService.cs @@ -52,6 +52,15 @@ await ExecuteWithRetryAsync( { return false; } + catch (FormatException ex) + { + _logger.LogWarning( + ex, + "The S3 client could not parse the metadata response for {ObjectKey}. Falling back to a presigned HEAD request.", + objectKey); + + return await ExistsWithPresignedHeadAsync(objectKey, cancellationToken); + } } public async Task UploadAsync(string objectKey, Stream content, string contentType, CancellationToken cancellationToken) @@ -230,6 +239,69 @@ private async Task UploadWithPresignedUrlAsync(string objectKey, Stream content, throw new InvalidOperationException($"Presigned PUT upload retry loop terminated unexpectedly for {objectKey}."); } + private string CreatePresignedHeadUrl(string objectKey) + { + return _s3Client.GetPreSignedURL(new GetPreSignedUrlRequest + { + BucketName = _options.Bucket, + Key = objectKey, + Verb = HttpVerb.HEAD, + Expires = DateTime.UtcNow.AddMinutes(PresignedPutUrlExpiryMinutes) + }); + } + + private async Task ExistsWithPresignedHeadAsync(string objectKey, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(nameof(S3StorageService)); + + for (var attempt = 1; attempt <= MaxRetryAttempts; attempt++) + { + using var request = new HttpRequestMessage(HttpMethod.Head, CreatePresignedHeadUrl(objectKey)); + + try + { + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + + if (response.StatusCode == HttpStatusCode.NotFound) + { + return false; + } + + if (response.IsSuccessStatusCode) + { + return true; + } + + var exception = new HttpRequestException( + $"Presigned HEAD existence check for {objectKey} failed with status code {(int)response.StatusCode}.", + null, + response.StatusCode); + + if (attempt < MaxRetryAttempts && IsTransientStatusCode(response.StatusCode)) + { + await DelayRetryAsync($"checking existence of {objectKey} via presigned HEAD", attempt, exception, cancellationToken); + continue; + } + + response.EnsureSuccessStatusCode(); + } + catch (HttpRequestException ex) when (attempt < MaxRetryAttempts && (!ex.StatusCode.HasValue || IsTransientStatusCode(ex.StatusCode.Value))) + { + await DelayRetryAsync($"checking existence of {objectKey} via presigned HEAD", attempt, ex, cancellationToken); + } + catch (IOException ex) when (attempt < MaxRetryAttempts) + { + await DelayRetryAsync($"checking existence of {objectKey} via presigned HEAD", attempt, ex, cancellationToken); + } + catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested && attempt < MaxRetryAttempts) + { + await DelayRetryAsync($"checking existence of {objectKey} via presigned HEAD", attempt, ex, cancellationToken); + } + } + + throw new InvalidOperationException($"Presigned HEAD existence check retry loop terminated unexpectedly for {objectKey}."); + } + private string CreatePresignedPutUrl(string objectKey, string contentType) { return _s3Client.GetPreSignedURL(new GetPreSignedUrlRequest