RE1-T115 S3 fix

Directory.Build.props

@@ -0,0 +1,8 @@
+<Project>
+  <PropertyGroup>
+    <ResgridHostOS>unix</ResgridHostOS>
+    <ResgridHostOS Condition="$([MSBuild]::IsOSPlatform('Windows'))">windows</ResgridHostOS>
+    <BaseIntermediateOutputPath>obj/$(ResgridHostOS)/</BaseIntermediateOutputPath>
+    <DefaultItemExcludes>$(DefaultItemExcludes);obj/**</DefaultItemExcludes>
+  </PropertyGroup>
+</Project>

Tests/Resgrid.Tests/Services/DepartmentSettingsServiceTtsLanguageTests.cs

@@ -12,6 +12,7 @@
 namespace Resgrid.Tests.Services
 {
 	[TestFixture]
+	[NonParallelizable]
 	public class DepartmentSettingsServiceTtsLanguageTests
 	{
 		private Mock<IDepartmentSettingsRepository> _departmentSettingsRepository;

Tests/Resgrid.Tests/Web/Tts/S3StorageServiceTests.cs

@@ -55,6 +55,39 @@ public async Task upload_async_should_buffer_non_seekable_stream_for_retries()
 			s3Client.Verify(x => x.PutObjectAsync(It.IsAny<PutObjectRequest>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
 		}
 
+		[Test]
+		public async Task exists_async_should_fall_back_to_presigned_head_when_metadata_response_is_malformed()
+		{
+			var s3Client = new Mock<IAmazonS3>(MockBehavior.Strict);
+			s3Client
+				.Setup(x => x.GetObjectMetadataAsync(It.IsAny<GetObjectMetadataRequest>(), It.IsAny<CancellationToken>()))
+				.ThrowsAsync(new FormatException("bad metadata expiration header"));
+			s3Client
+				.Setup(x => x.GetPreSignedURL(It.IsAny<GetPreSignedUrlRequest>()))
+				.Returns<GetPreSignedUrlRequest>(request =>
+				{
+					request.BucketName.Should().Be("tts-bucket");
+					request.Key.Should().Be("tts/audio.wav");
+					request.Verb.Should().Be(HttpVerb.HEAD);
+					return "https://upload.example.com/tts/audio.wav?signature=head";
+				});
+
+			var handler = new RecordingHttpMessageHandler((request, _) =>
+			{
+				request.Method.Should().Be(HttpMethod.Head);
+				request.RequestUri.Should().Be(new Uri("https://upload.example.com/tts/audio.wav?signature=head"));
+				return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK));
+			});
+			var service = CreateService(s3Client.Object, handler);
+
+			var exists = await service.ExistsAsync("tts/audio.wav", CancellationToken.None);
+
+			exists.Should().BeTrue();
+			handler.Requests.Should().HaveCount(1);
+			s3Client.Verify(x => x.GetObjectMetadataAsync(It.Is<GetObjectMetadataRequest>(request => request.BucketName == "tts-bucket" && request.Key == "tts/audio.wav"), It.IsAny<CancellationToken>()), Times.Once);
+			s3Client.Verify(x => x.GetPreSignedURL(It.IsAny<GetPreSignedUrlRequest>()), Times.Once);
+		}
+
 		[Test]
 		public async Task upload_async_should_treat_format_exception_as_success_when_object_exists_after_upload()
 		{
@@ -91,15 +124,39 @@ public async Task upload_async_should_fall_back_to_presigned_put_when_metadata_r
 				.ThrowsAsync(new FormatException("bad metadata expiration header"));
 			s3Client
 				.Setup(x => x.GetPreSignedURL(It.IsAny<GetPreSignedUrlRequest>()))
-				.Returns("https://upload.example.com/tts/audio.wav?signature=metadata-format");
+				.Returns<GetPreSignedUrlRequest>(request =>
+				{
+					request.BucketName.Should().Be("tts-bucket");
+					request.Key.Should().Be("tts/audio.wav");
+
+					return request.Verb switch
+					{
+						HttpVerb.HEAD => "https://upload.example.com/tts/audio.wav?signature=metadata-head",
+						HttpVerb.PUT => "https://upload.example.com/tts/audio.wav?signature=metadata-put",
+						_ => throw new AssertionException($"Unexpected presigned verb {request.Verb}")
+					};
+				});
 
+			var headRequests = 0;
+			var putRequests = 0;
 			var handler = new RecordingHttpMessageHandler(async (request, cancellationToken) =>
 			{
+				request.RequestUri.Should().NotBeNull();
+
+				if (request.Method == HttpMethod.Head)
+				{
+					headRequests++;
+					request.RequestUri.Should().Be(new Uri("https://upload.example.com/tts/audio.wav?signature=metadata-head"));
+					return new HttpResponseMessage(HttpStatusCode.NotFound);
+				}
+
+				putRequests++;
+				request.Method.Should().Be(HttpMethod.Put);
+				request.RequestUri.Should().Be(new Uri("https://upload.example.com/tts/audio.wav?signature=metadata-put"));
+
 				var body = await request.Content!.ReadAsByteArrayAsync(cancellationToken);
 				body.Should().Equal(2, 4, 6, 8);
-				request.Method.Should().Be(HttpMethod.Put);
-				request.RequestUri.Should().Be(new Uri("https://upload.example.com/tts/audio.wav?signature=metadata-format"));
-				request.Content!.Headers.ContentType!.MediaType.Should().Be("audio/wav");
+				request.Content.Headers.ContentType!.MediaType.Should().Be("audio/wav");
 
 				return new HttpResponseMessage(HttpStatusCode.OK);
 			});
@@ -109,9 +166,12 @@ public async Task upload_async_should_fall_back_to_presigned_put_when_metadata_r
 
 			await service.UploadAsync("tts/audio.wav", content, "audio/wav", CancellationToken.None);
 
-			handler.Requests.Should().HaveCount(1);
+			headRequests.Should().Be(1);
+			putRequests.Should().Be(1);
+			handler.Requests.Should().HaveCount(2);
 			s3Client.Verify(x => x.GetObjectMetadataAsync(It.IsAny<GetObjectMetadataRequest>(), It.IsAny<CancellationToken>()), Times.Once);
-			s3Client.Verify(x => x.GetPreSignedURL(It.IsAny<GetPreSignedUrlRequest>()), Times.Once);
+			s3Client.Verify(x => x.GetPreSignedURL(It.Is<GetPreSignedUrlRequest>(request => request.Verb == HttpVerb.HEAD)), Times.Once);
+			s3Client.Verify(x => x.GetPreSignedURL(It.Is<GetPreSignedUrlRequest>(request => request.Verb == HttpVerb.PUT)), Times.Once);
 		}
 
 		[Test]

Tests/Resgrid.Tests/Web/Tts/TtsConfigurationRegistrationTests.cs

@@ -8,6 +8,7 @@
 namespace Resgrid.Tests.Web.Tts
 {
 	[TestFixture]
+	[NonParallelizable]
 	public class TtsConfigurationRegistrationTests
 	{
 		private string _originalS3AccessKey;

Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs

@@ -17,6 +17,7 @@
 namespace Resgrid.Tests.Workers.Console.Tasks
 {
 	[TestFixture]
+	[NonParallelizable]
 	public class TtsStaticPromptRefreshTaskTests
 	{
 		private static readonly FieldInfo WorkerBootstrapperContainerField = typeof(Resgrid.Workers.Framework.Bootstrapper)

Web/Resgrid.Web.Tts/Services/S3StorageService.cs

@@ -52,6 +52,15 @@ await ExecuteWithRetryAsync(
 			{
 				return false;
 			}
+			catch (FormatException ex)
+			{
+				_logger.LogWarning(
+					ex,
+					"The S3 client could not parse the metadata response for {ObjectKey}. Falling back to a presigned HEAD request.",
+					objectKey);
+
+				return await ExistsWithPresignedHeadAsync(objectKey, cancellationToken);
+			}
 		}
 
 		public async Task UploadAsync(string objectKey, Stream content, string contentType, CancellationToken cancellationToken)
@@ -230,6 +239,69 @@ private async Task UploadWithPresignedUrlAsync(string objectKey, Stream content,
 			throw new InvalidOperationException($"Presigned PUT upload retry loop terminated unexpectedly for {objectKey}.");
 		}
 
+		private string CreatePresignedHeadUrl(string objectKey)
+		{
+			return _s3Client.GetPreSignedURL(new GetPreSignedUrlRequest
+			{
+				BucketName = _options.Bucket,
+				Key = objectKey,
+				Verb = HttpVerb.HEAD,
+				Expires = DateTime.UtcNow.AddMinutes(PresignedPutUrlExpiryMinutes)
+			});
+		}
+
+		private async Task<bool> ExistsWithPresignedHeadAsync(string objectKey, CancellationToken cancellationToken)
+		{
+			var client = _httpClientFactory.CreateClient(nameof(S3StorageService));
+
+			for (var attempt = 1; attempt <= MaxRetryAttempts; attempt++)
+			{
+				using var request = new HttpRequestMessage(HttpMethod.Head, CreatePresignedHeadUrl(objectKey));
+
+				try
+				{
+					using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+
+					if (response.StatusCode == HttpStatusCode.NotFound)
+					{
+						return false;
+					}
+
+					if (response.IsSuccessStatusCode)
+					{
+						return true;
+					}
+
+					var exception = new HttpRequestException(
+						$"Presigned HEAD existence check for {objectKey} failed with status code {(int)response.StatusCode}.",
+						null,
+						response.StatusCode);
+
+					if (attempt < MaxRetryAttempts && IsTransientStatusCode(response.StatusCode))
+					{
+						await DelayRetryAsync($"checking existence of {objectKey} via presigned HEAD", attempt, exception, cancellationToken);
+						continue;
+					}
+
+					response.EnsureSuccessStatusCode();
+				}
+				catch (HttpRequestException ex) when (attempt < MaxRetryAttempts && (!ex.StatusCode.HasValue || IsTransientStatusCode(ex.StatusCode.Value)))
+				{
+					await DelayRetryAsync($"checking existence of {objectKey} via presigned HEAD", attempt, ex, cancellationToken);
+				}
+				catch (IOException ex) when (attempt < MaxRetryAttempts)
+				{
+					await DelayRetryAsync($"checking existence of {objectKey} via presigned HEAD", attempt, ex, cancellationToken);
+				}
+				catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested && attempt < MaxRetryAttempts)
+				{
+					await DelayRetryAsync($"checking existence of {objectKey} via presigned HEAD", attempt, ex, cancellationToken);
+				}
+			}
+
+			throw new InvalidOperationException($"Presigned HEAD existence check retry loop terminated unexpectedly for {objectKey}.");
+		}
+
 		private string CreatePresignedPutUrl(string objectKey, string contentType)
 		{
 			return _s3Client.GetPreSignedURL(new GetPreSignedUrlRequest