perf(s3): LIST-based existence + parallel tile upload (MAPCO-11322)

MergerLogic/Clients/S3Client.cs

@@ -5,6 +5,7 @@
 using MergerLogic.ImageProcessing;
 using MergerLogic.Utils;
 using Microsoft.Extensions.Logging;
+using System.Linq;
 using System.Reflection;
 
 namespace MergerLogic.Clients
@@ -81,7 +82,7 @@
 
         public override Tile? GetTile(int z, int x, int y)
         {
             string methodName = MethodBase.GetCurrentMethod().Name;
             this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y}");
             string keyPrefix = this._pathUtils.GetTilePath(this.path, z, x, y, TileFormat.Jpeg, true);
 
@@ -102,7 +103,7 @@
 
         public Tile? GetTile(string key)
         {
             string methodName = MethodBase.GetCurrentMethod().Name;
             this._logger.LogDebug($"[{methodName}] start key: {key}");
             byte[]? imageBytes = this.GetImageBytes(key);
             if (imageBytes == null)
@@ -117,7 +118,7 @@
 
         public override bool TileExists(int z, int x, int y)
         {
             string methodName = MethodBase.GetCurrentMethod().Name;
             this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y}");
             bool exists = this.GetTileKey(z, x, y) != null;
             this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y}");
@@ -126,7 +127,7 @@
 
         public void UpdateTile(Tile tile)
         {
             string methodName = MethodBase.GetCurrentMethod().Name;
             this._logger.LogDebug($"[{methodName}] start {tile.ToString()}");
             string key = this._pathUtils.GetTilePath(this.path, tile, true);
 
@@ -151,26 +152,15 @@
 
         private string? GetTileKey(int z, int x, int y)
         {
             string methodName = MethodBase.GetCurrentMethod().Name;
             string keyPrefix = this._pathUtils.GetTilePathWithoutExtension(this.path, z, x, y, true);
 
-            try
-            {
-                var getRequest = new GetObjectRequest { BucketName = this._bucket, Key = keyPrefix };
-                var getObjectTask = this._client.GetObjectAsync(getRequest);
-                string result = getObjectTask.Result.Key;
-                return result;
-            }
-            catch (AggregateException e)
-            {
-                if (IsKeyError(e))
-                {
-                    this._logger.LogDebug($"[{methodName}] error getting key: {e.Message}");
-                    return null;
-                }
-                // In case there are other errors such as connection to S3
-                throw e;
-            }
+            // A single prefixed LIST (MaxKeys=1) resolves existence and the real extension without
+            // downloading the object body the way GetObject did.
+            var listRequest = new ListObjectsV2Request { BucketName = this._bucket, Prefix = keyPrefix, MaxKeys = 1 };
+            this._logger.LogDebug($"[{methodName}] ListObjectsV2Async BucketName: {this._bucket}, Prefix: {keyPrefix}");
+            var listObjectsTask = this._client.ListObjectsV2Async(listRequest);
+            return listObjectsTask.Result.S3Objects.FirstOrDefault()?.Key;
         }
     }
 }

MergerLogic/DataTypes/S3.cs

@@ -198,10 +198,9 @@ public override long TileCount()
         protected override void InternalUpdateTiles(IEnumerable<Tile> targetTiles)
         {
             this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] start");
-            foreach (var tile in targetTiles)
-            {
-                this.Utils.UpdateTile(tile);
-            }
+            // Materialize first so the upstream (single-threaded) grid/origin projection runs before the
+            // uploads fan out; the S3 client and its PutObject calls are independent per tile.
+            Parallel.ForEach(targetTiles.ToList(), tile => this.Utils.UpdateTile(tile));
             this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] end");
         }
     }

MergerLogicUnitTests/DataTypes/S3Test.cs

@@ -473,8 +473,9 @@ public void UpdateTiles(bool isOneXOne, GridOrigin origin)
 
                 if (!isOneXOne || tile.Z != 7)
                 {
+                    // Uploads run in parallel after the (ordered) grid/origin projection, so UpdateTile is
+                    // intentionally outside the strict sequence; per-tile Times.Once is verified below.
                     this._s3UtilsMock
-                        .InSequence(seq)
                         .Setup(utils => utils.UpdateTile(It.IsAny<Tile>()));
                 }
             }

MergerLogicUnitTests/Utils/S3UtilsTest.cs

@@ -330,33 +330,26 @@ public void TileExists(bool exist)
                         .Setup(utils => utils.GetTilePathWithoutExtension("test", 0, 0, 0, true))
                         .Returns("key");
 
+            var listResponse = new ListObjectsV2Response();
             if (exist)
             {
-                this._amazonS3ClientMock
-                .InSequence(seq)
-                .Setup(s3 => s3.GetObjectAsync(It.Is<GetObjectRequest>(req =>
-                        req.BucketName == "bucket" && req.Key == "key"),
-                    It.IsAny<CancellationToken>()))
-                .ReturnsAsync(new GetObjectResponse() { Key = "key" });
+                listResponse.S3Objects.Add(new S3Object() { Key = "key" });
             }
-            else
-            {
-                this._amazonS3ClientMock
+            this._amazonS3ClientMock
                 .InSequence(seq)
-                .Setup(s3 => s3.GetObjectAsync(It.Is<GetObjectRequest>(req =>
-                        req.BucketName == "bucket" && req.Key == "key"),
+                .Setup(s3 => s3.ListObjectsV2Async(It.Is<ListObjectsV2Request>(req =>
+                        req.BucketName == "bucket" && req.Prefix == "key" && req.MaxKeys == 1),
                     It.IsAny<CancellationToken>()))
-                .ThrowsAsync(new AmazonS3Exception("", Amazon.Runtime.ErrorType.Unknown, "NoSuchKey", "", System.Net.HttpStatusCode.NoContent));
-            }
+                .ReturnsAsync(listResponse);
 
             var s3Utils = new S3Client(this._amazonS3ClientMock.Object, this._pathUtilsMock.Object,
                 this._geoUtilsMock.Object, this._loggerMock.Object, "STANDARD", "bucket", "test");
 
             Assert.AreEqual(exist, s3Utils.TileExists(0, 0, 0));
 
             this._pathUtilsMock.Verify(utils => utils.GetTilePathWithoutExtension("test", 0, 0, 0, true), Times.Once);
-            this._amazonS3ClientMock.Verify(s3 => s3.GetObjectAsync(It.Is<GetObjectRequest>(req =>
-                        req.BucketName == "bucket" && req.Key == "key"), It.IsAny<CancellationToken>()), Times.Once);
+            this._amazonS3ClientMock.Verify(s3 => s3.ListObjectsV2Async(It.Is<ListObjectsV2Request>(req =>
+                        req.BucketName == "bucket" && req.Prefix == "key" && req.MaxKeys == 1), It.IsAny<CancellationToken>()), Times.Once);
             this.VerifyAll();
         }