feat: support mixed output

MergerCli/IProcess.cs

@@ -1,12 +1,11 @@
 using MergerCli.Utils;
 using MergerLogic.DataTypes;
-using MergerLogic.ImageProcessing;
 
 namespace MergerCli
 {
     internal interface IProcess
     {
-        void Start(TileFormat targetFormat, IData baseData, IData newData, BatchStatusManager batchStatusManager);
+        void Start(IData baseData, IData newData, BatchStatusManager batchStatusManager);
 
         void Validate(IData baseData, IData newData, string? incompleteBatchIdentifier = null);
     }

MergerCli/ISourceParser.cs

@@ -1,10 +1,9 @@
 using MergerLogic.DataTypes;
-using MergerLogic.ImageProcessing;
 
 namespace MergerCli
 {
     internal interface ISourceParser
     {
-        List<IData> ParseSources(string[] args, int batchSize, out TileFormat format);
+        List<IData> ParseSources(string[] args, int batchSize);
     }
 }

MergerCli/Process.cs

@@ -6,6 +6,7 @@
 using Microsoft.Extensions.Logging;
 using System.Collections.Concurrent;
 using System.Reflection;
+using static MergerLogic.ImageProcessing.TileFormatStrategy;
 
 namespace MergerCli
 {
@@ -15,16 +16,21 @@ internal class Process : IProcess
         private readonly IConfigurationManager _configManager;
         private readonly ITileMerger _tileMerger;
         private readonly ILogger _logger;
+        private TileFormatStrategy _tileFormatStrategy;
         static readonly object _locker = new object();
 
         public Process(IConfigurationManager configuration, ITileMerger tileMerger, ILogger<Process> logger)
         {
             this._configManager = configuration;
             this._tileMerger = tileMerger;
             this._logger = logger;
+
+            FormatStrategy outputFormatStrategy = this._configManager.GetConfiguration<FormatStrategy>("TILE", "outputFormatStrategy");
+            TileFormat outputFormat = this._configManager.GetConfiguration<TileFormat>("TILE", "outputFormat");
+            this._tileFormatStrategy = new TileFormatStrategy(outputFormat, outputFormatStrategy);
         }
 
-        public void Start(TileFormat targetFormat, IData baseData, IData newData, BatchStatusManager batchStatusManager)
+        public void Start(IData baseData, IData newData, BatchStatusManager batchStatusManager)
         {
             long totalTileCount = newData.TileCount();
             batchStatusManager.InitializeLayer(newData.Path);
@@ -36,6 +42,10 @@ public void Start(TileFormat targetFormat, IData baseData, IData newData, BatchS
             {
                 resumeMode = true;
                 this._logger.LogDebug($"[{MethodBase.GetCurrentMethod().Name}] Resume mode activated, resume batchId: {resumeBatchIdentifier}");
+
+                // Set strategy from status manager
+                this._tileFormatStrategy = new TileFormatStrategy(batchStatusManager.Format, batchStatusManager.Strategy);
+
                 // fix resume progress bug for gpkg, fs and web, fixing it for s3 requires storing additional data.
                 if (newData.Type != DataType.S3)
                 {
@@ -47,13 +57,14 @@ public void Start(TileFormat targetFormat, IData baseData, IData newData, BatchS
             this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] Total amount of tiles to merge: {totalTileCount - tileProgressCount}");
             var uploadOnly = this._configManager.GetConfiguration<bool>("GENERAL", "uploadOnly");
 
-            bool shouldUpscale = !(uploadOnly || baseData.IsNew);
-            _getTileByCoord = uploadOnly || baseData.IsNew ?
+            uploadOnly = uploadOnly || baseData.IsNew;
+            bool shouldUpscale = !uploadOnly;
+            _getTileByCoord = uploadOnly ?
                 (_) => null
                 :
                 (targetCoords) => baseData.GetCorrespondingTile(targetCoords, shouldUpscale);
 
-            ParallelRun(targetFormat, baseData, newData, batchStatusManager,
+            ParallelRun(baseData, newData, batchStatusManager,
                 tileProgressCount, totalTileCount, resumeBatchIdentifier, resumeMode, pollForBatch);
 
             batchStatusManager.CompleteLayer(newData.Path);
@@ -97,7 +108,7 @@ public void Start(TileFormat targetFormat, IData baseData, IData newData, BatchS
             }
         }
 
-        private void ProcessBatch(TileFormat targetFormat, IData baseData, List<Tile> newTiles, ref long tileProgressCount, long totalTileCount,ref bool pollForBatch)
+        private void ProcessBatch(IData baseData, List<Tile> newTiles, ref long tileProgressCount, long totalTileCount,ref bool pollForBatch)
         {
             ConcurrentBag<Tile> tiles = new ConcurrentBag<Tile>();
 
@@ -116,12 +127,11 @@ private void ProcessBatch(TileFormat targetFormat, IData baseData, List<Tile> ne
                     () => _getTileByCoord(targetCoords), () => newTile
                 };
 
-                byte[]? image = this._tileMerger.MergeTiles(correspondingTileBuilders, targetCoords, targetFormat);
+                Tile? tile = this._tileMerger.MergeTiles(correspondingTileBuilders, targetCoords, this._tileFormatStrategy);
 
-                if (image != null)
+                if (tile != null)
                 {
-                    newTile = new Tile(newTile.Z, newTile.X, newTile.Y, image);
-                    tiles.Add(newTile);
+                    tiles.Add(tile);
                 }
             }
 
@@ -131,7 +141,7 @@ private void ProcessBatch(TileFormat targetFormat, IData baseData, List<Tile> ne
             this._logger.LogInformation($"[{MethodBase.GetCurrentMethod().Name}] Tile Count: {tileProgressCount} / {totalTileCount}");
         }
 
-        private void ParallelRun(TileFormat targetFormat, IData baseData, IData newData,
+        private void ParallelRun(IData baseData, IData newData,
             BatchStatusManager batchStatusManager, long tileProgressCount, long totalTileCount, string? resumeBatchIdentifier, bool resumeMode,bool pollForBatch)
         {
             var numOfThreads = this._configManager.GetConfiguration<int>("GENERAL", "parallel", "numOfThreads");
@@ -140,7 +150,7 @@ private void ParallelRun(TileFormat targetFormat, IData baseData, IData newData,
                 while (tileProgressCount != totalTileCount && pollForBatch)
                 {
                     var batchResult = ManageBatchIdentifier(batchStatusManager, newData, resumeBatchIdentifier, totalTileCount, ref resumeMode);
-                    ProcessBatch(targetFormat, baseData, batchResult.newTiles, ref tileProgressCount,
+                    ProcessBatch(baseData, batchResult.newTiles, ref tileProgressCount,
                         totalTileCount, ref pollForBatch);
                     batchStatusManager.CompleteBatch(newData.Path, batchResult.currentBatchIdentifier, tileProgressCount);
                 }

MergerCli/Program.cs

@@ -7,6 +7,7 @@
 using Microsoft.Extensions.Logging;
 using System.Diagnostics;
 using System.Runtime.Loader;
+using static MergerLogic.ImageProcessing.TileFormatStrategy;
 
 namespace MergerCli
 {
@@ -38,6 +39,9 @@ private static void Main(string[] args)
             var config = container.GetRequiredService<IConfigurationManager>();
             var pathUtils = container.GetRequiredService<IPathUtils>();
             string outputPath = pathUtils.RemoveTrailingSlash(config.GetConfiguration("GENERAL", "resumeOutputFolder"));
+
+            FormatStrategy outputFormatStrategy = config.GetConfiguration<FormatStrategy>("TILE", "outputFormatStrategy");
+            TileFormat outputFormat = config.GetConfiguration<TileFormat>("TILE", "outputFormat");
             _resumeFilePath = $"{outputPath}/status.json";
 
             // If should resume, load status manager file and update states, else create from arguments
@@ -48,17 +52,16 @@ private static void Main(string[] args)
             }
             else
             {
-                _batchStatusManager = new BatchStatusManager(args);
+                _batchStatusManager = new BatchStatusManager(args, outputFormat, outputFormatStrategy);
             }
             PrepareStatusManger();
 
             int batchSize = int.Parse(args[1]);
-            TileFormat format;
             List<IData> sources;
             try
             {
                 var parser = container.GetRequiredService<ISourceParser>();
-                sources = parser.ParseSources(args, batchSize, out format);
+                sources = parser.ParseSources(args, batchSize);
             }
             catch (Exception ex)
             {
@@ -101,7 +104,7 @@ private static void Main(string[] args)
                         continue;
                     }
 
-                    process.Start(format, baseData, sources[i], _batchStatusManager);
+                    process.Start(baseData, sources[i], _batchStatusManager);
                     baseData.IsNew = false;
                     stopWatch.Stop();
 
@@ -161,7 +164,7 @@ web sources (cant be base source):
                         gpkg <path>  [bbox - in format 'minX,minY,maxX,maxY' - required base] [--1x1] [--UL / --LL] 
                     **** please note all layers must be 2X1 EPSG:4326 layers ****
 
-                merge sources: {programName} <batch_size> <target tiles format: png/jpeg> <base source> <additional source> [<another source>...]
+                merge sources: {programName} <batch_size> <base source> <additional source> [<another source>...]
                 Examples:
                 {programName} 1000 gpkg area1.gpkg gpkg area2.gpkg
                 {programName} 1000 s3 /path1/on/s3 s3 /path2/on/s3

MergerCli/SourceParser.cs

@@ -26,16 +26,11 @@ private void LogDataErrorAndExit(bool isBase, Exception e) {
             Environment.Exit(1);
         }
 
-        public List<IData> ParseSources(string[] args, int batchSize, out TileFormat format)
+        public List<IData> ParseSources(string[] args, int batchSize)
         {
             List<IData> sources = new List<IData>();
-            if (!TileFormat.TryParse(args[2], true, out format))
-            {
-                this._logger.LogError($"invalid target tile format: {args[2]}");
-                Environment.Exit(1);
-            }
 
-            int idx = 3;
+            int idx = 2;
             bool isBase = true;
             while (idx < args.Length)
             {

MergerCli/appsettings.json

@@ -7,6 +7,10 @@
       "numOfThreads": 1
     }
   },
+  "TILE": {
+    "outputFormatStrategy": "fixed",
+    "outputFormat": "jpeg"
+  },
   "GPKG": {
     "vacuum": false
   },

MergerCli/utils/BatchStatusManager.cs

@@ -1,6 +1,9 @@
-using System.Collections.Concurrent;
-using System.Text.Json;
+using MergerLogic.ImageProcessing;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using System.Collections.Concurrent;
 using System.Text.Json.Serialization;
+using static MergerLogic.ImageProcessing.TileFormatStrategy;
 namespace MergerCli.Utils
 {
     internal class BatchStatusManager
@@ -38,16 +41,30 @@ public BaseLayerStatus()
         [JsonInclude]
         public Dictionary<string, LayerStatus> States { get; private set; }
 
+        [JsonInclude]
+        public FormatStrategy Strategy { get; private set; }
+
+        [JsonInclude]
+        public TileFormat Format { get; private set; }
+
         [JsonInclude]
         public string[] Command { get; private set; }
 
         static readonly object _locker = new object();
 
-        public BatchStatusManager(string[] command)
+        [System.Text.Json.Serialization.JsonIgnore]
+        private JsonSerializerSettings _jsonSerializerSettings;
+
+        public BatchStatusManager(string[] command, TileFormat format, FormatStrategy strategy = FormatStrategy.Fixed)
         {
             this.BaseLayer = new BaseLayerStatus();
             this.States = new Dictionary<string, LayerStatus>();
+            this.Strategy = strategy;
+            this.Format = format;
             this.Command = command;
+
+            this._jsonSerializerSettings = new JsonSerializerSettings();
+            this._jsonSerializerSettings.Converters.Add(new StringEnumConverter());
         }
 
         public void SetCurrentBatch(string layer, string? batchIdentifier)
@@ -177,12 +194,15 @@ public void ResetBatchStatus()
 
         public override string ToString()
         {
-            return JsonSerializer.Serialize(this);
+            return JsonConvert.SerializeObject(this, this._jsonSerializerSettings);
         }
 
         public static BatchStatusManager FromJson(string json)
         {
-            BatchStatusManager? batchStatusManager = JsonSerializer.Deserialize<BatchStatusManager>(json);
+            JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings();
+            jsonSerializerSettings.Converters.Add(new StringEnumConverter());
+
+            BatchStatusManager? batchStatusManager = JsonConvert.DeserializeObject<BatchStatusManager>(json, jsonSerializerSettings)!;
             if (batchStatusManager == null)
             {
                 throw new Exception("invalid batch status manager json");

MergerLogic/Batching/Tile.cs

@@ -1,3 +1,4 @@
+using ImageMagick;
 using MergerLogic.DataTypes;
 using MergerLogic.ImageProcessing;
 using System.ComponentModel.DataAnnotations;
@@ -6,6 +7,10 @@ namespace MergerLogic.Batching
 {
     public delegate Tile? CorrespondingTileBuilder();
 
+    // TODO: add to README that the Merger assumes EPSG:4326
+    /// <summary>
+    /// Class <c>Tile</c> represents a position of an image in the EPSG:4326 geographic tiling scheme
+    /// </summary>
     public class Tile
     {
         public int Z
@@ -48,6 +53,15 @@ public Tile(Coord cords, byte[] data)
             this._data = data;
         }
 
+        public Tile(Coord cords, IMagickImage<byte> image)
+        {
+            this.Z = cords.Z;
+            this.X = cords.X;
+            this.Y = cords.Y;
+            this.Format = ImageFormatter.GetTileFormat(image) ?? throw new ValidationException($"Cannot create tile {this}, data is in invalid format");
+            this._data = image.ToByteArray();
+        }
+
         public bool HasCoords(int z, int x, int y)
         {
             return z == this.Z && x == this.X && y == this.Y;
@@ -72,12 +86,12 @@ public virtual byte[] GetImageBytes()
             return this._data;
         }
 
+        public int Size() {
+            return this._data.Length;
+        }
+
         public void ConvertToFormat(TileFormat format)
         {
-            if (this.Format == format) {
-                return;
-            }
-
             this._data = ImageFormatter.ConvertToFormat(this._data, format);
             this.Format = format;
         }

MergerLogic/ImageProcessing/ITileMerger.cs

@@ -5,6 +5,6 @@ namespace MergerLogic.ImageProcessing
 {
     public interface ITileMerger
     {
-        byte[]? MergeTiles(List<CorrespondingTileBuilder> tiles, Coord targetCoords, TileFormat format);
+        Tile? MergeTiles(List<CorrespondingTileBuilder> tiles, Coord targetCoords, TileFormatStrategy strategy);
     }
 }

MergerLogic/ImageProcessing/ImageFormatter.cs

@@ -9,11 +9,35 @@ public enum TileFormat
         [EnumMember(Value = "jpeg")] Jpeg,
     }
 
+    public class TileFormatStrategy {
+        public enum FormatStrategy {
+            [EnumMember(Value = "fixed")] Fixed,
+            [EnumMember(Value = "mixed")] Mixed,
+        }
+
+        private FormatStrategy _strategy;
+        private TileFormat _format;
+
+        public TileFormatStrategy(TileFormat format, FormatStrategy strategy = FormatStrategy.Fixed)
+        {
+            this._strategy = strategy;
+            this._format = format;
+        }
+
+        public TileFormat ApplyStrategy(TileFormat format) {
+            if (this._strategy == FormatStrategy.Fixed) {
+                return this._format;
+            }
+
+            return format;
+        }
+    }
+
     public class ImageFormatter
     {
         public static byte[] ConvertToFormat(byte[] tile, TileFormat format)
         {
-            var currentFormat = GetTileFormat(tile);
+            TileFormat? currentFormat = GetTileFormat(tile);
             if (currentFormat != format)
             {
                 using (var image = new MagickImage(tile))
@@ -77,6 +101,22 @@ public static void ConvertToFormat(IMagickImage image, TileFormat format)
             return null;
         }
 
+        public static TileFormat? GetTileFormat(IMagickImage<byte> image) {
+            if(image.IsOpaque) {
+                image.Format = MagickFormat.Jpeg;
+            }
+
+            if (image.Format == MagickFormat.Jpg || image.Format == MagickFormat.Jpeg) {
+                return TileFormat.Jpeg;
+            }
+
+            if (image.Format == MagickFormat.Png) {
+                return TileFormat.Png;
+            }
+
+            return null;
+        }
+
         public static void RemoveImageDateAttributes(IMagickImage? image)
         {
             if (image == null)