add tile-copy utility

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -41,7 +41,6 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Function;
-import java.util.stream.IntStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -656,18 +655,8 @@ public void run() throws Exception {
       System.exit(0);
     } else if (onlyDownloadSources) {
       // don't check files if not generating map
-    } else if (config.append()) {
-      if (!output.format().supportsAppend()) {
-        throw new IllegalArgumentException("cannot append to " + output.format().id());
-      }
-      if (!output.exists()) {
-        throw new IllegalArgumentException(output.uri() + " must exist when appending");
-      }
-    } else if (overwrite || config.force()) {
-      output.delete();
-    } else if (output.exists()) {
-      throw new IllegalArgumentException(
-        output.uri() + " already exists, use the --force argument to overwrite or --append.");
+    } else {
+      output.setup(config.force() || overwrite, config.append(), config.tileWriteThreads());
     }
 
     Path layerStatsPath = arguments.file("layer_stats", "layer stats output path",
@@ -677,23 +666,6 @@ public void run() throws Exception {
     if (config.tileWriteThreads() < 1) {
       throw new IllegalArgumentException("require tile_write_threads >= 1");
     }
-    if (config.tileWriteThreads() > 1) {
-      if (!output.format().supportsConcurrentWrites()) {
-        throw new IllegalArgumentException(output.format() + " doesn't support concurrent writes");
-      }
-      IntStream.range(1, config.tileWriteThreads())
-        .mapToObj(output::getPathForMultiThreadedWriter)
-        .forEach(p -> {
-          if (!config.append() && (overwrite || config.force())) {
-            FileUtils.delete(p);
-          }
-          if (config.append() && !output.exists(p)) {
-            throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending");
-          } else if (!config.append() && output.exists(p)) {
-            throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending");
-          }
-        });
-    }
 
     LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output.uri());
 
@@ -718,7 +690,7 @@ public void run() throws Exception {
     // in case any temp files are left from a previous run...
     FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath);
     Files.createDirectories(tmpDir);
-    FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalBasePath());
+    FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath);
 
     if (!toDownload.isEmpty()) {
       download();

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java

@@ -16,6 +16,7 @@
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 /**
@@ -158,15 +159,14 @@ public Path getLocalPath() {
   /**
    * Returns the local <b>base</b> path for this archive, for which directories should be pre-created for.
    */
-  public Path getLocalBasePath() {
+  Path getLocalBasePath() {
     Path p = getLocalPath();
     if (format() == Format.FILES) {
       p = FilesArchiveUtils.cleanBasePath(p);
     }
     return p;
   }
 
-
   /**
    * Deletes the archive if possible.
    */
@@ -187,7 +187,7 @@ public boolean exists() {
    * @param p path to the archive
    * @return {@code true} if the archive already exists, {@code false} otherwise.
    */
-  public boolean exists(Path p) {
+  private boolean exists(Path p) {
     if (p == null) {
       return false;
     }
@@ -229,14 +229,49 @@ public Path getPathForMultiThreadedWriter(int index) {
     };
   }
 
+  public void setup(boolean force, boolean append, int tileWriteThreads) {
+    if (append) {
+      if (!format().supportsAppend()) {
+        throw new IllegalArgumentException("cannot append to " + format().id());
+      }
+      if (!exists()) {
+        throw new IllegalArgumentException(uri() + " must exist when appending");
+      }
+    } else if (force) {
+      delete();
+    } else if (exists()) {
+      throw new IllegalArgumentException(uri() + " already exists, use the --force argument to overwrite or --append.");
+    }
+
+    if (tileWriteThreads > 1) {
+      if (!format().supportsConcurrentWrites()) {
+        throw new IllegalArgumentException(format() + " doesn't support concurrent writes");
+      }
+      IntStream.range(1, tileWriteThreads)
+        .mapToObj(this::getPathForMultiThreadedWriter)
+        .forEach(p -> {
+          if (!append && force) {
+            FileUtils.delete(p);
+          }
+          if (append && !exists(p)) {
+            throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending");
+          } else if (!append && exists(p)) {
+            throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending");
+          }
+        });
+    }
+
+    FileUtils.createParentDirectories(getLocalBasePath());
+  }
+
   public enum Format {
     MBTILES("mbtiles",
       false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */,
-      false, TileOrder.TMS),
-    PMTILES("pmtiles", false, false, TileOrder.HILBERT),
+      false, false, TileOrder.TMS),
+    PMTILES("pmtiles", false, false, false, TileOrder.HILBERT),
 
     // should be before PBF in order to avoid collisions
-    FILES("files", true, true, TileOrder.TMS) {
+    FILES("files", true, true, true, TileOrder.TMS) {
       @Override
       boolean isUriSupported(URI uri) {
         final String path = uri.getPath();
@@ -245,25 +280,28 @@ boolean isUriSupported(URI uri) {
       }
     },
 
-    CSV("csv", true, true, TileOrder.TMS),
+    CSV("csv", true, true, false, TileOrder.TMS),
     /** identical to {@link Format#CSV} - except for the column separator */
-    TSV("tsv", true, true, TileOrder.TMS),
+    TSV("tsv", true, true, false, TileOrder.TMS),
 
-    PROTO("proto", true, true, TileOrder.TMS),
+    PROTO("proto", true, true, false, TileOrder.TMS),
     /** identical to {@link Format#PROTO} */
-    PBF("pbf", true, true, TileOrder.TMS),
+    PBF("pbf", true, true, false, TileOrder.TMS),
 
-    JSON("json", true, true, TileOrder.TMS);
+    JSON("json", true, true, false, TileOrder.TMS);
 
     private final String id;
     private final boolean supportsAppend;
     private final boolean supportsConcurrentWrites;
+    private final boolean supportsConcurrentReads;
     private final TileOrder order;
 
-    Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites, TileOrder order) {
+    Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites, boolean supportsConcurrentReads,
+      TileOrder order) {
       this.id = id;
       this.supportsAppend = supportsAppend;
       this.supportsConcurrentWrites = supportsConcurrentWrites;
+      this.supportsConcurrentReads = supportsConcurrentReads;
       this.order = order;
     }
 
@@ -283,6 +321,10 @@ public boolean supportsConcurrentWrites() {
       return supportsConcurrentWrites;
     }
 
+    public boolean supportsConcurrentReads() {
+      return supportsConcurrentReads;
+    }
+
     boolean isUriSupported(URI uri) {
       final String path = uri.getPath();
       return path != null && path.endsWith("." + id);

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java

@@ -170,6 +170,11 @@ public TileArchiveMetadata withJson(TileArchiveMetadataJson json) {
       maxzoom, json, others, tileCompression);
   }
 
+  public TileArchiveMetadata withTileCompression(TileCompression tileCompression) {
+    return new TileArchiveMetadata(name, description, attribution, version, type, format, bounds, center, minzoom,
+      maxzoom, json, others, tileCompression);
+  }
+
   /*
    * few workarounds to make collect unknown fields to others work,
    * because @JsonAnySetter does not yet work on constructor/creator arguments

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java

@@ -261,7 +261,7 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
      * To optimize emitting many identical consecutive tiles (like large ocean areas), memoize output to avoid
      * recomputing if the input hasn't changed.
      */
-    byte[] lastBytes = null, lastEncoded = null;
+    byte[] lastBytes = null;
     Long lastTileDataHash = null;
     boolean lastIsFill = false;
     List<TileSizeStats.LayerStats> lastLayerStats = null;
@@ -276,24 +276,22 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
       for (int i = 0; i < batch.in.size(); i++) {
         FeatureGroup.TileFeatures tileFeatures = batch.in.get(i);
         featuresProcessed.incBy(tileFeatures.getNumFeaturesProcessed());
-        byte[] bytes, encoded;
+        byte[] bytes;
         List<TileSizeStats.LayerStats> layerStats;
         Long tileDataHash;
         if (tileFeatures.hasSameContents(last)) {
           bytes = lastBytes;
-          encoded = lastEncoded;
           tileDataHash = lastTileDataHash;
           layerStats = lastLayerStats;
           memoizedTiles.inc();
         } else {
           VectorTile tile = tileFeatures.getVectorTile(layerAttrStatsUpdater);
           if (skipFilled && (lastIsFill = tile.containsOnlyFills())) {
-            encoded = null;
             layerStats = null;
             bytes = null;
           } else {
             var proto = tile.toProto();
-            encoded = proto.toByteArray();
+            var encoded = proto.toByteArray();
             bytes = switch (config.tileCompression()) {
               case GZIP -> gzip(encoded);
               case NONE -> encoded;
@@ -307,7 +305,6 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
             }
           }
           lastLayerStats = layerStats;
-          lastEncoded = encoded;
           lastBytes = bytes;
           last = tileFeatures;
           if (archive.deduplicates() && tile.likelyToBeDuplicated() && bytes != null) {
@@ -326,7 +323,6 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
             new TileEncodingResult(
               tileFeatures.tileCoord(),
               bytes,
-              encoded.length,
               tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash),
               layerStatsRows
             )

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java

@@ -1,17 +1,23 @@
 package com.onthegomap.planetiler.archive;
 
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.config.CommonConfigs;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.files.ReadableFilesArchive;
 import com.onthegomap.planetiler.files.WriteableFilesArchive;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
 import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
 import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
+import com.onthegomap.planetiler.stream.ReadableCsvArchive;
+import com.onthegomap.planetiler.stream.ReadableJsonStreamArchive;
+import com.onthegomap.planetiler.stream.ReadableProtoStreamArchive;
 import com.onthegomap.planetiler.stream.StreamArchiveConfig;
 import com.onthegomap.planetiler.stream.WriteableCsvArchive;
 import com.onthegomap.planetiler.stream.WriteableJsonStreamArchive;
 import com.onthegomap.planetiler.stream.WriteableProtoStreamArchive;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.function.Supplier;
 
 /** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */
 public class TileArchives {
@@ -37,45 +43,58 @@ public static ReadableTileArchive newReader(String archive, PlanetilerConfig con
     return newReader(TileArchiveConfig.from(archive), config);
   }
 
+  public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config)
+    throws IOException {
+    return newWriter(archive, config.arguments());
+  }
+
   /**
    * Returns a new {@link WriteableTileArchive} from the string definition in {@code archive}.
    *
    * @throws IOException if an error occurs creating the resource.
    */
-  public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config)
+  public static WriteableTileArchive newWriter(TileArchiveConfig archive, Arguments baseArguments)
     throws IOException {
-    var options = archive.applyFallbacks(config.arguments());
+    var options = archive.applyFallbacks(baseArguments);
     var format = archive.format();
     return switch (format) {
       case MBTILES ->
         // pass-through legacy arguments for fallback
-        Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(config.arguments()
+        Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(baseArguments
           .subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION)));
       case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath());
       case CSV, TSV -> WriteableCsvArchive.newWriteToFile(format, archive.getLocalPath(),
-        new StreamArchiveConfig(config, options));
+        new StreamArchiveConfig(baseArguments, options));
       case PROTO, PBF -> WriteableProtoStreamArchive.newWriteToFile(archive.getLocalPath(),
-        new StreamArchiveConfig(config, options));
+        new StreamArchiveConfig(baseArguments, options));
       case JSON -> WriteableJsonStreamArchive.newWriteToFile(archive.getLocalPath(),
-        new StreamArchiveConfig(config, options));
-      case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options, config.force() || config.append());
+        new StreamArchiveConfig(baseArguments, options));
+      case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options,
+        CommonConfigs.appendToArchive(baseArguments) || CommonConfigs.force(baseArguments));
     };
   }
 
+  public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config)
+    throws IOException {
+    return newReader(archive, config.arguments());
+  }
+
   /**
    * Returns a new {@link ReadableTileArchive} from the string definition in {@code archive}.
    *
    * @throws IOException if an error occurs opening the resource.
    */
-  public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config)
+  public static ReadableTileArchive newReader(TileArchiveConfig archive, Arguments baseArguments)
     throws IOException {
-    var options = archive.applyFallbacks(config.arguments());
+    var options = archive.applyFallbacks(baseArguments);
+    Supplier<StreamArchiveConfig> streamArchiveConfig = () -> new StreamArchiveConfig(baseArguments, options);
     return switch (archive.format()) {
       case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options);
       case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath());
-      case CSV, TSV -> throw new UnsupportedOperationException("reading CSV is not supported");
-      case PROTO, PBF -> throw new UnsupportedOperationException("reading PROTO is not supported");
-      case JSON -> throw new UnsupportedOperationException("reading JSON is not supported");
+      case CSV, TSV ->
+        ReadableCsvArchive.newReader(archive.format(), archive.getLocalPath(), streamArchiveConfig.get());
+      case PROTO, PBF -> ReadableProtoStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get());
+      case JSON -> ReadableJsonStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get());
       case FILES -> ReadableFilesArchive.newReader(archive.getLocalPath(), options);
     };
   }

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java

@@ -54,10 +54,5 @@ static class Deserializer extends JsonDeserializer<TileCompression> {
     public TileCompression deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
       return findById(p.getValueAsString()).orElse(TileCompression.UNKNOWN);
     }
-
-    @Override
-    public TileCompression getNullValue(DeserializationContext ctxt) {
-      return TileCompression.GZIP;
-    }
   }
 }

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java

@@ -5,12 +5,10 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.OptionalLong;
-import javax.annotation.Nonnull;
 
 public record TileEncodingResult(
   TileCoord coord,
-  @Nonnull byte[] tileData,
-  int rawTileSize,
+  byte[] tileData,
   /* will always be empty in non-compact mode and might also be empty in compact mode */
   OptionalLong tileDataHash,
   List<String> layerStats
@@ -20,7 +18,7 @@ public TileEncodingResult(
     byte[] tileData,
     OptionalLong tileDataHash
   ) {
-    this(coord, tileData, tileData.length, tileDataHash, List.of());
+    this(coord, tileData, tileDataHash, List.of());
   }
 
   @Override