Implement tile caching (#1105)

Add a top level config parameter -- the size of cache memory (in MB) to
use for caching tiles and PMT directories, defaulting to 512, and 0 to
disable. This also removes the `pmtiles.dir_cache_size_mb` parameter (it
will be ignored, but will give a warning)

```
cache_size_mb: 512
```

The new cache will contain all tiles as provided by the source. So if
PostgreSQL returns a non-compressed tile, the cache will contain the
uncompressed variant, and will be compressed for each response. This
will be fixed in the later releases.

Note that fonts and sprites are not cached at this time, and are still a
TODO.

debian/config.yaml

@@ -7,6 +7,9 @@ listen_addresses: '0.0.0.0:3000'
 # Number of web server workers
 worker_processes: 8
 
+# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable]
+cache_size_mb: 512
+
 # see https://maplibre.org/martin/config-file.html
 
 # postgres:
@@ -17,7 +20,6 @@ worker_processes: 8
 #   auto_bounds: skip
 
 # pmtiles:
-#   dir_cache_size_mb: 100
 #   paths:
 #     - /dir-path
 #     - /path/to/pmtiles.pmtiles

docs/src/config-file.md

@@ -24,6 +24,9 @@ listen_addresses: '0.0.0.0:3000'
 # Number of web server workers
 worker_processes: 8
 
+# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable]
+cache_size_mb: 1024
+
 # Database configuration. This can also be a list of PG configs.
 postgres:
   # Database connection string. You can use env vars too, for example:
@@ -155,8 +158,6 @@ postgres:
 
 # Publish PMTiles files from local disk or proxy to a web server
 pmtiles:
-  # Memory (in MB) to use for caching PMTiles directories [default: 32, 0 to disable]] 
-  dir_cache_size_mb: 100
   paths:
     # scan this whole dir, matching all *.pmtiles files
     - /dir-path

martin-tile-utils/Cargo.toml

@@ -2,7 +2,7 @@ lints.workspace = true
 
 [package]
 name = "martin-tile-utils"
-version = "0.4.0"
+version = "0.4.1"
 authors = ["Yuri Astrakhan <YuriAstrakhan@gmail.com>", "MapLibre contributors"]
 description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server."
 keywords = ["maps", "tiles", "mvt", "tileserver"]

martin-tile-utils/src/lib.rs

@@ -86,7 +86,7 @@ impl Display for Format {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
 pub enum Encoding {
     /// Data is not compressed, but it can be
     Uncompressed = 0b0000_0000,

martin/Cargo.toml

@@ -3,7 +3,7 @@ lints.workspace = true
 [package]
 name = "martin"
 # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin
-version = "0.12.0"
+version = "0.13.0"
 authors = ["Stepan Kuzmin <to.stepan.kuzmin@gmail.com>", "Yuri Astrakhan <YuriAstrakhan@gmail.com>", "MapLibre contributors"]
 description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support"
 keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"]
@@ -62,7 +62,7 @@ harness = false
 default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"]
 fonts = ["dep:bit-set", "dep:pbf_font_tools"]
 mbtiles = []
-pmtiles = ["dep:moka"]
+pmtiles = []
 postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"]
 sprites = ["dep:spreet"]
 bless-tests = []
@@ -85,7 +85,7 @@ json-patch = { workspace = true, optional = true }
 log.workspace = true
 martin-tile-utils.workspace = true
 mbtiles.workspace = true
-moka = { workspace = true, optional = true }
+moka.workspace = true
 num_cpus.workspace = true
 pbf_font_tools = { workspace = true, optional = true }
 pmtiles.workspace = true

martin/benches/bench.rs

@@ -1,7 +1,7 @@
 use async_trait::async_trait;
 use criterion::async_executor::FuturesExecutor;
 use criterion::{criterion_group, criterion_main, Criterion};
-use martin::srv::get_tile_response;
+use martin::srv::DynTileSource;
 use martin::{
     CatalogSourceEntry, MartinResult, Source, TileCoord, TileData, TileSources, UrlQuery,
 };
@@ -58,7 +58,8 @@ impl Source for NullSource {
 }
 
 async fn process_tile(sources: &TileSources) {
-    get_tile_response(sources, TileCoord { z: 0, x: 0, y: 0 }, "null", "", None)
+    let src = DynTileSource::new(sources, "null", Some(0), "", None, None).unwrap();
+    src.get_http_response(TileCoord { z: 0, x: 0, y: 0 })
         .await
         .unwrap();
 }

martin/src/args/root.rs

@@ -43,6 +43,9 @@ pub struct MetaArgs {
     /// By default, only print if sources are auto-detected.
     #[arg(long)]
     pub save_config: Option<PathBuf>,
+    /// Main cache size (in MB)
+    #[arg(short = 'C', long)]
+    pub cache_size: Option<u64>,
     /// **Deprecated** Scan for new sources on sources list requests
     #[arg(short, long, hide = true)]
     pub watch: bool,
@@ -74,6 +77,10 @@ impl Args {
             return Err(ConfigAndConnectionsError(self.meta.connection));
         }
 
+        if self.meta.cache_size.is_some() {
+            config.cache_size_mb = self.meta.cache_size;
+        }
+
         self.srv.merge_into_config(&mut config.srv);
 
         #[allow(unused_mut)]

martin/src/bin/martin-cp.rs

@@ -12,10 +12,10 @@ use futures::stream::{self, StreamExt};
 use futures::TryStreamExt;
 use log::{debug, error, info, log_enabled};
 use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, SrvArgs};
-use martin::srv::{get_tile_content, merge_tilejson, RESERVED_KEYWORDS};
+use martin::srv::{merge_tilejson, DynTileSource};
 use martin::{
-    append_rect, read_config, Config, IdResolver, MartinError, MartinResult, ServerState, Source,
-    TileCoord, TileData, TileRect,
+    append_rect, read_config, Config, MartinError, MartinResult, ServerState, Source, TileCoord,
+    TileData, TileRect,
 };
 use martin_tile_utils::{bbox_to_xyz, TileInfo};
 use mbtiles::sqlx::SqliteConnection;
@@ -144,7 +144,8 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> {
 
     args.merge_into_config(&mut config, &env)?;
     config.finalize()?;
-    let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?;
+
+    let sources = config.resolve().await?;
 
     if let Some(file_name) = save_config {
         config.save_to_file(file_name)?;
@@ -274,9 +275,18 @@ fn iterate_tiles(tiles: Vec<TileRect>) -> impl Iterator<Item = TileCoord> {
 async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> {
     let output_file = &args.output_file;
     let concurrency = args.concurrency.unwrap_or(1);
-    let (sources, _use_url_query, info) = state.tiles.get_sources(args.source.as_str(), None)?;
-    let sources = sources.as_slice();
-    let tile_info = sources.first().unwrap().get_tile_info();
+
+    let src = DynTileSource::new(
+        &state.tiles,
+        args.source.as_str(),
+        None,
+        args.url_query.as_deref().unwrap_or_default(),
+        Some(parse_encoding(args.encoding.as_str())?),
+        None,
+    )?;
+    // parallel async below uses move, so we must only use copyable types
+    let src = &src;
+
     let (tx, mut rx) = channel::<TileXyz>(500);
     let tiles = compute_tile_ranges(&args);
     let mbt = Mbtiles::new(output_file)?;
@@ -288,30 +298,26 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()>
     } else {
         CopyDuplicateMode::Override
     };
-    let mbt_type = init_schema(&mbt, &mut conn, sources, tile_info, &args).await?;
-    let query = args.url_query.as_deref();
-    let req = TestRequest::default()
-        .insert_header((ACCEPT_ENCODING, args.encoding.as_str()))
-        .finish();
-    let accept_encoding = AcceptEncoding::parse(&req)?;
-    let encodings = Some(&accept_encoding);
+    let mbt_type = init_schema(&mbt, &mut conn, src.sources.as_slice(), src.info, &args).await?;
 
     let progress = Progress::new(&tiles);
     info!(
-        "Copying {} {tile_info} tiles from {} to {}",
+        "Copying {} {} tiles from {} to {}",
         progress.total,
+        src.info,
         args.source,
         args.output_file.display()
     );
 
     try_join!(
+        // Note: for some reason, tests hang here without the `move` keyword
         async move {
             stream::iter(iterate_tiles(tiles))
                 .map(MartinResult::Ok)
                 .try_for_each_concurrent(concurrency, |xyz| {
                     let tx = tx.clone();
                     async move {
-                        let tile = get_tile_content(sources, info, xyz, query, encodings).await?;
+                        let tile = src.get_tile_content(xyz).await?;
                         let data = tile.data;
                         tx.send(TileXyz { xyz, data })
                             .await
@@ -375,6 +381,13 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()>
     Ok(())
 }
 
+fn parse_encoding(encoding: &str) -> MartinCpResult<AcceptEncoding> {
+    let req = TestRequest::default()
+        .insert_header((ACCEPT_ENCODING, encoding))
+        .finish();
+    Ok(AcceptEncoding::parse(&req)?)
+}
+
 async fn init_schema(
     mbt: &Mbtiles,
     conn: &mut SqliteConnection,

martin/src/bin/martin.rs

@@ -4,8 +4,8 @@ use actix_web::dev::Server;
 use clap::Parser;
 use log::{error, info, log_enabled};
 use martin::args::{Args, OsEnv};
-use martin::srv::{new_server, RESERVED_KEYWORDS};
-use martin::{read_config, Config, IdResolver, MartinResult};
+use martin::srv::new_server;
+use martin::{read_config, Config, MartinResult};
 
 const VERSION: &str = env!("CARGO_PKG_VERSION");
 
@@ -24,7 +24,7 @@ async fn start(args: Args) -> MartinResult<Server> {
 
     args.merge_into_config(&mut config, &env)?;
     config.finalize()?;
-    let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?;
+    let sources = config.resolve().await?;
 
     if let Some(file_name) = save_config {
         config.save_to_file(file_name)?;