Refactor mbtiles lib, mbtiles tool tests & CI (#702)

* Broke up martin-mbtiles into multiple files
* Made all mbtiles functions take a `SqliteExecutor` -- this way they
can be used with any SQLX connection structs - either a pool connection
or an individual non-pooled connection.
* Simplified mbtiles bin a bit - I realized there is really no need to
pretty print the output for the single value retrieval. Easier to just
dump it to console as is.
* Bump martin-mbtiles to v0.2.0
* Minor fixes in tools docs, cargo.toml, and justfile
* MBTiles tool Integration tests and release publishing

Major thanks to the
[stackoverflow](https://stackoverflow.com/questions/76394665/how-to-pass-sqlx-connection-a-mut-trait-as-a-fn-parameter-in-rust/76395111)
quick reply by @cafce25 on how to use generic sql executor!

.github/workflows/ci.yml

@@ -61,15 +61,19 @@ jobs:
           if [[ "${{ matrix.target }}" == "aarch64-apple-darwin" ]]; then
             rustup target add aarch64-apple-darwin
             # compile without debug symbols
-            RUSTFLAGS='-C link-arg=-s' cargo build --release --target ${{ matrix.target }} --features=vendored-openssl
+            RUSTFLAGS='-C link-arg=-s' cargo build --release --target ${{ matrix.target }} --features=vendored-openssl --package martin
+            RUSTFLAGS='-C link-arg=-s' cargo build --release --target ${{ matrix.target }} --features=tools --package martin-mbtiles
           else
-            cargo build --release --target ${{ matrix.target }} --features=ssl
+            cargo build --release --target ${{ matrix.target }} --features=ssl --package martin
+            cargo build --release --target ${{ matrix.target }} --features=tools --package martin-mbtiles
           fi
           mkdir target_releases
           if [[ "${{ runner.os }}" == "Windows" ]]; then
             mv target/${{ matrix.target }}/release/martin.exe target_releases
+            mv target/${{ matrix.target }}/release/mbtiles.exe target_releases
           else
             mv target/${{ matrix.target }}/release/martin target_releases
+            mv target/${{ matrix.target }}/release/mbtiles target_releases
           fi
       - name: Save build artifact build-${{ matrix.target }}
         uses: actions/upload-artifact@v3
@@ -86,19 +90,19 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            bin: martin
+            ext: ''
             target: x86_64-unknown-linux-gnu
             name: martin-Linux-x86_64.tar.gz
           - os: windows-latest
-            bin: martin.exe
+            ext: '.exe'
             target: x86_64-pc-windows-msvc
             name: martin-Windows-x86_64.zip
           - os: macOS-latest
-            bin: martin
+            ext: ''
             target: x86_64-apple-darwin
             name: martin-Darwin-x86_64.tar.gz
           - os: ubuntu-latest
-            bin: martin
+            ext: ''
             target: aarch64-apple-darwin
             name: martin-Darwin-aarch64.tar.gz
     runs-on: ${{ matrix.os }}
@@ -147,13 +151,17 @@ jobs:
         shell: bash
         run: |
           if [[ "${{ runner.os }}" != "Windows" ]]; then
-            chmod +x target/${{ matrix.bin }}
+            chmod +x target/martin${{ matrix.ext }}
+            chmod +x target/mbtiles${{ matrix.ext }}
           fi
           tests/test.sh
         env:
           DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
           MARTIN_BUILD: "-"
-          MARTIN_BIN: target/${{ matrix.bin }}
+          MARTIN_BIN: target/martin${{ matrix.ext }}
+          MBTILES_BUILD: "-"
+          MBTILES_BIN: target/mbtiles${{ matrix.ext }}
+
       - name: Compare test output results (Linux)
         if: matrix.target == 'x86_64-unknown-linux-gnu'
         run: diff --brief --recursive --new-file tests/output tests/expected
@@ -170,12 +178,13 @@ jobs:
           cd target/
           # Symbol stripping does not work cross-platform
           if [[ "${{ matrix.target }}" != "aarch64-apple-darwin" ]]; then
-            strip ${{ matrix.bin }}
+            strip martin${{ matrix.ext }}
+            strip mbtiles${{ matrix.ext }}
           fi
           if [[ "${{ runner.os }}" == "Windows" ]]; then
-            7z a ../${{ matrix.name }} ${{ matrix.bin }}
+            7z a ../${{ matrix.name }} martin${{ matrix.ext }} mbtiles${{ matrix.ext }}
           else
-            tar czvf ../${{ matrix.name }} ${{ matrix.bin }}
+            tar czvf ../${{ matrix.name }} martin${{ matrix.ext }} mbtiles${{ matrix.ext }}
           fi
           cd -
       - name: Generate SHA-256 (MacOS)
@@ -276,9 +285,11 @@ jobs:
         path: target/
     - name: Integration Tests
       run: |
-        chmod +x target/martin
+        chmod +x target/martin target/mbtiles
         tests/test.sh
       env:
         DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
         MARTIN_BUILD: "-"
         MARTIN_BIN: target/martin
+        MBTILES_BUILD: "-"
+        MBTILES_BIN: target/mbtiles

.github/workflows/docker.yml

@@ -94,6 +94,8 @@ jobs:
         if: matrix.platform == 'linux/amd64'
         run: |
           TAG=$(echo '${{ steps.docker_meta.outputs.json }}' | jq -r '.tags[0]')
+          export MBTILES_BUILD=-
+          export MBTILES_BIN=-
           export MARTIN_BUILD=-
           export MARTIN_BIN="docker run --rm --net host -e DATABASE_URL -v $PWD/tests:/tests $TAG"
           echo "MARTIN_BIN=$MARTIN_BIN"

Cargo.toml

@@ -27,11 +27,11 @@ ssl = ["openssl", "postgres-openssl"]
 vendored-openssl = ["ssl", "openssl/vendored"]
 
 [dependencies]
-actix.workspace = true
 actix-cors.workspace = true
 actix-http.workspace = true
 actix-rt.workspace = true
 actix-web.workspace = true
+actix.workspace = true
 async-trait.workspace = true
 brotli.workspace = true
 clap.workspace = true
@@ -41,15 +41,13 @@ flate2.workspace = true
 futures.workspace = true
 itertools.workspace = true
 log.workspace = true
-martin-mbtiles = { path = "./martin-mbtiles", version = "0.1.0" }
-martin-tile-utils = { path = "./martin-tile-utils", version = "0.1.0" }
+martin-mbtiles.workspace = true
+martin-tile-utils.workspace = true
 num_cpus.workspace = true
-openssl = { workspace = true, optional = true }
 pmtiles.workspace = true
 postgis.workspace = true
-postgres.workspace = true
-postgres-openssl = { workspace = true, optional = true }
 postgres-protocol.workspace = true
+postgres.workspace = true
 regex.workspace = true
 semver.workspace = true
 serde.workspace = true
@@ -59,6 +57,10 @@ subst.workspace = true
 thiserror.workspace = true
 tilejson.workspace = true
 
+# Optional dependencies for openssl support
+openssl = { workspace = true, optional = true }
+postgres-openssl = { workspace = true, optional = true }
+
 [dev-dependencies]
 cargo-husky.workspace = true
 criterion.workspace = true
@@ -104,6 +106,8 @@ futures = "0.3"
 indoc = "2"
 itertools = "0.10"
 log = "0.4"
+martin-mbtiles = { path = "./martin-mbtiles", version = "0.2.0" }
+martin-tile-utils = { path = "./martin-tile-utils", version = "0.1.0" }
 num_cpus = "1"
 openssl = "0.10"
 pmtiles = { version = "0.2.2", features = ["mmap-async-tokio", "tilejson"] }

docs/src/tools.md

@@ -1,7 +1,13 @@
 # Tools
 
+Martin has a few additional tools that can be used to interact with the data.
+
 ## MBTiles tools
-A small utility that allows users to interact with mbtiles files from the CLI as follows: `mbtiles <command> ...`
+A small utility that allows users to interact with the `*.mbtiles` files from the command line. Use `mbtiles --help` to see a list of available commands, and `mbtiles <command> --help` to see help for a specific command.
+
+### meta-get
+Retrieve raw metadata value by its name. The value is printed to stdout without any modifications.
 
-#### `meta-get`
-Retrieve a metadata value by its name: `mbtiles meta-get <file.mbtiles> <key>`. See `mbtiles meta-get --help` to see available options.
+```shell
+mbtiles meta-get <file.mbtiles> <key>
+```

justfile

@@ -176,9 +176,15 @@ print-conn-str:
     @echo {{ DATABASE_URL }}
 
 # Run cargo fmt and cargo clippy
-lint:
+lint: fmt clippy
+
+# Run cargo fmt
+fmt:
     cargo fmt --all -- --check
-    cargo clippy --workspace --all-targets --all-features -- -D warnings
+
+# Run cargo clippy
+clippy:
+    cargo clippy --workspace --all-targets --all-features --bins --tests --lib --benches -- -D warnings
 
 # These steps automatically run before git push via a git hook
 [private]

martin-mbtiles/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "martin-mbtiles"
-version = "0.1.0"
+version = "0.2.0"
 authors = ["Yuri Astrakhan <YuriAstrakhan@gmail.com>", "MapLibre contributors"]
 description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics."
 keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"]
@@ -13,19 +13,24 @@ rust-version.workspace = true
 repository.workspace = true
 license.workspace = true
 
+[features]
+# TODO: Disable "tools" feature in default builds
+default = ["tools"]
+tools = ["anyhow", "clap", "tokio"]
+
 [dependencies]
 futures.workspace = true
 log.workspace = true
-martin-tile-utils = { path = "../martin-tile-utils", version = "0.1.0" }
+martin-tile-utils.workspace = true
 serde_json.workspace = true
 sqlx.workspace = true
 thiserror.workspace = true
 tilejson.workspace = true
 
 # Bin dependencies
-anyhow.workspace = true
-clap.workspace = true
-tokio.workspace = true
+anyhow = { workspace = true, optional = true }
+clap = { workspace = true, optional = true }
+tokio = { workspace = true, optional = true }
 
 [dev-dependencies]
 # For testing, might as well use the same async framework as the Martin itself
@@ -37,3 +42,4 @@ path = "src/lib.rs"
 [[bin]]
 name = "mbtiles"
 path = "src/bin/main.rs"
+required-features = ["tools"]

martin-mbtiles/src/bin/main.rs

@@ -1,13 +1,16 @@
+use std::path::{Path, PathBuf};
+
 use anyhow::Result;
 use clap::{Parser, Subcommand};
 use martin_mbtiles::Mbtiles;
-use std::path::PathBuf;
+use sqlx::sqlite::SqliteConnectOptions;
+use sqlx::{Connection, SqliteConnection};
 
 #[derive(Parser, Debug)]
 #[command(
     version,
     name = "mbtiles",
-    about = "A utility to work with .mbtiles files content"
+    about = "A utility to work with .mbtiles file content"
 )]
 pub struct Args {
     #[command(subcommand)]
@@ -16,63 +19,54 @@ pub struct Args {
 
 #[derive(Subcommand, Debug)]
 enum Commands {
-    /// Prints all values in the metadata table.
-    #[command(name = "meta-all")]
-    MetaAll {
-        /// MBTiles file to read from
-        file: PathBuf,
-    },
+    // /// Prints all values in the metadata table.
+    // #[command(name = "meta-all")]
+    // MetaAll {
+    //     /// MBTiles file to read from
+    //     file: PathBuf,
+    // },
     /// Gets a single value from metadata table.
     #[command(name = "meta-get")]
     MetaGetValue {
         /// MBTiles file to read a value from
         file: PathBuf,
         /// Value to read
         key: String,
-        /// Output the raw value
-        #[arg(short, long)]
-        raw: bool,
-    },
-    /// Sets a single value in the metadata table, or deletes it if no value.
-    #[command(name = "meta-set")]
-    MetaSetValue {
-        /// MBTiles file to modify
-        file: PathBuf,
-    },
-    /// Copy tiles from one mbtiles file to another.
-    Copy {
-        /// MBTiles file to read from
-        src_file: PathBuf,
-        /// MBTiles file to write to
-        dst_file: PathBuf,
     },
+    // /// Sets a single value in the metadata table, or deletes it if no value.
+    // #[command(name = "meta-set")]
+    // MetaSetValue {
+    //     /// MBTiles file to modify
+    //     file: PathBuf,
+    // },
+    // /// Copy tiles from one mbtiles file to another.
+    // Copy {
+    //     /// MBTiles file to read from
+    //     src_file: PathBuf,
+    //     /// MBTiles file to write to
+    //     dst_file: PathBuf,
+    // },
 }
 
 #[tokio::main]
 async fn main() -> Result<()> {
     let args = Args::parse();
 
     match args.command {
-        Commands::MetaGetValue { file, key, raw } => {
-            let mbt = Mbtiles::new(&file).await?;
-
-            let value = mbt.get_metadata_value(&key).await?;
-
-            if raw {
-                if let Some(s) = value {
-                    println!("{s}")
-                }
-            } else {
-                match value {
-                    Some(s) => println!(r#"The value for metadata key "{key}" is:\n "{s}""#),
-                    None => println!(r#"No value for metadata key "{key}""#),
-                }
-            }
-        }
-        _ => {
-            unimplemented!("Oops! This command is not yet available, stay tuned for future updates")
+        Commands::MetaGetValue { file, key } => {
+            meta_get_value(file.as_path(), &key).await?;
         }
     }
 
     Ok(())
 }
+
+async fn meta_get_value(file: &Path, key: &str) -> Result<()> {
+    let mbt = Mbtiles::new(file)?;
+    let opt = SqliteConnectOptions::new().filename(file).read_only(true);
+    let mut conn = SqliteConnection::connect_with(&opt).await?;
+    if let Some(s) = mbt.get_metadata_value(&mut conn, key).await? {
+        println!("{s}")
+    }
+    Ok(())
+}

martin-mbtiles/src/errors.rs

@@ -0,0 +1,20 @@
+use std::path::PathBuf;
+
+use martin_tile_utils::TileInfo;
+
+#[derive(thiserror::Error, Debug)]
+pub enum MbtError {
+    #[error("SQL Error {0}")]
+    SqlError(#[from] sqlx::Error),
+
+    #[error("MBTile filepath contains unsupported characters: {}", .0.display())]
+    UnsupportedCharsInFilepath(PathBuf),
+
+    #[error("Inconsistent tile formats detected: {0} vs {1}")]
+    InconsistentMetadata(TileInfo, TileInfo),
+
+    #[error("No tiles found")]
+    NoTilesFound,
+}
+
+pub type MbtResult<T> = Result<T, MbtError>;