diff --git a/mbtiles/src/bin/mbtiles.rs b/mbtiles/src/bin/mbtiles.rs index 74c5392c7..aaf0a471f 100644 --- a/mbtiles/src/bin/mbtiles.rs +++ b/mbtiles/src/bin/mbtiles.rs @@ -158,6 +158,9 @@ pub struct SharedCopyOpts { /// Force copy operation, ignoring some warnings that otherwise would prevent the operation. Use with caution. #[arg(short, long)] force: bool, + /// Perform agg_hash validation on the original and destination files. + #[arg(long)] + validate: bool, } impl SharedCopyOpts { @@ -184,6 +187,7 @@ impl SharedCopyOpts { bbox: self.bbox, skip_agg_tiles_hash: self.skip_agg_tiles_hash, force: self.force, + validate: self.validate, // Constants dst_type: None, // Taken from dst_type_cli } @@ -262,7 +266,7 @@ async fn main_int() -> anyhow::Result<()> { } }); let mbt = Mbtiles::new(file.as_path())?; - mbt.validate(integrity_check, agg_hash).await?; + mbt.open_and_validate(integrity_check, agg_hash).await?; } Commands::Summary { file } => { let mbt = Mbtiles::new(file.as_path())?; diff --git a/mbtiles/src/copier.rs b/mbtiles/src/copier.rs index 9ceec73ec..1197f189f 100644 --- a/mbtiles/src/copier.rs +++ b/mbtiles/src/copier.rs @@ -14,6 +14,8 @@ use crate::errors::MbtResult; use crate::queries::{ create_tiles_with_hash_view, detach_db, init_mbtiles_schema, is_empty_database, }; +use crate::AggHashType::Verify; +use crate::IntegrityCheckType::Quick; use crate::MbtType::{Flat, FlatWithHash, Normalized}; use crate::{ invert_y_value, reset_db_settings, CopyType, MbtError, MbtType, MbtTypeCli, Mbtiles, @@ -70,6 +72,8 @@ pub struct MbtilesCopier { pub skip_agg_tiles_hash: bool, /// Ignore some warnings and continue with the copying operation pub force: bool, + /// Perform agg_hash validation on the original and destination files. + pub validate: bool, } #[derive(Clone, Debug)] @@ -193,6 +197,9 @@ impl MbtileCopierInt { let mut dif_conn = dif_mbt.open_readonly().await?; let dif_info = dif_mbt.get_diff_info(&mut dif_conn).await?; let dif_type = dif_info.mbt_type; + if self.options.validate { + dif_mbt.validate(&mut dif_conn, Quick, Verify).await?; + } if is_creating_diff { dif_mbt.validate_file_info(&dif_info, self.options.force)?; } else { @@ -203,11 +210,15 @@ impl MbtileCopierInt { let src_mbt = &self.src_mbtiles; let mut src_conn = src_mbt.open_readonly().await?; let src_info = src_mbt.get_diff_info(&mut src_conn).await?; + if self.options.validate { + src_mbt.validate(&mut src_conn, Quick, Verify).await?; + } let src_type = src_info.mbt_type; src_mbt.validate_file_info(&src_info, self.options.force)?; drop(src_conn); - let mut conn = self.dst_mbtiles.open_or_new().await?; + let dst_mbt = &self.dst_mbtiles; + let mut conn = dst_mbt.open_or_new().await?; if !is_empty_database(&mut conn).await? { return Err(MbtError::NonEmptyTargetFile(self.options.dst_file)); } @@ -216,7 +227,7 @@ impl MbtileCopierInt { dif_mbt.attach_to(&mut conn, "diffDb").await?; let what = self.copy_text(); - let dst_path = self.dst_mbtiles.filepath(); + let dst_path = dst_mbt.filepath(); let dif_path = dif_mbt.filepath(); let dst_type = self.options.dst_type().unwrap_or(src_type); if is_creating_diff { @@ -243,12 +254,12 @@ impl MbtileCopierInt { if is_creating_diff { if let Some(hash) = src_info.agg_tiles_hash { - self.dst_mbtiles + dst_mbt .set_metadata_value(&mut conn, AGG_TILES_HASH_BEFORE_APPLY, &hash) .await?; } if let Some(hash) = dif_info.agg_tiles_hash { - self.dst_mbtiles + dst_mbt .set_metadata_value(&mut conn, AGG_TILES_HASH_AFTER_APPLY, &hash) .await?; } @@ -256,10 +267,10 @@ impl MbtileCopierInt { // TODO: perhaps disable all except --copy all when using with diffs, or else is not making much sense if self.options.copy.copy_tiles() && !self.options.skip_agg_tiles_hash { - self.dst_mbtiles.update_agg_tiles_hash(&mut conn).await?; + dst_mbt.update_agg_tiles_hash(&mut conn).await?; if !is_creating_diff { - let new_hash = self.dst_mbtiles.get_agg_tiles_hash(&mut conn).await?; + let new_hash = dst_mbt.get_agg_tiles_hash(&mut conn).await?; match (dif_info.agg_tiles_hash_after_apply, new_hash) { (Some(expected), Some(actual)) if expected != actual => { let err = MbtError::AggHashMismatchAfterApply( @@ -281,6 +292,10 @@ impl MbtileCopierInt { detach_db(&mut conn, "diffDb").await?; detach_db(&mut conn, "sourceDb").await?; + if self.options.validate { + dst_mbt.validate(&mut conn, Quick, Verify).await?; + } + Ok(conn) } diff --git a/mbtiles/src/validation.rs b/mbtiles/src/validation.rs index dab783b22..9a4fdcefc 100644 --- a/mbtiles/src/validation.rs +++ b/mbtiles/src/validation.rs @@ -75,7 +75,7 @@ pub enum AggHashType { } impl Mbtiles { - pub async fn validate( + pub async fn open_and_validate( &self, check_type: IntegrityCheckType, agg_hash: AggHashType, @@ -85,12 +85,24 @@ impl Mbtiles { } else { self.open_readonly().await? }; - self.check_integrity(&mut conn, check_type).await?; - self.check_tiles_type_validity(&mut conn).await?; - self.check_each_tile_hash(&mut conn).await?; + self.validate(&mut conn, check_type, agg_hash).await + } + + pub async fn validate( + &self, + conn: &mut T, + check_type: IntegrityCheckType, + agg_hash: AggHashType, + ) -> MbtResult + where + for<'e> &'e mut T: SqliteExecutor<'e>, + { + self.check_integrity(&mut *conn, check_type).await?; + self.check_tiles_type_validity(&mut *conn).await?; + self.check_each_tile_hash(&mut *conn).await?; match agg_hash { - AggHashType::Verify => self.check_agg_tiles_hashes(&mut conn).await, - AggHashType::Update => self.update_agg_tiles_hash(&mut conn).await, + AggHashType::Verify => self.check_agg_tiles_hashes(conn).await, + AggHashType::Update => self.update_agg_tiles_hash(conn).await, AggHashType::Off => Ok(String::new()), } } diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index 098a0ab64..70b5886d6 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -242,7 +242,7 @@ fn databases() -> Databases { copy!(result.path("empty_no_hash", mbt_typ), path(&empty_mbt)); let dmp = dump(&mut empty_cn).await.unwrap(); assert_dump!(&dmp, "{typ}__empty"); - let hash = empty_mbt.validate(Off, Verify).await.unwrap(); + let hash = empty_mbt.open_and_validate(Off, Verify).await.unwrap(); allow_duplicates! { assert_snapshot!(hash, @"D41D8CD98F00B204E9800998ECF8427E"); } @@ -265,7 +265,7 @@ fn databases() -> Databases { copy!(result.path("v1_no_hash", mbt_typ), path(&v1_mbt)); let dmp = dump(&mut v1_cn).await.unwrap(); assert_dump!(&dmp, "{typ}__v1"); - let hash = v1_mbt.validate(Off, Verify).await.unwrap(); + let hash = v1_mbt.open_and_validate(Off, Verify).await.unwrap(); allow_duplicates! { assert_snapshot!(hash, @"9ED9178D7025276336C783C2B54D6258"); } @@ -276,7 +276,7 @@ fn databases() -> Databases { new_file!(databases, mbt_typ, METADATA_V2, TILES_V2, "{typ}__v2"); let dmp = dump(&mut v2_cn).await.unwrap(); assert_dump!(&dmp, "{typ}__v2"); - let hash = v2_mbt.validate(Off, Verify).await.unwrap(); + let hash = v2_mbt.open_and_validate(Off, Verify).await.unwrap(); allow_duplicates! { assert_snapshot!(hash, @"3BCDEE3F52407FF1315629298CB99133"); } @@ -291,7 +291,7 @@ fn databases() -> Databases { }; let dmp = dump(&mut dif_cn).await.unwrap(); assert_dump!(&dmp, "{typ}__dif"); - let hash = dif_mbt.validate(Off, Verify).await.unwrap(); + let hash = dif_mbt.open_and_validate(Off, Verify).await.unwrap(); allow_duplicates! { assert_snapshot!(hash, @"B86122579EDCDD4C51F3910894FCC1A1"); } @@ -300,7 +300,7 @@ fn databases() -> Databases { // ----------------- v1_clone ----------------- let (v1_clone_mbt, v1_clone_cn) = open!(databases, "{typ}__v1-clone"); let dmp = copy_dump!(result.path("v1", mbt_typ), path(&v1_clone_mbt)); - let hash = v1_clone_mbt.validate(Off, Verify).await.unwrap(); + let hash = v1_clone_mbt.open_and_validate(Off, Verify).await.unwrap(); allow_duplicates! { assert_snapshot!(hash, @"9ED9178D7025276336C783C2B54D6258"); } @@ -322,7 +322,7 @@ fn databases() -> Databases { }; let dmp = dump(&mut dif_empty_cn).await.unwrap(); assert_dump!(&dmp, "{typ}__dif_empty"); - let hash = dif_empty_mbt.validate(Off, Verify).await.unwrap(); + let hash = dif_empty_mbt.open_and_validate(Off, Verify).await.unwrap(); allow_duplicates! { assert_snapshot!(hash, @"D41D8CD98F00B204E9800998ECF8427E"); } @@ -484,7 +484,7 @@ async fn diff_and_patch( let (clone_mbt, mut clone_cn) = open!(diff_and_patch, "{prefix}__1"); copy!(databases.path(a_db, *dst_type), path(&clone_mbt)); apply_patch(path(&clone_mbt), path(&dif_mbt), false).await?; - let hash = clone_mbt.validate(Off, Verify).await?; + let hash = clone_mbt.open_and_validate(Off, Verify).await?; assert_eq!(hash, databases.hash(b_db, *dst_type)); let dmp = dump(&mut clone_cn).await?; pretty_assert_eq!(&dmp, expected_b); @@ -493,7 +493,7 @@ async fn diff_and_patch( let (clone_mbt, mut clone_cn) = open!(diff_and_patch, "{prefix}__2"); copy!(databases.path(b_db, *dst_type), path(&clone_mbt)); apply_patch(path(&clone_mbt), path(&dif_mbt), true).await?; - let hash = clone_mbt.validate(Off, Verify).await?; + let hash = clone_mbt.open_and_validate(Off, Verify).await?; assert_eq!(hash, databases.hash(b_db, *dst_type)); let dmp = dump(&mut clone_cn).await?; pretty_assert_eq!(&dmp, expected_b);