Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@nyurik
nyurik committed Apr 11, 2024
1 parent bb619b3 commit ab6f832
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 21 deletions.
6 changes: 5 additions & 1 deletion mbtiles/src/bin/mbtiles.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,9 @@ pub struct SharedCopyOpts {
/// Force copy operation, ignoring some warnings that otherwise would prevent the operation. Use with caution.
#[arg(short, long)]
force: bool,
/// Perform agg_hash validation on the original and destination files.
#[arg(long)]
validate: bool,
}

impl SharedCopyOpts {
Expand All @@ -184,6 +187,7 @@ impl SharedCopyOpts {
bbox: self.bbox,
skip_agg_tiles_hash: self.skip_agg_tiles_hash,
force: self.force,
validate: self.validate,
// Constants
dst_type: None, // Taken from dst_type_cli
}
Expand Down Expand Up @@ -262,7 +266,7 @@ async fn main_int() -> anyhow::Result<()> {
}
});
let mbt = Mbtiles::new(file.as_path())?;
mbt.validate(integrity_check, agg_hash).await?;
mbt.open_and_validate(integrity_check, agg_hash).await?;
}
Commands::Summary { file } => {
let mbt = Mbtiles::new(file.as_path())?;
Expand Down
27 changes: 21 additions & 6 deletions mbtiles/src/copier.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ use crate::errors::MbtResult;
use crate::queries::{
create_tiles_with_hash_view, detach_db, init_mbtiles_schema, is_empty_database,
};
use crate::AggHashType::Verify;
use crate::IntegrityCheckType::Quick;
use crate::MbtType::{Flat, FlatWithHash, Normalized};
use crate::{
invert_y_value, reset_db_settings, CopyType, MbtError, MbtType, MbtTypeCli, Mbtiles,
Expand Down Expand Up @@ -70,6 +72,8 @@ pub struct MbtilesCopier {
pub skip_agg_tiles_hash: bool,
/// Ignore some warnings and continue with the copying operation
pub force: bool,
/// Perform agg_hash validation on the original and destination files.
pub validate: bool,
}

#[derive(Clone, Debug)]
Expand Down Expand Up @@ -193,6 +197,9 @@ impl MbtileCopierInt {
let mut dif_conn = dif_mbt.open_readonly().await?;
let dif_info = dif_mbt.get_diff_info(&mut dif_conn).await?;
let dif_type = dif_info.mbt_type;
if self.options.validate {
dif_mbt.validate(&mut dif_conn, Quick, Verify).await?;
}
if is_creating_diff {
dif_mbt.validate_file_info(&dif_info, self.options.force)?;
} else {
Expand All @@ -203,11 +210,15 @@ impl MbtileCopierInt {
let src_mbt = &self.src_mbtiles;
let mut src_conn = src_mbt.open_readonly().await?;
let src_info = src_mbt.get_diff_info(&mut src_conn).await?;
if self.options.validate {
src_mbt.validate(&mut src_conn, Quick, Verify).await?;
}
let src_type = src_info.mbt_type;
src_mbt.validate_file_info(&src_info, self.options.force)?;
drop(src_conn);

let mut conn = self.dst_mbtiles.open_or_new().await?;
let dst_mbt = &self.dst_mbtiles;
let mut conn = dst_mbt.open_or_new().await?;
if !is_empty_database(&mut conn).await? {
return Err(MbtError::NonEmptyTargetFile(self.options.dst_file));
}
Expand All @@ -216,7 +227,7 @@ impl MbtileCopierInt {
dif_mbt.attach_to(&mut conn, "diffDb").await?;

let what = self.copy_text();
let dst_path = self.dst_mbtiles.filepath();
let dst_path = dst_mbt.filepath();
let dif_path = dif_mbt.filepath();
let dst_type = self.options.dst_type().unwrap_or(src_type);
if is_creating_diff {
Expand All @@ -243,23 +254,23 @@ impl MbtileCopierInt {

if is_creating_diff {
if let Some(hash) = src_info.agg_tiles_hash {
self.dst_mbtiles
dst_mbt
.set_metadata_value(&mut conn, AGG_TILES_HASH_BEFORE_APPLY, &hash)
.await?;
}
if let Some(hash) = dif_info.agg_tiles_hash {
self.dst_mbtiles
dst_mbt
.set_metadata_value(&mut conn, AGG_TILES_HASH_AFTER_APPLY, &hash)
.await?;
}
}

// TODO: perhaps disable all except --copy all when using with diffs, or else is not making much sense
if self.options.copy.copy_tiles() && !self.options.skip_agg_tiles_hash {
self.dst_mbtiles.update_agg_tiles_hash(&mut conn).await?;
dst_mbt.update_agg_tiles_hash(&mut conn).await?;

if !is_creating_diff {
let new_hash = self.dst_mbtiles.get_agg_tiles_hash(&mut conn).await?;
let new_hash = dst_mbt.get_agg_tiles_hash(&mut conn).await?;
match (dif_info.agg_tiles_hash_after_apply, new_hash) {
(Some(expected), Some(actual)) if expected != actual => {
let err = MbtError::AggHashMismatchAfterApply(
Expand All @@ -281,6 +292,10 @@ impl MbtileCopierInt {
detach_db(&mut conn, "diffDb").await?;
detach_db(&mut conn, "sourceDb").await?;

if self.options.validate {
dst_mbt.validate(&mut conn, Quick, Verify).await?;
}

Ok(conn)
}

Expand Down
24 changes: 18 additions & 6 deletions mbtiles/src/validation.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ pub enum AggHashType {
}

impl Mbtiles {
pub async fn validate(
pub async fn open_and_validate(
&self,
check_type: IntegrityCheckType,
agg_hash: AggHashType,
Expand All @@ -85,12 +85,24 @@ impl Mbtiles {
} else {
self.open_readonly().await?
};
self.check_integrity(&mut conn, check_type).await?;
self.check_tiles_type_validity(&mut conn).await?;
self.check_each_tile_hash(&mut conn).await?;
self.validate(&mut conn, check_type, agg_hash).await
}

pub async fn validate<T>(
&self,
conn: &mut T,
check_type: IntegrityCheckType,
agg_hash: AggHashType,
) -> MbtResult<String>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
self.check_integrity(&mut *conn, check_type).await?;
self.check_tiles_type_validity(&mut *conn).await?;
self.check_each_tile_hash(&mut *conn).await?;
match agg_hash {
AggHashType::Verify => self.check_agg_tiles_hashes(&mut conn).await,
AggHashType::Update => self.update_agg_tiles_hash(&mut conn).await,
AggHashType::Verify => self.check_agg_tiles_hashes(conn).await,
AggHashType::Update => self.update_agg_tiles_hash(conn).await,
AggHashType::Off => Ok(String::new()),
}
}
Expand Down
16 changes: 8 additions & 8 deletions mbtiles/tests/copy.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ fn databases() -> Databases {
copy!(result.path("empty_no_hash", mbt_typ), path(&empty_mbt));
let dmp = dump(&mut empty_cn).await.unwrap();
assert_dump!(&dmp, "{typ}__empty");
let hash = empty_mbt.validate(Off, Verify).await.unwrap();
let hash = empty_mbt.open_and_validate(Off, Verify).await.unwrap();
allow_duplicates! {
assert_snapshot!(hash, @"D41D8CD98F00B204E9800998ECF8427E");
}
Expand All @@ -265,7 +265,7 @@ fn databases() -> Databases {
copy!(result.path("v1_no_hash", mbt_typ), path(&v1_mbt));
let dmp = dump(&mut v1_cn).await.unwrap();
assert_dump!(&dmp, "{typ}__v1");
let hash = v1_mbt.validate(Off, Verify).await.unwrap();
let hash = v1_mbt.open_and_validate(Off, Verify).await.unwrap();
allow_duplicates! {
assert_snapshot!(hash, @"9ED9178D7025276336C783C2B54D6258");
}
Expand All @@ -276,7 +276,7 @@ fn databases() -> Databases {
new_file!(databases, mbt_typ, METADATA_V2, TILES_V2, "{typ}__v2");
let dmp = dump(&mut v2_cn).await.unwrap();
assert_dump!(&dmp, "{typ}__v2");
let hash = v2_mbt.validate(Off, Verify).await.unwrap();
let hash = v2_mbt.open_and_validate(Off, Verify).await.unwrap();
allow_duplicates! {
assert_snapshot!(hash, @"3BCDEE3F52407FF1315629298CB99133");
}
Expand All @@ -291,7 +291,7 @@ fn databases() -> Databases {
};
let dmp = dump(&mut dif_cn).await.unwrap();
assert_dump!(&dmp, "{typ}__dif");
let hash = dif_mbt.validate(Off, Verify).await.unwrap();
let hash = dif_mbt.open_and_validate(Off, Verify).await.unwrap();
allow_duplicates! {
assert_snapshot!(hash, @"B86122579EDCDD4C51F3910894FCC1A1");
}
Expand All @@ -300,7 +300,7 @@ fn databases() -> Databases {
// ----------------- v1_clone -----------------
let (v1_clone_mbt, v1_clone_cn) = open!(databases, "{typ}__v1-clone");
let dmp = copy_dump!(result.path("v1", mbt_typ), path(&v1_clone_mbt));
let hash = v1_clone_mbt.validate(Off, Verify).await.unwrap();
let hash = v1_clone_mbt.open_and_validate(Off, Verify).await.unwrap();
allow_duplicates! {
assert_snapshot!(hash, @"9ED9178D7025276336C783C2B54D6258");
}
Expand All @@ -322,7 +322,7 @@ fn databases() -> Databases {
};
let dmp = dump(&mut dif_empty_cn).await.unwrap();
assert_dump!(&dmp, "{typ}__dif_empty");
let hash = dif_empty_mbt.validate(Off, Verify).await.unwrap();
let hash = dif_empty_mbt.open_and_validate(Off, Verify).await.unwrap();
allow_duplicates! {
assert_snapshot!(hash, @"D41D8CD98F00B204E9800998ECF8427E");
}
Expand Down Expand Up @@ -484,7 +484,7 @@ async fn diff_and_patch(
let (clone_mbt, mut clone_cn) = open!(diff_and_patch, "{prefix}__1");
copy!(databases.path(a_db, *dst_type), path(&clone_mbt));
apply_patch(path(&clone_mbt), path(&dif_mbt), false).await?;
let hash = clone_mbt.validate(Off, Verify).await?;
let hash = clone_mbt.open_and_validate(Off, Verify).await?;
assert_eq!(hash, databases.hash(b_db, *dst_type));
let dmp = dump(&mut clone_cn).await?;
pretty_assert_eq!(&dmp, expected_b);
Expand All @@ -493,7 +493,7 @@ async fn diff_and_patch(
let (clone_mbt, mut clone_cn) = open!(diff_and_patch, "{prefix}__2");
copy!(databases.path(b_db, *dst_type), path(&clone_mbt));
apply_patch(path(&clone_mbt), path(&dif_mbt), true).await?;
let hash = clone_mbt.validate(Off, Verify).await?;
let hash = clone_mbt.open_and_validate(Off, Verify).await?;
assert_eq!(hash, databases.hash(b_db, *dst_type));
let dmp = dump(&mut clone_cn).await?;
pretty_assert_eq!(&dmp, expected_b);
Expand Down

0 comments on commit ab6f832

Please sign in to comment.