diff --git a/Cargo.lock b/Cargo.lock index 6b89872..b6ddb2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler" version = "1.0.2" @@ -8,18 +10,18 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ansi_term" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ "winapi", ] [[package]] name = "anyhow" -version = "1.0.40" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "atty" @@ -32,17 +34,11 @@ dependencies = [ "winapi", ] -[[package]] -name = "autocfg" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" - [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "byteorder" @@ -52,9 +48,12 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "cc" -version = "1.0.68" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" +checksum = "581f5dba903aac52ea3feb5ec4810848460ee833876f1f9b0fdeab1f19091574" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -64,9 +63,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "2.33.3" +version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", @@ -79,43 +78,50 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if", ] [[package]] name = "flate2" -version = "1.0.20" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ - "cfg-if", "crc32fast", - "libc", "miniz_oxide", ] [[package]] name = "hermit-abi" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" dependencies = [ "libc", ] [[package]] name = "libc" -version = "0.2.94" +version = "0.2.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" [[package]] name = "mapeditr" -version = "1.0.0" +version = "1.1.0" dependencies = [ "anyhow", "byteorder", @@ -124,6 +130,7 @@ dependencies = [ "memmem", "sqlite", "thiserror", + "zstd", ] [[package]] @@ -134,34 +141,33 @@ checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15" [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" dependencies = [ "adler", - "autocfg", ] [[package]] name = "pkg-config" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "proc-macro2" -version = "1.0.27" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] name = "quote" -version = "1.0.9" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" dependencies = [ "proc-macro2", ] @@ -204,13 +210,13 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.72" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -224,18 +230,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.25" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.25" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" dependencies = [ "proc-macro2", "quote", @@ -243,16 +249,16 @@ dependencies = [ ] [[package]] -name = "unicode-width" -version = "0.1.8" +name = "unicode-ident" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] -name = "unicode-xid" -version = "0.2.2" +name = "unicode-width" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "vec_map" @@ -281,3 +287,32 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +dependencies = [ + "cc", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index 1e219d2..7667116 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mapeditr" -version = "1.0.0" +version = "1.1.0" authors = ["random-geek (github.com/random-geek)"] edition = "2018" @@ -12,3 +12,4 @@ edition = "2018" memmem = "0.1" sqlite = "0.26" thiserror = "1" + zstd = "0.11" diff --git a/src/block_utils.rs b/src/block_utils.rs index 9de0a1e..a8d1ca4 100644 --- a/src/block_utils.rs +++ b/src/block_utils.rs @@ -24,8 +24,8 @@ pub fn merge_blocks( ) { assert!(block_parts_valid(&src_area, &dst_area)); - let src_nd = src_block.node_data.get_ref(); - let dst_nd = dst_block.node_data.get_mut(); + let src_nd = &src_block.node_data; + let dst_nd = &mut dst_block.node_data; let offset = dst_area.min - src_area.min; // Warning: diff can be negative! let diff = offset.x + offset.y * 16 + offset.z * 256; @@ -104,12 +104,11 @@ pub fn merge_metadata( /// Culls duplicate and unused IDs from the name-ID map and node data. pub fn clean_name_id_map(block: &mut MapBlock) { - let nd = block.node_data.get_mut(); let id_count = (block.nimap.get_max_id().unwrap() + 1) as usize; // Determine which IDs are used. let mut used = vec![false; id_count]; - for id in &nd.nodes { + for id in &block.node_data.nodes { used[*id as usize] = true; } @@ -136,7 +135,7 @@ pub fn clean_name_id_map(block: &mut MapBlock) { block.nimap = new_nimap; // Re-assign node IDs. - for id in &mut nd.nodes { + for id in &mut block.node_data.nodes { *id = map[*id as usize]; } } diff --git a/src/commands/clone.rs b/src/commands/clone.rs index 7f54f9f..5fdb0b3 100644 --- a/src/commands/clone.rs +++ b/src/commands/clone.rs @@ -3,8 +3,7 @@ use super::{Command, ArgResult, BLOCK_CACHE_SIZE}; use crate::{unwrap_or, opt_unwrap_or}; use crate::spatial::{Vec3, Area, MAP_LIMIT}; use crate::map_database::MapDatabase; -use crate::map_block::{MapBlock, MapBlockError, is_valid_generated, - NodeMetadataList, NodeMetadataListExt}; +use crate::map_block::{MapBlock, MapBlockError, is_valid_generated}; use crate::block_utils::{merge_blocks, merge_metadata, clean_name_id_map}; use crate::instance::{ArgType, InstBundle, InstArgs}; use crate::utils::{CacheMap, query_keys}; @@ -69,14 +68,11 @@ fn clone(inst: &mut InstBundle) { for dst_key in dst_keys { inst.status.inc_done(); - let (mut dst_block, mut dst_meta) = unwrap_or!( + let mut dst_block = unwrap_or!( opt_unwrap_or!( get_cached(&mut inst.db, &mut block_cache, dst_key), continue - ).and_then(|b| -> Result<_, MapBlockError> { - let m = NodeMetadataList::deserialize(b.metadata.get_ref())?; - Ok((b, m)) - }), + ), { inst.status.inc_failed(); continue; } ); @@ -90,14 +86,10 @@ fn clone(inst: &mut InstBundle) { continue; } let src_key = src_pos.to_block_key(); - let (src_block, src_meta) = opt_unwrap_or!( - || -> Option<_> { - let b = get_cached( - &mut inst.db, &mut block_cache, src_key)?.ok()?; - let m = NodeMetadataList::deserialize(b.metadata.get_ref()) - .ok()?; - Some((b, m)) - }(), + // Continue if a None or Some(Err) value is retrieved. + let src_block = opt_unwrap_or!( + get_cached(&mut inst.db, &mut block_cache, src_key) + .map(|res| res.ok()).flatten(), continue ); @@ -109,12 +101,11 @@ fn clone(inst: &mut InstBundle) { merge_blocks(&src_block, &mut dst_block, src_frag_rel, dst_frag_rel); - merge_metadata(&src_meta, &mut dst_meta, + merge_metadata(&src_block.metadata, &mut dst_block.metadata, src_frag_rel, dst_frag_rel); } clean_name_id_map(&mut dst_block); - *dst_block.metadata.get_mut() = dst_meta.serialize(dst_block.version); inst.db.set_block(dst_key, &dst_block.serialize()).unwrap(); } diff --git a/src/commands/delete_meta.rs b/src/commands/delete_meta.rs index 3153c2f..bc22236 100644 --- a/src/commands/delete_meta.rs +++ b/src/commands/delete_meta.rs @@ -3,7 +3,7 @@ use super::{Command, ArgResult}; use crate::unwrap_or; use crate::spatial::Vec3; use crate::instance::{ArgType, InstArgs, InstBundle}; -use crate::map_block::{MapBlock, NodeMetadataList, NodeMetadataListExt}; +use crate::map_block::MapBlock; use crate::utils::{query_keys, to_bytes, to_slice, fmt_big_num}; @@ -32,20 +32,15 @@ fn delete_metadata(inst: &mut InstBundle) { let mut block = unwrap_or!(MapBlock::deserialize(&data), { inst.status.inc_failed(); continue; }); - let node_data = block.node_data.get_ref(); let node_id = node.as_deref().and_then(|n| block.nimap.get_id(n)); if node.is_some() && node_id.is_none() { continue; // Block doesn't contain the required node. } - let mut meta = unwrap_or!( - NodeMetadataList::deserialize(block.metadata.get_ref()), - { inst.status.inc_failed(); continue; }); - let block_corner = Vec3::from_block_key(key) * 16; - let mut to_delete = Vec::with_capacity(meta.len()); + let mut to_delete = Vec::with_capacity(block.metadata.len()); - for (&idx, _) in &meta { + for (&idx, _) in &block.metadata { let abs_pos = Vec3::from_u16_key(idx) + block_corner; if let Some(a) = inst.args.area { @@ -54,7 +49,7 @@ fn delete_metadata(inst: &mut InstBundle) { } } if let Some(id) = node_id { - if node_data.nodes[idx as usize] != id { + if block.node_data.nodes[idx as usize] != id { continue; } } @@ -64,10 +59,9 @@ fn delete_metadata(inst: &mut InstBundle) { if !to_delete.is_empty() { for idx in &to_delete { - meta.remove(idx); + block.metadata.remove(idx); } count += to_delete.len() as u64; - *block.metadata.get_mut() = meta.serialize(block.version); inst.db.set_block(key, &block.serialize()).unwrap(); } } diff --git a/src/commands/delete_timers.rs b/src/commands/delete_timers.rs index d6c9225..2d69b09 100644 --- a/src/commands/delete_timers.rs +++ b/src/commands/delete_timers.rs @@ -26,7 +26,6 @@ fn delete_timers(inst: &mut InstBundle) { if node.is_some() && node_id.is_none() { continue; // Block doesn't contain the required node. } - let node_data = block.node_data.get_ref(); let block_corner = Vec3::from_block_key(key) * 16; let mut modified = false; @@ -42,7 +41,7 @@ fn delete_timers(inst: &mut InstBundle) { } } if let Some(id) = node_id { - if node_data.nodes[pos_idx as usize] != id { + if block.node_data.nodes[pos_idx as usize] != id { continue; } } diff --git a/src/commands/fill.rs b/src/commands/fill.rs index 3380409..e838b2a 100644 --- a/src/commands/fill.rs +++ b/src/commands/fill.rs @@ -9,11 +9,9 @@ use crate::utils::{query_keys, to_bytes, fmt_big_num}; fn fill_area(block: &mut MapBlock, id: u16, area: Area, invert: bool) { - let nd = block.node_data.get_mut(); - if invert { for i in InverseBlockIterator::new(area) { - nd.nodes[i] = id; + block.node_data.nodes[i] = id; } } else { for z in area.min.z ..= area.max.z { @@ -21,7 +19,7 @@ fn fill_area(block: &mut MapBlock, id: u16, area: Area, invert: bool) { for y in area.min.y ..= area.max.y { let zy_start = z_start + y * 16; for x in area.min.x ..= area.max.x { - nd.nodes[(zy_start + x) as usize] = id; + block.node_data.nodes[(zy_start + x) as usize] = id; } } } @@ -59,11 +57,10 @@ fn fill(inst: &mut InstBundle) { clean_name_id_map(&mut block); count += block_part.volume(); } else { // Fill entire block - let nd = block.node_data.get_mut(); - nd.nodes.fill(0); + block.node_data.nodes.fill(0); block.nimap.0.clear(); block.nimap.0.insert(0, node.to_vec()); - count += nd.nodes.len() as u64; + count += block.node_data.nodes.len() as u64; } inst.db.set_block(key, &block.serialize()).unwrap(); diff --git a/src/commands/overlay.rs b/src/commands/overlay.rs index 7134a52..ce9e14d 100644 --- a/src/commands/overlay.rs +++ b/src/commands/overlay.rs @@ -4,8 +4,7 @@ use crate::{unwrap_or, opt_unwrap_or}; use crate::spatial::{Vec3, Area, MAP_LIMIT}; use crate::instance::{ArgType, InstArgs, InstBundle}; use crate::map_database::MapDatabase; -use crate::map_block::{MapBlock, MapBlockError, is_valid_generated, - NodeMetadataList, NodeMetadataListExt}; +use crate::map_block::{MapBlock, MapBlockError, is_valid_generated}; use crate::block_utils::{merge_blocks, merge_metadata, clean_name_id_map}; use crate::utils::{query_keys, CacheMap}; @@ -73,10 +72,6 @@ fn overlay_no_offset(inst: &mut InstBundle) { let mut src_block = MapBlock::deserialize(&src_data)?; let mut dst_block = MapBlock::deserialize(&dst_data)?; - let mut src_meta = NodeMetadataList::deserialize( - &src_block.metadata.get_ref())?; - let mut dst_meta = NodeMetadataList::deserialize( - &dst_block.metadata.get_ref())?; let block_part = area.rel_block_overlap(pos).unwrap(); if invert { @@ -84,14 +79,14 @@ fn overlay_no_offset(inst: &mut InstBundle) { // overlay operations. merge_blocks(&dst_block, &mut src_block, block_part, block_part); - merge_metadata(&dst_meta, &mut src_meta, + merge_metadata(&dst_block.metadata, &mut src_block.metadata, block_part, block_part); clean_name_id_map(&mut src_block); db.set_block(key, &src_block.serialize()).unwrap(); } else { merge_blocks(&src_block, &mut dst_block, block_part, block_part); - merge_metadata(&src_meta, &mut dst_meta, + merge_metadata(&src_block.metadata, &mut dst_block.metadata, block_part, block_part); clean_name_id_map(&mut dst_block); db.set_block(key, &dst_block.serialize()).unwrap(); @@ -157,12 +152,8 @@ fn overlay_with_offset(inst: &mut InstBundle) { inst.db.get_block(dst_key).ok().filter(|d| is_valid_generated(d)), continue ); - let (mut dst_block, mut dst_meta) = unwrap_or!( - || -> Result<_, MapBlockError> { - let b = MapBlock::deserialize(&dst_data)?; - let m = NodeMetadataList::deserialize(b.metadata.get_ref())?; - Ok((b, m)) - }(), + let mut dst_block = unwrap_or!( + MapBlock::deserialize(&dst_data), { inst.status.inc_failed(); continue; } ); @@ -179,13 +170,8 @@ fn overlay_with_offset(inst: &mut InstBundle) { continue; } let src_key = src_pos.to_block_key(); - let (src_block, src_meta) = opt_unwrap_or!( - || -> Option<_> { - let b = get_cached(idb, &mut src_block_cache, src_key)?; - let m = NodeMetadataList::deserialize(b.metadata.get_ref()) - .ok()?; - Some((b, m)) - }(), + let src_block = opt_unwrap_or!( + get_cached(idb, &mut src_block_cache, src_key), continue ); @@ -197,12 +183,11 @@ fn overlay_with_offset(inst: &mut InstBundle) { merge_blocks(&src_block, &mut dst_block, src_frag_rel, dst_frag_rel); - merge_metadata(&src_meta, &mut dst_meta, + merge_metadata(&src_block.metadata, &mut dst_block.metadata, src_frag_rel, dst_frag_rel); } clean_name_id_map(&mut dst_block); - *dst_block.metadata.get_mut() = dst_meta.serialize(dst_block.version); inst.db.set_block(dst_key, &dst_block.serialize()).unwrap(); } diff --git a/src/commands/replace_in_inv.rs b/src/commands/replace_in_inv.rs index d74336e..7ebabb5 100644 --- a/src/commands/replace_in_inv.rs +++ b/src/commands/replace_in_inv.rs @@ -3,7 +3,7 @@ use super::{Command, ArgResult}; use crate::unwrap_or; use crate::spatial::Vec3; use crate::instance::{ArgType, InstArgs, InstBundle}; -use crate::map_block::{MapBlock, NodeMetadataList, NodeMetadataListExt}; +use crate::map_block::MapBlock; use crate::utils::{query_keys, to_bytes, fmt_big_num}; @@ -82,21 +82,16 @@ fn replace_in_inv(inst: &mut InstBundle) { let mut block = unwrap_or!(MapBlock::deserialize(&data), { inst.status.inc_failed(); continue; }); - let node_data = block.node_data.get_ref(); let node_ids: Vec<_> = nodes.iter() .filter_map(|n| block.nimap.get_id(n)).collect(); if !nodes.is_empty() && node_ids.is_empty() { continue; // Block doesn't contain any of the required nodes. } - let mut meta = unwrap_or!( - NodeMetadataList::deserialize(block.metadata.get_ref()), - { inst.status.inc_failed(); continue; }); - let block_corner = Vec3::from_block_key(key) * 16; let mut modified = false; - for (&idx, data) in &mut meta { + for (&idx, data) in &mut block.metadata { let pos = Vec3::from_u16_key(idx); let abs_pos = pos + block_corner; if let Some(a) = inst.args.area { @@ -105,7 +100,7 @@ fn replace_in_inv(inst: &mut InstBundle) { } } if !node_ids.is_empty() - && !node_ids.contains(&node_data.nodes[idx as usize]) + && !node_ids.contains(&block.node_data.nodes[idx as usize]) { continue; } @@ -120,7 +115,6 @@ fn replace_in_inv(inst: &mut InstBundle) { } if modified { - *block.metadata.get_mut() = meta.serialize(block.version); inst.db.set_block(key, &block.serialize()).unwrap(); } } diff --git a/src/commands/replace_nodes.rs b/src/commands/replace_nodes.rs index c96c466..48653eb 100644 --- a/src/commands/replace_nodes.rs +++ b/src/commands/replace_nodes.rs @@ -16,6 +16,7 @@ fn do_replace( invert: bool ) -> u64 { + let nodes = &mut block.node_data.nodes; let block_pos = Vec3::from_block_key(key); let mut replaced = 0; @@ -30,20 +31,19 @@ fn do_replace( Some(id) => (id, false), None => (block.nimap.get_max_id().unwrap() + 1, true) }; - let nd = block.node_data.get_mut(); if invert { for idx in InverseBlockIterator::new(node_area) { - if nd.nodes[idx] == old_id { - nd.nodes[idx] = new_id; + if nodes[idx] == old_id { + nodes[idx] = new_id; replaced += 1; } } } else { for pos in &node_area { let idx = (pos.x + 16 * (pos.y + 16 * pos.z)) as usize; - if nd.nodes[idx] == old_id { - nd.nodes[idx] = new_id; + if nodes[idx] == old_id { + nodes[idx] = new_id; replaced += 1; } } @@ -55,8 +55,8 @@ fn do_replace( } // If all instances of the old ID were replaced, remove the old ID. - if !nd.nodes.contains(&old_id) { - for node in &mut nd.nodes { + if !nodes.contains(&old_id) { + for node in nodes { *node -= (*node > old_id) as u16; } block.nimap.remove_shift(old_id); @@ -72,8 +72,7 @@ fn do_replace( new_id -= (new_id > old_id) as u16; // Map old node IDs to new node IDs. - let nd = block.node_data.get_mut(); - for id in &mut nd.nodes { + for id in nodes { *id = if *id == old_id { replaced += 1; new_id @@ -85,8 +84,7 @@ fn do_replace( // Block does not contain replacement node. // Simply replace the node name in the name-ID map. else { - let nd = block.node_data.get_ref(); - for id in &nd.nodes { + for id in nodes { replaced += (*id == old_id) as u64; } block.nimap.0.insert(old_id, new_node.to_vec()); diff --git a/src/commands/set_meta_var.rs b/src/commands/set_meta_var.rs index fd1c668..bea0c18 100644 --- a/src/commands/set_meta_var.rs +++ b/src/commands/set_meta_var.rs @@ -3,7 +3,7 @@ use super::{Command, ArgResult}; use crate::unwrap_or; use crate::spatial::Vec3; use crate::instance::{ArgType, InstArgs, InstBundle}; -use crate::map_block::{MapBlock, NodeMetadataList, NodeMetadataListExt}; +use crate::map_block::MapBlock; use crate::utils::{query_keys, to_bytes, fmt_big_num}; @@ -39,21 +39,16 @@ fn set_meta_var(inst: &mut InstBundle) { let mut block = unwrap_or!(MapBlock::deserialize(&data), { inst.status.inc_failed(); continue; }); - let node_data = block.node_data.get_ref(); let node_ids: Vec<_> = nodes.iter() .filter_map(|n| block.nimap.get_id(n)).collect(); if !nodes.is_empty() && node_ids.is_empty() { continue; // Block doesn't contain any of the required nodes. } - let mut meta = unwrap_or!( - NodeMetadataList::deserialize(block.metadata.get_ref()), - { inst.status.inc_failed(); continue; }); - let block_corner = Vec3::from_block_key(block_key) * 16; let mut modified = false; - for (&idx, data) in &mut meta { + for (&idx, data) in &mut block.metadata { let pos = Vec3::from_u16_key(idx); if let Some(a) = inst.args.area { @@ -62,7 +57,7 @@ fn set_meta_var(inst: &mut InstBundle) { } } if !node_ids.is_empty() - && !node_ids.contains(&node_data.nodes[idx as usize]) + && !node_ids.contains(&block.node_data.nodes[idx as usize]) { continue; } @@ -80,7 +75,6 @@ fn set_meta_var(inst: &mut InstBundle) { } if modified { - *block.metadata.get_mut() = meta.serialize(block.version); inst.db.set_block(block_key, &block.serialize()).unwrap(); } } diff --git a/src/commands/set_param2.rs b/src/commands/set_param2.rs index d181e83..1860701 100644 --- a/src/commands/set_param2.rs +++ b/src/commands/set_param2.rs @@ -10,7 +10,7 @@ use crate::utils::{query_keys, to_bytes, to_slice, fmt_big_num}; fn set_param2_partial(block: &mut MapBlock, area: Area, invert: bool, node_id: Option, val: u8) -> u64 { - let nd = block.node_data.get_mut(); + let nd = &mut block.node_data; let mut count = 0; if invert { @@ -74,7 +74,7 @@ fn set_param2(inst: &mut InstBundle) { continue; } - let nd = block.node_data.get_mut(); + let nd = &mut block.node_data; if let Some(area) = inst.args.area .filter(|a| a.contains_block(pos) != a.touches_block(pos)) { // Modify part of block diff --git a/src/map_block/compression.rs b/src/map_block/compression.rs deleted file mode 100644 index f815ee2..0000000 --- a/src/map_block/compression.rs +++ /dev/null @@ -1,76 +0,0 @@ -use super::*; - -use flate2::write::ZlibEncoder; -use flate2::read::ZlibDecoder; -use flate2::Compression; - - -pub trait Compress { - fn compress(&self, dst: &mut Cursor>); - fn decompress(src: &mut Cursor<&[u8]>) -> Result - where Self: std::marker::Sized; -} - - -impl Compress for Vec { - fn compress(&self, dst: &mut Cursor>) { - let mut encoder = ZlibEncoder::new(dst, Compression::default()); - encoder.write_all(self.as_ref()).unwrap(); - encoder.finish().unwrap(); - } - - fn decompress(src: &mut Cursor<&[u8]>) -> Result { - let start = src.position(); - - let mut decoder = ZlibDecoder::new(src); - let mut dst = Self::new(); - decoder.read_to_end(&mut dst).unwrap(); - let total_in = decoder.total_in(); - let src = decoder.into_inner(); - src.set_position(start + total_in); - - Ok(dst) - } -} - - -/// Stores some data and its zlib-compressed equivalent. -/// -/// The goal of this struct is to avoid recompressing data that was not -/// modified from its original compressed form. -#[derive(Clone, Debug)] -pub struct ZlibContainer { - compressed: Option>, - data: T -} - -impl ZlibContainer { - pub fn read(src: &mut Cursor<&[u8]>) -> Result { - let start = src.position() as usize; - let data = T::decompress(src)?; - let end = src.position() as usize; - Ok(Self { - compressed: Some(src.get_ref()[start..end].to_vec()), - data - }) - } - - pub fn write(&self, dst: &mut Cursor>) { - if let Some(compressed) = self.compressed.as_deref() { - dst.write_all(compressed).unwrap(); - } else { - self.data.compress(dst); - } - } - - #[inline] - pub fn get_ref(&self) -> &T { - &self.data - } - - #[inline] - pub fn get_mut(&mut self) -> &mut T { - self.compressed = None; - &mut self.data - } -} diff --git a/src/map_block/map_block.rs b/src/map_block/map_block.rs index 8d92fb5..a146a5f 100644 --- a/src/map_block/map_block.rs +++ b/src/map_block/map_block.rs @@ -1,22 +1,42 @@ use super::*; +use zstd; /* Supported mapblock versions: 25: In use from 0.4.2-rc1 until 0.4.15. 26: Only ever sent over the network, not saved. 27: Existed for around 3 months during 0.4.16 development. -28: In use since 0.4.16. +28: In use from 0.4.16 to 5.4.x. +29: In use since 5.5.0 (mapblocks are now compressed with zstd instead of zlib). */ const MIN_BLOCK_VER: u8 = 25; -const MAX_BLOCK_VER: u8 = 28; -const BLOCK_BUF_SIZE: usize = 2048; +const MAX_BLOCK_VER: u8 = 29; +const SERIALIZE_BUF_SIZE: usize = 2048; -pub fn is_valid_generated(data: &[u8]) -> bool { - data.len() >= 2 - && MIN_BLOCK_VER <= data[0] && data[0] <= MAX_BLOCK_VER - && data[1] & 0x08 == 0 +pub fn is_valid_generated(src: &[u8]) -> bool { + if src.len() < 2 { + return false; + } + + let mut crs = Cursor::new(src); + let version = crs.read_u8().unwrap(); + if version < MIN_BLOCK_VER || version > MAX_BLOCK_VER { + return false; + } + + let flags = if version >= 29 { + let mut dec = zstd::stream::Decoder::new(crs).unwrap(); + match dec.read_u8() { + Ok(f) => f, + Err(_) => return false + } + } else { + crs.read_u8().unwrap() + }; + + flags & 0x08 == 0 // Bit 3 set if block is not generated. } @@ -27,8 +47,8 @@ pub struct MapBlock { pub lighting_complete: u16, pub content_width: u8, pub params_width: u8, - pub node_data: ZlibContainer, - pub metadata: ZlibContainer>, + pub node_data: NodeData, + pub metadata: NodeMetadataList, pub static_objects: StaticObjectList, pub timestamp: u32, pub nimap: NameIdMap, @@ -37,42 +57,63 @@ pub struct MapBlock { impl MapBlock { pub fn deserialize(src: &[u8]) -> Result { - let mut data = Cursor::new(src); + let mut raw_crs = Cursor::new(src); - // Version - let version = data.read_u8()?; + let version = raw_crs.read_u8()?; if version < MIN_BLOCK_VER || version > MAX_BLOCK_VER { return Err(MapBlockError::InvalidBlockVersion); } - // Flags - let flags = data.read_u8()?; + // TODO: use thread_local buffer for decompressed data. + let decompressed; + let mut crs = + if version >= 29 { + decompressed = zstd::stream::decode_all(raw_crs)?; + Cursor::new(decompressed.as_slice()) + } else { raw_crs }; - // Light data + let flags = crs.read_u8()?; let lighting_complete = - if version >= 27 { data.read_u16::()? } + if version >= 27 { crs.read_u16::()? } else { 0xFFFF }; - // Content width/param width - let content_width = data.read_u8()?; - let params_width = data.read_u8()?; + let mut timestamp = 0; + let mut nimap = None; // Use Option to avoid re-initializing the map. + + if version >= 29 { // Timestamp/Name-ID map were moved in v29. + timestamp = crs.read_u32::()?; + nimap = Some(NameIdMap::deserialize(&mut crs)?); + } + + let content_width = crs.read_u8()?; + let params_width = crs.read_u8()?; // TODO: support content_width == 1? if content_width != 2 || params_width != 2 { return Err(MapBlockError::InvalidFeature); } - // Node data - let node_data = ZlibContainer::read(&mut data)?; - // Node metadata - let metadata = ZlibContainer::read(&mut data)?; - // Static objects - let static_objects = deserialize_objects(&mut data)?; - // Timestamp - let timestamp = data.read_u32::()?; - // Name-ID mappings - let nimap = NameIdMap::deserialize(&mut data)?; - // Node timers - let node_timers = deserialize_timers(&mut data)?; + let node_data = + if version >= 29 { + NodeData::deserialize(&mut crs)? + } else { + NodeData::decompress(&mut crs)? + }; + + let metadata = + if version >= 29 { + NodeMetadataList::deserialize(&mut crs)? + } else { + NodeMetadataList::decompress(&mut crs)? + }; + + let static_objects = deserialize_objects(&mut crs)?; + + if version < 29 { + timestamp = crs.read_u32::()?; + nimap = Some(NameIdMap::deserialize(&mut crs)?); + } + + let node_timers = deserialize_timers(&mut crs)?; Ok(Self { version, @@ -84,48 +125,54 @@ impl MapBlock { metadata, static_objects, timestamp, - nimap, + nimap: nimap.unwrap(), node_timers }) } pub fn serialize(&self) -> Vec { // TODO: Retain compression level used by Minetest? - // TODO: Use a bigger buffer (unsafe?) to reduce heap allocations. - let mut buf = Vec::with_capacity(BLOCK_BUF_SIZE); - let mut data = Cursor::new(buf); - assert!(MIN_BLOCK_VER <= self.version && self.version <= MAX_BLOCK_VER, "Invalid mapblock version."); - // Version - data.write_u8(self.version).unwrap(); - // Flags - data.write_u8(self.flags).unwrap(); + // TODO: Use a bigger buffer (unsafe?) to reduce heap allocations. + let mut buf = Vec::with_capacity(SERIALIZE_BUF_SIZE); + let mut crs = Cursor::new(buf); + crs.write_u8(self.version).unwrap(); + + if self.version >= 29 { + let mut enc = zstd::stream::Encoder::new(crs, 0).unwrap(); + + enc.write_u8(self.flags).unwrap(); + enc.write_u16::(self.lighting_complete).unwrap(); + enc.write_u32::(self.timestamp).unwrap(); + self.nimap.serialize(&mut enc); + enc.write_u8(self.content_width).unwrap(); + enc.write_u8(self.params_width).unwrap(); + self.node_data.serialize(&mut enc); + self.metadata.serialize(&mut enc, self.version); + serialize_objects(&self.static_objects, &mut enc); + serialize_timers(&self.node_timers, &mut enc); + + crs = enc.finish().unwrap(); + } else { // version <= 28 + crs.write_u8(self.flags).unwrap(); + + if self.version >= 27 { + crs.write_u16::(self.lighting_complete).unwrap(); + } - // Light data - if self.version >= 27 { - data.write_u16::(self.lighting_complete).unwrap(); + crs.write_u8(self.content_width).unwrap(); + crs.write_u8(self.params_width).unwrap(); + self.node_data.compress(&mut crs); + self.metadata.compress(&mut crs, self.version); + serialize_objects(&self.static_objects, &mut crs); + crs.write_u32::(self.timestamp).unwrap(); + self.nimap.serialize(&mut crs); + serialize_timers(&self.node_timers, &mut crs); } - // Content width/param width - data.write_u8(self.content_width).unwrap(); - data.write_u8(self.params_width).unwrap(); - - // Node data - self.node_data.write(&mut data); - // Node metadata - self.metadata.write(&mut data); - // Static objects - serialize_objects(&self.static_objects, &mut data); - // Timestamp - data.write_u32::(self.timestamp).unwrap(); - // Name-ID mappings - self.nimap.serialize(&mut data); - // Node timers - serialize_timers(&self.node_timers, &mut data); - - buf = data.into_inner(); + buf = crs.into_inner(); buf.shrink_to_fit(); buf } @@ -138,126 +185,201 @@ mod tests { use crate::spatial::Vec3; use std::path::Path; + #[test] + fn test_is_valid_generated() { + let ivg = is_valid_generated; + + // Too short + assert_eq!(ivg(b""), false); + assert_eq!(ivg(b"\x18"), false); // v24 + assert_eq!(ivg(b"\x1D"), false); // v29 + // Invalid version + assert_eq!(ivg(b"\x18\x00\x00\x00"), false); // v24 + assert_eq!(ivg(b"\x1E\x00\x00\x00"), false); // v30 + // v28, "not generated" flag set + assert_eq!(ivg(b"\x1C\x08"), false); + // v29, zstd compressed data is unreadable + assert_eq!(ivg(b"\x1D\x00\xFF"), false); + // v29, "not generated" flag set + assert_eq!(ivg(b"\x1D\x28\xB5\x2F\xFD\x00\x58\x19\x00\x00\x08\xFF\xFF"), false); + // v28, good + assert_eq!(ivg(b"\x1C\x00"), true); + // v29, good + assert_eq!(ivg(b"\x1D\x28\xB5\x2F\xFD\x00\x58\x19\x00\x00\x00\xFF\xFF"), true); + } + fn read_test_file(filename: &str) -> anyhow::Result> { let cargo_path = std::env::var("CARGO_MANIFEST_DIR")?; let path = Path::new(&cargo_path).join("testing").join(filename); Ok(std::fs::read(path)?) } + #[test] + fn test_mapblock_v29() { + // Original block positioned at (0, 0, 0). + let data1 = read_test_file("mapblock_v29.bin").unwrap(); + let block1 = MapBlock::deserialize(&data1).unwrap(); + // Re-serialize and re-deserialize to test serialization, since + // serialization results can vary. + let data2 = block1.serialize(); + let block2 = MapBlock::deserialize(&data2).unwrap(); + + for block in &[block1, block2] { + /* Ensure that all block data is correct. */ + assert_eq!(block.version, 29); + assert_eq!(block.flags, 0x03); + assert_eq!(block.lighting_complete, 0xFFFF); + assert_eq!(block.content_width, 2); + assert_eq!(block.params_width, 2); + + // Probe a few spots in the node data. + let nd = &block.node_data; + let timer_node_id = block.nimap.get_id(b"test_mod:timer").unwrap(); + let meta_node_id = block.nimap.get_id(b"test_mod:metadata").unwrap(); + let air_id = block.nimap.get_id(b"air").unwrap(); + assert_eq!(nd.nodes[0x000], timer_node_id); + assert!(nd.nodes[0x001..=0xFFE].iter().all(|&n| n == air_id)); + assert_eq!(nd.nodes[0xFFF], meta_node_id); + assert_eq!(nd.param2[0x000], 19); + assert_eq!(nd.param1[0x111], 0x0F); + assert!(nd.param2[0x001..=0xFFF].iter().all(|&n| n == 0)); + + assert_eq!(block.metadata.len(), 1); + let meta = &block.metadata[&4095]; + assert_eq!(meta.vars.len(), 2); + let formspec_var = meta.vars.get(&b"formspec".to_vec()).unwrap(); + assert_eq!(formspec_var.0.len(), 75); + assert_eq!(formspec_var.1, false); + let infotext_var = meta.vars.get(&b"infotext".to_vec()).unwrap(); + assert_eq!(infotext_var.0, b"Test Chest"); + assert_eq!(infotext_var.1, false); + assert_eq!(meta.inv.len(), 70); + + let obj1 = &block.static_objects[0]; + assert_eq!(obj1.obj_type, 7); + assert_eq!(obj1.f_pos, Vec3::new(1, 2, 2) * 10_000); + assert_eq!(obj1.data.len(), 75); + let obj2 = &block.static_objects[1]; + assert_eq!(obj2.obj_type, 7); + assert_eq!(obj2.f_pos, Vec3::new(8, 9, 12) * 10_000); + assert_eq!(obj2.data.len(), 62); + + assert_eq!(block.timestamp, 542); + + assert_eq!(block.nimap.0[&0], b"test_mod:timer"); + assert_eq!(block.nimap.0[&1], b"air"); + + assert_eq!(block.node_timers[0].pos, 0x000); + assert_eq!(block.node_timers[0].timeout, 1337); + assert_eq!(block.node_timers[0].elapsed, 399); + } + } + #[test] fn test_mapblock_v28() { // Original block positioned at (0, 0, 0). - let original_data = read_test_file("mapblock_v28.bin").unwrap(); - let block = MapBlock::deserialize(&original_data).unwrap(); - - /* Ensure that all block data is correct. */ - assert_eq!(block.version, 28); - assert_eq!(block.flags, 0x03); - assert_eq!(block.lighting_complete, 0xF1C4); - assert_eq!(block.content_width, 2); - assert_eq!(block.params_width, 2); - - // Probe a few spots in the node data. - let nd = block.node_data.get_ref(); - let test_node_id = block.nimap.get_id(b"test_mod:timer").unwrap(); - let air_id = block.nimap.get_id(b"air").unwrap(); - assert_eq!(nd.nodes[0x000], test_node_id); - assert!(nd.nodes[0x001..=0xFFE].iter().all(|&n| n == air_id)); - assert_eq!(nd.nodes[0xFFF], test_node_id); - assert_eq!(nd.param1[0x111], 0x0F); - assert_eq!(nd.param2[0x000], 4); - assert!(nd.param2[0x001..=0xFFE].iter().all(|&n| n == 0)); - assert_eq!(nd.param2[0xFFF], 16); - - assert_eq!(block.metadata.get_ref(), b"\x00"); - - let obj1 = &block.static_objects[0]; - assert_eq!(obj1.obj_type, 7); - assert_eq!(obj1.f_pos, Vec3::new(8, 9, 12) * 10_000); - assert_eq!(obj1.data.len(), 62); - let obj2 = &block.static_objects[1]; - assert_eq!(obj2.obj_type, 7); - assert_eq!(obj2.f_pos, Vec3::new(1, 2, 2) * 10_000); - assert_eq!(obj2.data.len(), 81); - - assert_eq!(block.timestamp, 2756); - - assert_eq!(block.nimap.0[&0], b"test_mod:timer"); - assert_eq!(block.nimap.0[&1], b"air"); - - assert_eq!(block.node_timers[0].pos, 0xFFF); - assert_eq!(block.node_timers[0].timeout, 1337); - assert_eq!(block.node_timers[0].elapsed, 600); - assert_eq!(block.node_timers[1].pos, 0x000); - assert_eq!(block.node_timers[1].timeout, 1337); - assert_eq!(block.node_timers[1].elapsed, 200); - - /* Test re-serialized data */ - let new_data = block.serialize(); - - // If zlib-compressed data is reused, it should be identical. - assert_eq!(new_data, original_data); - - // Triggering a data modification should change the compressed data, - // since Minetest and MapEditr use different compression levels. - let mut block2 = block.clone(); - block2.node_data.get_mut(); - assert_ne!(block2.serialize(), original_data); + let data1 = read_test_file("mapblock_v28.bin").unwrap(); + let block1 = MapBlock::deserialize(&data1).unwrap(); + let data2 = block1.serialize(); + let block2 = MapBlock::deserialize(&data2).unwrap(); + + for block in &[block1, block2] { + /* Ensure that all block data is correct. */ + assert_eq!(block.version, 28); + assert_eq!(block.flags, 0x03); + assert_eq!(block.lighting_complete, 0xF1C4); + assert_eq!(block.content_width, 2); + assert_eq!(block.params_width, 2); + + // Probe a few spots in the node data. + let nd = &block.node_data; + let test_node_id = block.nimap.get_id(b"test_mod:timer").unwrap(); + let air_id = block.nimap.get_id(b"air").unwrap(); + assert_eq!(nd.nodes[0x000], test_node_id); + assert!(nd.nodes[0x001..=0xFFE].iter().all(|&n| n == air_id)); + assert_eq!(nd.nodes[0xFFF], test_node_id); + assert_eq!(nd.param1[0x111], 0x0F); + assert_eq!(nd.param2[0x000], 4); + assert!(nd.param2[0x001..=0xFFE].iter().all(|&n| n == 0)); + assert_eq!(nd.param2[0xFFF], 16); + + assert!(block.metadata.is_empty()); + + let obj1 = &block.static_objects[0]; + assert_eq!(obj1.obj_type, 7); + assert_eq!(obj1.f_pos, Vec3::new(8, 9, 12) * 10_000); + assert_eq!(obj1.data.len(), 62); + let obj2 = &block.static_objects[1]; + assert_eq!(obj2.obj_type, 7); + assert_eq!(obj2.f_pos, Vec3::new(1, 2, 2) * 10_000); + assert_eq!(obj2.data.len(), 81); + + assert_eq!(block.timestamp, 2756); + + assert_eq!(block.nimap.0[&0], b"test_mod:timer"); + assert_eq!(block.nimap.0[&1], b"air"); + + assert_eq!(block.node_timers[0].pos, 0xFFF); + assert_eq!(block.node_timers[0].timeout, 1337); + assert_eq!(block.node_timers[0].elapsed, 600); + assert_eq!(block.node_timers[1].pos, 0x000); + assert_eq!(block.node_timers[1].timeout, 1337); + assert_eq!(block.node_timers[1].elapsed, 200); + } } #[test] fn test_mapblock_v25() { // Original block positioned at (-1, -1, -1). - let original_data = read_test_file("mapblock_v25.bin").unwrap(); - let block = MapBlock::deserialize(&original_data).unwrap(); - - /* Ensure that all block data is correct. */ - assert_eq!(block.version, 25); - assert_eq!(block.flags, 0x03); - assert_eq!(block.lighting_complete, 0xFFFF); - assert_eq!(block.content_width, 2); - assert_eq!(block.params_width, 2); - - let nd = block.node_data.get_ref(); - let test_node_id = block.nimap.get_id(b"test_mod:stone").unwrap(); - for z in &[0, 15] { - for y in &[0, 15] { - for x in &[0, 15] { - assert_eq!(nd.nodes[x + 16 * (y + 16 * z)], test_node_id); + let data1 = read_test_file("mapblock_v25.bin").unwrap(); + let block1 = MapBlock::deserialize(&data1).unwrap(); + let data2 = block1.serialize(); + let block2 = MapBlock::deserialize(&data2).unwrap(); + + for block in &[block1, block2] { + /* Ensure that all block data is correct. */ + assert_eq!(block.version, 25); + assert_eq!(block.flags, 0x03); + assert_eq!(block.lighting_complete, 0xFFFF); + assert_eq!(block.content_width, 2); + assert_eq!(block.params_width, 2); + + let nd = &block.node_data; + let test_node_id = block.nimap.get_id(b"test_mod:stone").unwrap(); + for z in &[0, 15] { + for y in &[0, 15] { + for x in &[0, 15] { + assert_eq!(nd.nodes[x + 16 * (y + 16 * z)], test_node_id); + } } } - } - assert_eq!(nd.nodes[0x001], block.nimap.get_id(b"air").unwrap()); - assert_eq!(nd.nodes[0x111], - block.nimap.get_id(b"test_mod:timer").unwrap()); - assert_eq!(nd.param2[0x111], 12); - - assert_eq!(block.metadata.get_ref(), b"\x00"); + assert_eq!(nd.nodes[0x001], block.nimap.get_id(b"air").unwrap()); + assert_eq!(nd.nodes[0x111], + block.nimap.get_id(b"test_mod:timer").unwrap()); + assert_eq!(nd.param2[0x111], 12); - let obj1 = &block.static_objects[0]; - assert_eq!(obj1.obj_type, 7); - assert_eq!(obj1.f_pos, Vec3::new(-5, -10, -15) * 10_000); - assert_eq!(obj1.data.len(), 72); + assert!(block.metadata.is_empty()); - let obj2 = &block.static_objects[1]; - assert_eq!(obj2.obj_type, 7); - assert_eq!(obj2.f_pos, Vec3::new(-14, -12, -10) * 10_000); - assert_eq!(obj2.data.len(), 54); + let obj1 = &block.static_objects[0]; + assert_eq!(obj1.obj_type, 7); + assert_eq!(obj1.f_pos, Vec3::new(-5, -10, -15) * 10_000); + assert_eq!(obj1.data.len(), 72); - assert_eq!(block.timestamp, 2529); + let obj2 = &block.static_objects[1]; + assert_eq!(obj2.obj_type, 7); + assert_eq!(obj2.f_pos, Vec3::new(-14, -12, -10) * 10_000); + assert_eq!(obj2.data.len(), 54); - assert_eq!(block.nimap.0[&0], b"test_mod:stone"); - assert_eq!(block.nimap.0[&1], b"air"); - assert_eq!(block.nimap.0[&2], b"test_mod:timer"); + assert_eq!(block.timestamp, 2529); - assert_eq!(block.node_timers[0].pos, 0x111); - assert_eq!(block.node_timers[0].timeout, 1337); - assert_eq!(block.node_timers[0].elapsed, 0); + assert_eq!(block.nimap.0[&0], b"test_mod:stone"); + assert_eq!(block.nimap.0[&1], b"air"); + assert_eq!(block.nimap.0[&2], b"test_mod:timer"); - /* Test re-serialized data */ - let mut block2 = block.clone(); - block2.node_data.get_mut(); - assert_ne!(block2.serialize(), original_data); + assert_eq!(block.node_timers[0].pos, 0x111); + assert_eq!(block.node_timers[0].timeout, 1337); + assert_eq!(block.node_timers[0].elapsed, 0); + } } #[test] @@ -279,7 +401,7 @@ mod tests { // Invalid versions check_error(|d| d[0x0] = 24, MapBlockError::InvalidBlockVersion); - check_error(|d| d[0x0] = 29, MapBlockError::InvalidBlockVersion); + check_error(|d| d[0x0] = 30, MapBlockError::InvalidBlockVersion); // Invalid content width check_error(|d| d[0x4] = 1, MapBlockError::InvalidFeature); // Invalid parameter width @@ -293,12 +415,12 @@ mod tests { { // Invalid node data size let mut block = MapBlock::deserialize(&data).unwrap(); - block.node_data.get_mut().param1.push(0); + block.node_data.param1.push(0); let new_data = block.serialize(); assert_eq!(MapBlock::deserialize(&new_data).unwrap_err(), MapBlockError::BadData); - block.node_data.get_mut().param1.truncate(4095); + block.node_data.param1.truncate(4095); let new_data = block.serialize(); assert_eq!(MapBlock::deserialize(&new_data).unwrap_err(), MapBlockError::BadData); diff --git a/src/map_block/metadata.rs b/src/map_block/metadata.rs index 5242980..84c5aac 100644 --- a/src/map_block/metadata.rs +++ b/src/map_block/metadata.rs @@ -4,6 +4,9 @@ use std::collections::{HashMap, BTreeMap}; use std::cmp::min; use memmem::{Searcher, TwoWaySearcher}; +use flate2::write::ZlibEncoder; +use flate2::read::ZlibDecoder; +use flate2::Compression; const END_STR: &[u8; 13] = b"EndInventory\n"; @@ -16,18 +19,18 @@ pub struct NodeMetadata { } impl NodeMetadata { - fn deserialize(data: &mut Cursor<&[u8]>, version: u8) + fn deserialize(src: &mut Cursor<&[u8]>, version: u8) -> Result { - let var_count = data.read_u32::()?; + let var_count = src.read_u32::()?; // Avoid allocating huge numbers of variables (bad data handling). let mut vars = HashMap::with_capacity(min(var_count as usize, 64)); for _ in 0..var_count { - let name = read_string16(data)?; - let val = read_string32(data)?; + let name = read_string16(src)?; + let val = read_string32(src)?; let private = if version >= 2 { - data.read_u8()? != 0 + src.read_u8()? != 0 } else { false }; vars.insert(name.clone(), (val, private)); } @@ -36,27 +39,27 @@ impl NodeMetadata { // This should be safe; EndInventory\n cannot appear in item metadata // since newlines are escaped. let end = end_finder - .search_in(&data.get_ref()[data.position() as usize ..]) + .search_in(&src.get_ref()[src.position() as usize ..]) .ok_or(MapBlockError::BadData)?; let mut inv = vec_with_len(end + END_STR.len()); - data.read_exact(&mut inv)?; + src.read_exact(&mut inv)?; Ok(Self { vars, inv }) } - fn serialize(&self, data: &mut Cursor>, version: u8) { - data.write_u32::(self.vars.len() as u32).unwrap(); + fn serialize(&self, dst: &mut T, version: u8) { + dst.write_u32::(self.vars.len() as u32).unwrap(); for (name, (val, private)) in &self.vars { - write_string16(data, name); - write_string32(data, &val); + write_string16(dst, name); + write_string32(dst, &val); if version >= 2 { - data.write_u8(*private as u8).unwrap(); + dst.write_u8(*private as u8).unwrap(); } } - data.write_all(&self.inv).unwrap(); + dst.write_all(&self.inv).unwrap(); } /// Return `true` if the metadata contains no variables or inventory lists. @@ -67,60 +70,83 @@ impl NodeMetadata { pub trait NodeMetadataListExt { - fn deserialize(src: &[u8]) -> Result + fn deserialize(src: &mut Cursor<&[u8]>) -> Result where Self: std::marker::Sized; - fn serialize(&self, block_version: u8) -> Vec; + fn decompress(src: &mut Cursor<&[u8]>) -> Result + where Self: std::marker::Sized; + fn serialize(&self, dst: &mut T, block_version: u8); + fn compress(&self, dst: &mut T, block_version: u8); } pub type NodeMetadataList = BTreeMap; impl NodeMetadataListExt for NodeMetadataList { - fn deserialize(src: &[u8]) -> Result { - let mut data = Cursor::new(src); - - let version = data.read_u8()?; + fn deserialize(src: &mut Cursor<&[u8]>) -> Result { + let version = src.read_u8()?; if version > 2 { return Err(MapBlockError::InvalidSubVersion) } let count = match version { 0 => 0, - _ => data.read_u16::()? + _ => src.read_u16::()? }; let mut list = BTreeMap::new(); for _ in 0..count { - let pos = data.read_u16::()?; - let meta = NodeMetadata::deserialize(&mut data, version)?; + let pos = src.read_u16::()?; + let meta = NodeMetadata::deserialize(src, version)?; list.insert(pos, meta); } Ok(list) } - fn serialize(&self, block_version: u8) -> Vec { - let buf = Vec::new(); - let mut data = Cursor::new(buf); - // Skip empty metadata when serializing. + fn decompress(src: &mut Cursor<&[u8]>) -> Result { + let start = src.position(); + let mut decoder = ZlibDecoder::new(src); + let mut buf = Vec::new(); + decoder.read_to_end(&mut buf)?; + + let mut cursor = Cursor::new(buf.as_slice()); + let metadata = Self::deserialize(&mut cursor)?; + + // Fail if there is leftover compressed data. + if decoder.read(&mut [0])? > 0 { + return Err(MapBlockError::BadData); + } + + let total_in = decoder.total_in(); + let src = decoder.into_inner(); + src.set_position(start + total_in); + + Ok(metadata) + } + + fn serialize(&self, dst: &mut T, block_version: u8) { let count = self.iter().filter(|&(_, m)| !m.is_empty()).count(); if count == 0 { - data.write_u8(0).unwrap(); + dst.write_u8(0).unwrap(); } else { let version = if block_version >= 28 { 2 } else { 1 }; - data.write_u8(version).unwrap(); - data.write_u16::(count as u16).unwrap(); + dst.write_u8(version).unwrap(); + dst.write_u16::(count as u16).unwrap(); for (&pos, meta) in self { if !meta.is_empty() { - data.write_u16::(pos).unwrap(); - meta.serialize(&mut data, version); + dst.write_u16::(pos).unwrap(); + meta.serialize(dst, version); } } } + } - data.into_inner() + fn compress(&self, dst: &mut T, block_version: u8) { + let mut encoder = ZlibEncoder::new(dst, Compression::default()); + self.serialize(&mut encoder, block_version); + encoder.finish().unwrap(); } } @@ -129,12 +155,24 @@ impl NodeMetadataListExt for NodeMetadataList { mod tests { use super::*; + fn meta_deserialize_slice(src: &[u8]) + -> Result + { + NodeMetadataList::deserialize(&mut Cursor::new(src)) + } + + fn meta_serialize_slice(meta: NodeMetadataList, version: u8) -> Vec { + let mut cursor = Cursor::new(Vec::new()); + meta.serialize(&mut cursor, version); + cursor.into_inner() + } + #[test] fn test_meta_serialize() { // Test empty metadata lists - assert!(NodeMetadataList::deserialize(b"\x00").unwrap().is_empty()); - for &ver in &[25, 28] { - assert_eq!(NodeMetadataList::new().serialize(ver), b"\x00"); + assert!(meta_deserialize_slice(b"\x00").unwrap().is_empty()); + for &ver in &[25, 29] { + assert_eq!(meta_serialize_slice(NodeMetadataList::new(), ver), b"\x00"); } // Test serialization/deserialization and filtering of empty metadata. @@ -161,17 +199,18 @@ mod tests { \x0f\xff\x00\x00\x00\x00List main 1\nWidth 0\nItem basenodes:dirt_\ with_grass 10\nEndInventoryList\nEndInventory\n"; - let meta_list = NodeMetadataList::deserialize(&meta_in[..]).unwrap(); + let meta_list = meta_deserialize_slice(&meta_in[..]).unwrap(); assert_eq!(meta_list.len(), 4); assert_eq!(meta_list[&0x010].vars[&b"formspec"[..]].1, false); assert_eq!(meta_list[&0xe21].vars[&b"secret"[..]].1, true); - assert_eq!(meta_list.serialize(28), meta_out); + // There is one empty variable which should be deleted. + assert_eq!(meta_serialize_slice(meta_list, 29), meta_out); // Test currently unsupported version let mut meta_future = meta_in.to_vec(); meta_future[0] = b'\x03'; assert_eq!( - NodeMetadataList::deserialize(&meta_future[..]).unwrap_err(), + meta_deserialize_slice(&meta_future[..]).unwrap_err(), MapBlockError::InvalidSubVersion ); @@ -185,10 +224,10 @@ mod tests { with_grass 10\nEndInventoryList\nEndInventory\n"; let meta_list_v1 = - NodeMetadataList::deserialize(&meta_v1[..]).unwrap(); + meta_deserialize_slice(&meta_v1[..]).unwrap(); assert_eq!(meta_list_v1.len(), 2); assert_eq!(meta_list_v1[&0x010].vars[&b"formspec"[..]].1, false); - assert_eq!(meta_list_v1.serialize(25), meta_v1); + assert_eq!(meta_serialize_slice(meta_list_v1, 25), meta_v1); // Test missing inventory let missing_inv = b"\x02\x00\x02\ @@ -196,7 +235,7 @@ mod tests { \x00\x03foo\x00\x00\x00\x03bar\x00 \x0f\xed\x00\x00\x00\x01\ \x00\x0dfake_inv_test\x00\x00\x00\x0cEndInventory\x00"; - assert_eq!(NodeMetadataList::deserialize(missing_inv).unwrap_err(), + assert_eq!(meta_deserialize_slice(missing_inv).unwrap_err(), MapBlockError::BadData); } } diff --git a/src/map_block/mod.rs b/src/map_block/mod.rs index 8accb67..df128c1 100644 --- a/src/map_block/mod.rs +++ b/src/map_block/mod.rs @@ -5,7 +5,6 @@ use std::convert::TryFrom; use byteorder::{ByteOrder, BigEndian, ReadBytesExt, WriteBytesExt}; mod map_block; -mod compression; mod node_data; mod metadata; mod static_object; @@ -13,8 +12,6 @@ mod node_timer; mod name_id_map; pub use map_block::{MapBlock, is_valid_generated}; -pub use compression::ZlibContainer; -use compression::Compress; pub use node_data::NodeData; pub use metadata::{NodeMetadataList, NodeMetadataListExt}; pub use static_object::{StaticObject, StaticObjectList, LuaEntityData}; @@ -79,14 +76,14 @@ fn read_string32(src: &mut Cursor<&[u8]>) -> Result, MapBlockError> { } -fn write_string16(dst: &mut Cursor>, data: &[u8]) { +fn write_string16(dst: &mut T, data: &[u8]) { let len = u16::try_from(data.len()).unwrap(); dst.write_u16::(len).unwrap(); dst.write(data).unwrap(); } -fn write_string32(dst: &mut Cursor>, data: &[u8]) { +fn write_string32(dst: &mut T, data: &[u8]) { let len = u32::try_from(data.len()).unwrap(); dst.write_u32::(len).unwrap(); dst.write(data).unwrap(); diff --git a/src/map_block/name_id_map.rs b/src/map_block/name_id_map.rs index cc11e5a..0403518 100644 --- a/src/map_block/name_id_map.rs +++ b/src/map_block/name_id_map.rs @@ -9,33 +9,31 @@ use std::collections::BTreeMap; pub struct NameIdMap(pub BTreeMap>); impl NameIdMap { - pub fn deserialize(data: &mut Cursor<&[u8]>) - -> Result - { - let version = data.read_u8()?; + pub fn deserialize(src: &mut Cursor<&[u8]>) -> Result { + let version = src.read_u8()?; if version != 0 { return Err(MapBlockError::InvalidSubVersion); } - let count = data.read_u16::()? as usize; + let count = src.read_u16::()? as usize; let mut map = BTreeMap::new(); for _ in 0..count { - let id = data.read_u16::()?; - let name = read_string16(data)?; + let id = src.read_u16::()?; + let name = read_string16(src)?; map.insert(id, name); } Ok(Self(map)) } - pub fn serialize(&self, out: &mut Cursor>) { - out.write_u8(0).unwrap(); - out.write_u16::(self.0.len() as u16).unwrap(); + pub fn serialize(&self, dst: &mut T) { + dst.write_u8(0).unwrap(); + dst.write_u16::(self.0.len() as u16).unwrap(); - for (id, name) in &self.0 { - out.write_u16::(*id).unwrap(); - write_string16(out, name); + for (&id, name) in &self.0 { + dst.write_u16::(id).unwrap(); + write_string16(dst, name); } } diff --git a/src/map_block/node_data.rs b/src/map_block/node_data.rs index b8d9e4e..00de807 100644 --- a/src/map_block/node_data.rs +++ b/src/map_block/node_data.rs @@ -16,46 +16,58 @@ pub struct NodeData { pub param2: Vec } -impl Compress for NodeData { - fn compress(&self, dst: &mut Cursor>) { - let mut encoder = ZlibEncoder::new(dst, Compression::default()); - +impl NodeData { + pub fn deserialize(src: &mut T) -> Result { let mut node_bytes = vec_with_len(NODE_COUNT * 2); - BigEndian::write_u16_into(&self.nodes, - &mut node_bytes[..NODE_COUNT * 2]); + src.read_exact(&mut node_bytes)?; + let mut nodes = vec_with_len(NODE_COUNT); + BigEndian::read_u16_into(&node_bytes, &mut nodes); - encoder.write_all(&node_bytes).unwrap(); - encoder.write_all(&self.param1).unwrap(); - encoder.write_all(&self.param2).unwrap(); - encoder.finish().unwrap(); + let mut param1 = vec_with_len(NODE_COUNT); + src.read_exact(&mut param1)?; + + let mut param2 = vec_with_len(NODE_COUNT); + src.read_exact(&mut param2)?; + + Ok(Self { + nodes, + param1, + param2 + }) } - fn decompress(src: &mut Cursor<&[u8]>) -> Result { + pub fn decompress(src: &mut Cursor<&[u8]>) -> Result { let start = src.position(); let mut decoder = ZlibDecoder::new(src); - let mut node_bytes = vec_with_len(NODE_COUNT * 2); - decoder.read_exact(&mut node_bytes)?; - let mut nodes = vec_with_len(NODE_COUNT); - BigEndian::read_u16_into(&node_bytes, &mut nodes); + let node_data = Self::deserialize(&mut decoder)?; - let mut param1 = vec_with_len(NODE_COUNT); - decoder.read_exact(&mut param1)?; - - let mut param2 = Vec::with_capacity(NODE_COUNT); - decoder.read_to_end(&mut param2)?; - if param2.len() != NODE_COUNT { - return Err(MapBlockError::BadData) + // Fail if there is leftover compressed data. + if decoder.read(&mut [0])? > 0 { + return Err(MapBlockError::BadData); } let total_in = decoder.total_in(); let src = decoder.into_inner(); src.set_position(start + total_in); - Ok(Self { - nodes, - param1, - param2 - }) + Ok(node_data) + } + + pub fn serialize(&self, dst: &mut T) { + // This allocation seems slow, but writing u16s iteratively is slower. + let mut node_bytes = vec_with_len(NODE_COUNT * 2); + BigEndian::write_u16_into(&self.nodes, + &mut node_bytes[..NODE_COUNT * 2]); + + dst.write_all(&node_bytes).unwrap(); + dst.write_all(&self.param1).unwrap(); + dst.write_all(&self.param2).unwrap(); + } + + pub fn compress(&self, dst: &mut T) { + let mut encoder = ZlibEncoder::new(dst, Compression::default()); + self.serialize(&mut encoder); + encoder.finish().unwrap(); } } diff --git a/src/map_block/node_timer.rs b/src/map_block/node_timer.rs index a0a1cb3..973a7ef 100644 --- a/src/map_block/node_timer.rs +++ b/src/map_block/node_timer.rs @@ -13,7 +13,7 @@ pub struct NodeTimer { pub type NodeTimerList = Vec; -pub fn deserialize_timers(src: &mut Cursor<&[u8]>) +pub fn deserialize_timers(src: &mut T) -> Result { let data_len = src.read_u8()?; @@ -36,7 +36,7 @@ pub fn deserialize_timers(src: &mut Cursor<&[u8]>) } -pub fn serialize_timers(timers: &NodeTimerList, dst: &mut Cursor>) { +pub fn serialize_timers(timers: &NodeTimerList, dst: &mut T) { dst.write_u8(10).unwrap(); dst.write_u16::(timers.len() as u16).unwrap(); diff --git a/src/map_block/static_object.rs b/src/map_block/static_object.rs index 34728cb..6f4795a 100644 --- a/src/map_block/static_object.rs +++ b/src/map_block/static_object.rs @@ -22,7 +22,7 @@ impl StaticObject { Ok(Self {obj_type, f_pos, data}) } - fn serialize(&self, dst: &mut Cursor>) { + fn serialize(&self, dst: &mut T) { dst.write_u8(self.obj_type).unwrap(); dst.write_i32::(self.f_pos.x).unwrap(); dst.write_i32::(self.f_pos.y).unwrap(); @@ -54,7 +54,7 @@ pub fn deserialize_objects(src: &mut Cursor<&[u8]>) } -pub fn serialize_objects(objects: &StaticObjectList, dst: &mut Cursor>) +pub fn serialize_objects(objects: &StaticObjectList, dst: &mut T) { dst.write_u8(0).unwrap(); dst.write_u16::(objects.len() as u16).unwrap(); diff --git a/src/utils.rs b/src/utils.rs index 866ec28..52b47ca 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -9,6 +9,8 @@ use crate::map_database::MapDatabase; use crate::spatial::{Area, Vec3}; +/// Note: For mapblock version 29 onwards, all block data is compressed, so +/// the `search_strs` argument is ignored. pub fn query_keys( db: &mut MapDatabase, status: &StatusServer, @@ -54,10 +56,16 @@ pub fn query_keys( } } } - if !data_searchers.is_empty() - && !data_searchers.iter().any(|s| s.search_in(&data).is_some()) - { // Data must match at least one search string. - continue; + if let Some(&block_version) = data.get(0) { + // If block version <= 28, data must match at least one search + // string. This optimization doesn't work for new mapblocks, as + // all block data is now compressed. + // TODO: Remove this legacy optimization? + if block_version <= 28 && !data_searchers.is_empty() + && !data_searchers.iter().any(|s| s.search_in(&data).is_some()) + { + continue; + } } keys.push(key); diff --git a/testing/mapblock_v29.bin b/testing/mapblock_v29.bin new file mode 100644 index 0000000..cf361b5 Binary files /dev/null and b/testing/mapblock_v29.bin differ diff --git a/testing/test_mod/init.lua b/testing/test_mod/init.lua index 528c318..d9e64a8 100644 --- a/testing/test_mod/init.lua +++ b/testing/test_mod/init.lua @@ -11,13 +11,34 @@ local colors = { minetest.register_node("test_mod:stone", { + description = "test_mod stone", drawtype = "normal", tiles = {"default_stone.png^[colorize:#3FFF3F:63"}, groups = {oddly_breakable_by_hand = 3}, }) +minetest.register_node("test_mod:metadata", { + description = "test_mod metadata", + drawtype = "normal", + tiles = {"default_stone.png^[colorize:#FF3F3F:63"}, + groups = {oddly_breakable_by_hand = 3}, + + on_construct = function(pos) + local meta = minetest.get_meta(pos) + meta:set_string("formspec", + "size[8,5]" .. + "list[current_name;main;0,0;1,1;]" .. + "list[current_player;main;0,1;8,4;]") + meta:set_string("infotext", "Test Chest") + local inv = meta:get_inventory() + inv:set_size("main", 1) + end, +}) + + minetest.register_node("test_mod:timer", { + description = "test_mod timer", drawtype = "nodebox", node_box = { type = "fixed",