From a451038ab358f6deb46f16f652ab653503420060 Mon Sep 17 00:00:00 2001 From: SW van Heerden Date: Tue, 14 Nov 2023 11:38:42 +0200 Subject: [PATCH 1/2] remove mutable mmr --- Cargo.lock | 20 -- base_layer/core/Cargo.toml | 2 +- base_layer/mmr/Cargo.toml | 3 - base_layer/mmr/README.md | 5 - base_layer/mmr/benches/smt_vs_mmr.rs | 64 ---- base_layer/mmr/src/functions.rs | 60 +--- base_layer/mmr/src/lib.rs | 27 +- base_layer/mmr/src/merkle_checkpoint.rs | 262 ----------------- base_layer/mmr/src/mmr_cache.rs | 243 ---------------- base_layer/mmr/src/mutable_mmr.rs | 273 ------------------ base_layer/mmr/src/mutable_mmr_leaf_nodes.rs | 155 ---------- base_layer/mmr/tests/support/mod.rs | 13 +- .../mmr/tests/tests/merkle_checkpoint.rs | 38 --- base_layer/mmr/tests/tests/mmr_cache.rs | 234 --------------- base_layer/mmr/tests/tests/mod.rs | 3 - base_layer/mmr/tests/tests/mutable_mmr.rs | 185 ------------ base_layer/mmr/tests/tests/pruned_mmr.rs | 25 +- 17 files changed, 9 insertions(+), 1603 deletions(-) delete mode 100644 base_layer/mmr/src/merkle_checkpoint.rs delete mode 100644 base_layer/mmr/src/mmr_cache.rs delete mode 100644 base_layer/mmr/src/mutable_mmr.rs delete mode 100644 base_layer/mmr/src/mutable_mmr_leaf_nodes.rs delete mode 100644 base_layer/mmr/tests/tests/merkle_checkpoint.rs delete mode 100644 base_layer/mmr/tests/tests/mmr_cache.rs delete mode 100644 base_layer/mmr/tests/tests/mutable_mmr.rs diff --git a/Cargo.lock b/Cargo.lock index f018c4d46b..be84aacba9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1160,25 +1160,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" -[[package]] -name = "croaring" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7539e9413f81db118bf258689110a9db0298c6089206817df2bd164fc7cd39e5" -dependencies = [ - "byteorder", - "croaring-sys", -] - -[[package]] -name = "croaring-sys" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e20dc23cebbac3da66d0eb2222341f6991ba779d96119f7bfa508f91784a495" -dependencies = [ - "cc", -] - [[package]] name = "crossbeam" version = "0.8.2" @@ -6017,7 +5998,6 @@ dependencies = [ "blake2", "borsh", "criterion 0.5.1", - "croaring", "digest 0.10.7", "log", "rand", diff --git a/base_layer/core/Cargo.toml b/base_layer/core/Cargo.toml index 105e51fc42..34c6d9d567 100644 --- a/base_layer/core/Cargo.toml +++ b/base_layer/core/Cargo.toml @@ -26,7 +26,7 @@ tari_comms_dht = { path = "../../comms/dht" } tari_comms_rpc_macros = { path = "../../comms/rpc_macros" } tari_crypto = { version = "0.19", features = ["borsh"] } tari_metrics = { path = "../../infrastructure/metrics", optional = true } -tari_mmr = { path = "../../base_layer/mmr", optional = true, features = ["native_bitmap"] } +tari_mmr = { path = "../../base_layer/mmr", optional = true} tari_p2p = { path = "../../base_layer/p2p" } tari_script = { path = "../../infrastructure/tari_script" } tari_service_framework = { path = "../service_framework" } diff --git a/base_layer/mmr/Cargo.toml b/base_layer/mmr/Cargo.toml index 45bb754986..a357a18d20 100644 --- a/base_layer/mmr/Cargo.toml +++ b/base_layer/mmr/Cargo.toml @@ -9,7 +9,6 @@ edition = "2018" [features] default = [] -native_bitmap = ["croaring"] [dependencies] tari_utilities = { version = "0.6" } @@ -20,7 +19,6 @@ borsh = "0.10" digest = "0.10" log = "0.4" serde = { version = "1.0", features = ["derive"] } -croaring = { version = "0.9", optional = true } [dev-dependencies] rand = "0.8" @@ -48,4 +46,3 @@ harness = false [[test]] name = "tari_mmr_integration_tests" path = "tests/mmr_integration_tests.rs" -required-features = ["native_bitmap"] diff --git a/base_layer/mmr/README.md b/base_layer/mmr/README.md index 0af3eda327..08d2f37257 100644 --- a/base_layer/mmr/README.md +++ b/base_layer/mmr/README.md @@ -11,8 +11,3 @@ at the bottom of the MMR is the hashes of the data. The MMR allows easy to add a tree. MMR always tries to have the largest possible single binary tree, so in effect it is possible to have more than one binary tree. Every time you have to get the merkle root (the single merkle proof of the whole MMR) you have the bag the peaks of the individual trees, or mountain peaks. - -### Features - -* `native_bitmap` - default feature, provides implementation for `MerkleCheckPoint`, `MmrCache`, `MutableMmr` via -using linked C library `croaring` (make sure to disable this feature when compiling to WASM). \ No newline at end of file diff --git a/base_layer/mmr/benches/smt_vs_mmr.rs b/base_layer/mmr/benches/smt_vs_mmr.rs index e89d2d7f3d..a14cd89580 100644 --- a/base_layer/mmr/benches/smt_vs_mmr.rs +++ b/base_layer/mmr/benches/smt_vs_mmr.rs @@ -4,25 +4,8 @@ use std::convert::TryFrom; use blake2::Blake2b; -#[cfg(feature = "native_bitmap")] -use croaring::Bitmap; use digest::consts::U32; -#[cfg(feature = "native_bitmap")] -use tari_crypto::hash_domain; -#[cfg(feature = "native_bitmap")] -use tari_crypto::hashing::DomainSeparatedHasher; -#[cfg(feature = "native_bitmap")] -use tari_crypto::tari_utilities::hex::Hex; use tari_mmr::sparse_merkle_tree::{NodeKey, SparseMerkleTree, ValueHash}; -#[cfg(feature = "native_bitmap")] -use tari_mmr::{Hash, MutableMmr}; - -#[cfg(feature = "native_bitmap")] -hash_domain!(MmrBenchTestHashDomain, "com.tari.base_layer.mmr.benches", 1); -#[cfg(feature = "native_bitmap")] -pub type MmrTestHasherBlake256 = DomainSeparatedHasher, MmrBenchTestHashDomain>; -#[cfg(feature = "native_bitmap")] -pub type TestMmr = MutableMmr>; fn random_key() -> NodeKey { let key = rand::random::<[u8; 32]>(); @@ -49,20 +32,6 @@ fn delete_from_smt(keys: &[NodeKey], tree: &mut SparseMerkleTree>) }); } -#[cfg(feature = "native_bitmap")] -fn insert_into_mmr(keys: &[Vec], mmr: &mut TestMmr) { - keys.iter().for_each(|key| { - mmr.push(key.clone()).unwrap(); - }); -} - -#[cfg(feature = "native_bitmap")] -fn delete_from_mmr(start: u32, n: u32, mmr: &mut TestMmr) { - (start..start + n).for_each(|i| { - mmr.delete(i); - }); -} - fn time_function(header: &str, f: impl FnOnce()) -> std::time::Duration { println!("Starting: {header}"); let now = std::time::Instant::now(); @@ -101,37 +70,4 @@ fn main() { let hash = tree.hash(); println!("Tree size: {size}. Root hash: {hash:x}"); }); - #[cfg(feature = "native_bitmap")] - { - let mut mmr = TestMmr::new(Vec::default(), Bitmap::default()).unwrap(); - let keys = keys.into_iter().map(|k| k.as_slice().to_vec()).collect::>(); - time_function(&format!("MMR: Inserting {size} keys"), || { - insert_into_mmr(&keys, &mut mmr); - }); - time_function("MMR: Calculating root hash", || { - let size = mmr.len().unwrap(); - let hash = mmr.get_merkle_root().unwrap(); - println!("Tree size: {size}. Root hash: {}", hash.to_hex()); - }); - time_function(&format!("MMR: Deleting {half_size} keys"), || { - delete_from_mmr(0, u32::try_from(half_size).unwrap(), &mut mmr); - }); - time_function("MMR: Calculating root hash", || { - let size = mmr.len().unwrap(); - let hash = mmr.get_merkle_root().unwrap(); - println!("Tree size: {size}. Root hash: {}", hash.to_hex()); - }); - time_function(&format!("MMR: Deleting another {half_size} keys"), || { - delete_from_mmr( - u32::try_from(half_size).unwrap(), - u32::try_from(half_size).unwrap(), - &mut mmr, - ); - }); - time_function("MMR: Calculating root hash", || { - let size = mmr.len().unwrap(); - let hash = mmr.get_merkle_root().unwrap(); - println!("Tree size: {size}. Root hash: {}", hash.to_hex()); - }); - } } diff --git a/base_layer/mmr/src/functions.rs b/base_layer/mmr/src/functions.rs index 33ebe42432..aeb3fd6e58 100644 --- a/base_layer/mmr/src/functions.rs +++ b/base_layer/mmr/src/functions.rs @@ -20,20 +20,14 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -use crate::{ - error::MerkleMountainRangeError, - pruned_hashset::PrunedHashSet, - ArrayLike, - Hash, - MerkleMountainRange, - MutableMmr, -}; -use digest::Digest; use std::{convert::TryFrom, marker::PhantomData}; + +use digest::Digest; use tari_common::DomainDigest; +use crate::{error::MerkleMountainRangeError, pruned_hashset::PrunedHashSet, ArrayLike, Hash, MerkleMountainRange}; + pub type PrunedMmr = MerkleMountainRange; -pub type PrunedMutableMmr = MutableMmr; /// Create a pruned Merkle Mountain Range from the provided MMR. Pruning entails throwing all the hashes of the /// pruned MMR away, except for the current peaks. A new MMR instance is returned that allows you to continue @@ -52,52 +46,6 @@ where }) } -/// A convenience function in the same vein as [prune_mmr], but applied to `MutableMmr` instances. -pub fn prune_mutable_mmr(mmr: &MutableMmr) -> Result, MerkleMountainRangeError> -where - D: Digest + DomainDigest, - B: ArrayLike, -{ - let backend = PrunedHashSet::try_from(&mmr.mmr)?; - Ok(MutableMmr { - mmr: MerkleMountainRange::new(backend), - deleted: mmr.deleted.clone(), - size: mmr.size, - }) -} - -/// `calculate_mmr_root`` takes an MMR instance and efficiently calculates the new MMR root by applying the given -/// additions to calculate a new MMR root without changing the original MMR. -/// -/// This is done by creating a memory-backed sparse (pruned) copy of the original MMR, applying the changes and then -/// calculating a new root. -/// -/// # Parameters -/// * `src`: A reference to the original MMR -/// * `additions`: A vector of leaf node hashes to append to the MMR -/// * `deletions`: A vector of leaf node _indices_ that will be marked as deleted. -/// -/// # Returns -/// The new MMR root as a result of applying the given changes -pub fn calculate_pruned_mmr_root( - src: &MutableMmr, - additions: Vec, - deletions: Vec, -) -> Result -where - D: Digest + DomainDigest, - B: ArrayLike, -{ - let mut pruned_mmr = prune_mutable_mmr(src)?; - for hash in additions { - pruned_mmr.push(hash)?; - } - for index in deletions { - pruned_mmr.delete(index); - } - pruned_mmr.get_merkle_root() -} - pub fn calculate_mmr_root( src: &MerkleMountainRange, additions: Vec, diff --git a/base_layer/mmr/src/lib.rs b/base_layer/mmr/src/lib.rs index 7e897324fd..db0054bf33 100644 --- a/base_layer/mmr/src/lib.rs +++ b/base_layer/mmr/src/lib.rs @@ -165,29 +165,4 @@ pub use merkle_mountain_range::MerkleMountainRange; /// A data structure for proving a hash inclusion in an MMR pub use merkle_proof::{MerkleProof, MerkleProofError}; -macro_rules! if_native_bitmap { - ($($item:item)*) => { - $( - #[cfg(feature = "native_bitmap")] - $item - )* - } -} - -if_native_bitmap! { - mod merkle_checkpoint; - mod mmr_cache; - mod mutable_mmr; - mod mutable_mmr_leaf_nodes; - pub mod functions; - - /// A Merkle checkpoint contains the set of hash additions and deletion indices. - pub use merkle_checkpoint::MerkleCheckPoint; - /// The MMR cache is used to calculate Merkle and Merklish roots based on the state of the set of shared - /// checkpoints. - pub use mmr_cache::{MmrCache, MmrCacheConfig}; - /// An append-only Merkle Mountain range (MMR) data structure that allows deletion of existing leaf nodes. - pub use mutable_mmr::MutableMmr; - /// A data structure for storing all the data required to restore the state of an MMR. - pub use mutable_mmr_leaf_nodes::MutableMmrLeafNodes; -} +pub mod functions; diff --git a/base_layer/mmr/src/merkle_checkpoint.rs b/base_layer/mmr/src/merkle_checkpoint.rs deleted file mode 100644 index dc75c100fe..0000000000 --- a/base_layer/mmr/src/merkle_checkpoint.rs +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright 2019. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use std::{convert::TryFrom, fmt, hash::Hasher}; - -use croaring::Bitmap; -use digest::Digest; -use serde::{ - de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor}, - ser::{Serialize, SerializeStruct, Serializer}, -}; -use tari_common::DomainDigest; - -use crate::{backend::ArrayLike, error::MerkleMountainRangeError, mutable_mmr::MutableMmr, Hash}; - -#[derive(Debug, Clone, PartialEq)] -pub struct MerkleCheckPoint { - nodes_added: Vec, - nodes_deleted: Bitmap, - prev_accumulated_nodes_added_count: u32, -} - -impl MerkleCheckPoint { - pub fn new( - nodes_added: Vec, - nodes_deleted: Bitmap, - prev_accumulated_nodes_added_count: u32, - ) -> MerkleCheckPoint { - MerkleCheckPoint { - nodes_added, - nodes_deleted, - prev_accumulated_nodes_added_count, - } - } - - /// Apply this checkpoint to the MMR provided. Take care: The `deleted` set is not compressed after returning - /// from here. - pub fn apply(&self, mmr: &mut MutableMmr) -> Result<(), MerkleMountainRangeError> - where - D: Digest + DomainDigest, - B2: ArrayLike, - { - for node in &self.nodes_added { - mmr.push(node.clone())?; - } - mmr.deleted.or_inplace(&self.nodes_deleted); - Ok(()) - } - - /// Resets the current MerkleCheckpoint. The accumulated_nodes_added_count is set to the current `MerkleCheckpoint`s - /// count. - pub fn reset(&mut self) { - self.prev_accumulated_nodes_added_count = self.accumulated_nodes_added_count(); - self.nodes_added.clear(); - self.nodes_deleted = Bitmap::create(); - } - - /// Resets the current MerkleCheckpoint. The accumulated_nodes_added_count is set to the given `MerkleCheckpoint`s - /// count. - pub fn reset_to(&mut self, checkpoint: &Self) { - self.prev_accumulated_nodes_added_count = checkpoint.accumulated_nodes_added_count(); - self.nodes_added.clear(); - self.nodes_deleted = Bitmap::create(); - } - - /// Add a hash to the set of nodes added. - pub fn push_addition(&mut self, hash: Hash) { - self.nodes_added.push(hash); - } - - /// Add a a deleted index to the set of deleted nodes. - pub fn push_deletion(&mut self, leaf_index: u32) { - self.nodes_deleted.add(leaf_index); - } - - /// Return a reference to the hashes of the nodes added in the checkpoint - pub fn nodes_added(&self) -> &Vec { - &self.nodes_added - } - - /// Return a reference to the roaring bitmap of nodes that were deleted in this checkpoint - pub fn nodes_deleted(&self) -> &Bitmap { - &self.nodes_deleted - } - - /// Return the the total accumulated added node count including this checkpoint - pub fn accumulated_nodes_added_count(&self) -> u32 { - self.prev_accumulated_nodes_added_count + u32::try_from(self.nodes_added.len()).unwrap() - } - - /// Merge the provided Merkle checkpoint into the current checkpoint. - pub fn append(&mut self, mut cp: MerkleCheckPoint) { - self.nodes_added.append(&mut cp.nodes_added); - self.nodes_deleted.or_inplace(&cp.nodes_deleted); - } - - /// Break a checkpoint up into its constituent parts - pub fn into_parts(self) -> (Vec, Bitmap) { - (self.nodes_added, self.nodes_deleted) - } -} - -impl Default for MerkleCheckPoint { - fn default() -> Self { - Self { - nodes_added: Default::default(), - nodes_deleted: Bitmap::create(), - prev_accumulated_nodes_added_count: Default::default(), - } - } -} - -impl Eq for MerkleCheckPoint {} - -#[allow(clippy::derived_hash_with_manual_eq)] -impl std::hash::Hash for MerkleCheckPoint { - fn hash(&self, state: &mut H) { - self.nodes_added.hash(state); - self.nodes_deleted.to_vec().hash(state); - self.prev_accumulated_nodes_added_count.hash(state); - } -} - -impl Serialize for MerkleCheckPoint { - fn serialize(&self, serializer: S) -> Result - where S: Serializer { - let mut state = serializer.serialize_struct("MerkleCheckPoint", 3)?; - state.serialize_field("nodes_added", &self.nodes_added)?; - state.serialize_field("nodes_deleted", &self.nodes_deleted.serialize())?; - state.serialize_field( - "prev_accumulated_nodes_added_count", - &self.prev_accumulated_nodes_added_count, - )?; - state.end() - } -} - -impl<'de> Deserialize<'de> for MerkleCheckPoint { - fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { - enum Field { - NodesAdded, - NodesDeleted, - PrevAccumulatedNodesAddedCount, - } - - impl<'de> Deserialize<'de> for Field { - fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { - struct FieldVisitor; - - impl<'de> Visitor<'de> for FieldVisitor { - type Value = Field; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("`nodes_added`, `nodes_deleted` or `prev_accumulated_nodes_added_count`") - } - - fn visit_str(self, value: &str) -> Result - where E: de::Error { - match value { - "nodes_added" => Ok(Field::NodesAdded), - "nodes_deleted" => Ok(Field::NodesDeleted), - "prev_accumulated_nodes_added_count" => Ok(Field::PrevAccumulatedNodesAddedCount), - _ => Err(de::Error::unknown_field(value, FIELDS)), - } - } - } - - deserializer.deserialize_identifier(FieldVisitor) - } - } - - struct MerkleCheckPointVisitor; - - impl<'de> Visitor<'de> for MerkleCheckPointVisitor { - type Value = MerkleCheckPoint; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("struct MerkleCheckPoint") - } - - fn visit_seq(self, mut seq: V) -> Result - where V: SeqAccess<'de> { - let nodes_added = seq.next_element()?.ok_or_else(|| de::Error::invalid_length(0, &self))?; - let nodes_deleted_buf: Vec = - seq.next_element()?.ok_or_else(|| de::Error::invalid_length(1, &self))?; - let nodes_deleted = Bitmap::deserialize(&nodes_deleted_buf); - let prev_accumulated_nodes_added_count = - seq.next_element()?.ok_or_else(|| de::Error::invalid_length(2, &self))?; - Ok(MerkleCheckPoint::new( - nodes_added, - nodes_deleted, - prev_accumulated_nodes_added_count, - )) - } - - fn visit_map(self, mut map: V) -> Result - where V: MapAccess<'de> { - let mut nodes_added = None; - let mut nodes_deleted = None; - let mut prev_accumulated_nodes_added_count = None; - while let Some(key) = map.next_key()? { - match key { - Field::NodesAdded => { - if nodes_added.is_some() { - return Err(de::Error::duplicate_field("nodes_added")); - } - nodes_added = Some(map.next_value()?); - }, - Field::NodesDeleted => { - if nodes_deleted.is_some() { - return Err(de::Error::duplicate_field("nodes_deleted")); - } - let nodes_deleted_buf: Vec = map.next_value()?; - nodes_deleted = Some(Bitmap::deserialize(&nodes_deleted_buf)); - }, - Field::PrevAccumulatedNodesAddedCount => { - if prev_accumulated_nodes_added_count.is_some() { - return Err(de::Error::duplicate_field("nodes_deleted")); - } - - prev_accumulated_nodes_added_count = Some(map.next_value()?); - }, - } - } - - let nodes_added = nodes_added.ok_or_else(|| de::Error::missing_field("nodes_added"))?; - let nodes_deleted = nodes_deleted.ok_or_else(|| de::Error::missing_field("nodes_deleted"))?; - let prev_accumulated_nodes_added_count = prev_accumulated_nodes_added_count - .ok_or_else(|| de::Error::missing_field("accumulated_nodes_added_count"))?; - Ok(MerkleCheckPoint::new( - nodes_added, - nodes_deleted, - prev_accumulated_nodes_added_count, - )) - } - } - - const FIELDS: &[&str] = &["nodes_added", "nodes_deleted", "prev_accumulated_nodes_added_count"]; - deserializer.deserialize_struct("MerkleCheckPoint", FIELDS, MerkleCheckPointVisitor) - } -} diff --git a/base_layer/mmr/src/mmr_cache.rs b/base_layer/mmr/src/mmr_cache.rs deleted file mode 100644 index f21d67add7..0000000000 --- a/base_layer/mmr/src/mmr_cache.rs +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2019. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use std::ops::Deref; - -use croaring::Bitmap; -use digest::Digest; -use tari_common::DomainDigest; - -use crate::{ - backend::ArrayLike, - common::LeafIndex, - error::MerkleMountainRangeError, - functions::{prune_mutable_mmr, PrunedMutableMmr}, - merkle_checkpoint::MerkleCheckPoint, - Hash, - MutableMmr, -}; - -/// Configuration for the MmrCache. -#[derive(Debug, Clone, Copy)] -pub struct MmrCacheConfig { - /// The rewind_hist_len specifies the point in history upto where the MMR can be efficiently rewound before the - /// base mmr needs to be reconstructed. - pub rewind_hist_len: usize, -} - -impl Default for MmrCacheConfig { - fn default() -> Self { - Self { rewind_hist_len: 100 } - } -} - -/// The MMR cache is used to calculate Merkle and Merklish roots based on the state of the set of shared checkpoints. It -/// can efficiently create an updated cache state when small checkpoint rewinds were detected or the checkpoint state -/// has been expanded. -#[derive(Debug)] -pub struct MmrCache -where - D: Digest + DomainDigest, - BaseBackend: ArrayLike, -{ - // The last checkpoint index applied to the base MMR. - base_cp_index: usize, - // One more than the last checkpoint index applied to the current MMR. - curr_cp_index: usize, - // The base MMR is the anchor point of the mmr cache. A rewind can start at this state if the checkpoint tip is - // beyond the base checkpoint index. It will have to rebuild the base MMR if the checkpoint tip index is less - // than the base MMR index. - base_mmr: MutableMmr, - // The current mmr represents the latest mmr with all checkpoints applied. - pub curr_mmr: PrunedMutableMmr, - // Access to the checkpoint set. - checkpoints: CpBackend, - // Configuration for the MMR cache. - config: MmrCacheConfig, -} - -impl MmrCache -where - D: Digest + DomainDigest, - BaseBackend: ArrayLike, - CpBackend: ArrayLike, -{ - /// Creates a new MMR cache with access to the provided set of shared checkpoints. - pub fn new( - base_mmr: BaseBackend, - checkpoints: CpBackend, - config: MmrCacheConfig, - ) -> Result, MerkleMountainRangeError> { - let base_mmr = MutableMmr::new(base_mmr, Bitmap::create())?; - let curr_mmr = prune_mutable_mmr::(&base_mmr)?; - let mut mmr_cache = MmrCache { - base_cp_index: 0, - curr_cp_index: 0, - base_mmr, - curr_mmr, - checkpoints, - config, - }; - mmr_cache.reset()?; - Ok(mmr_cache) - } - - // Calculate the base checkpoint index based on the rewind history length and the number of checkpoints. - fn calculate_base_cp_index(&mut self) -> Result { - let cp_count = self - .checkpoints - .len() - .map_err(|e| MerkleMountainRangeError::BackendError(e.to_string()))?; - if cp_count > self.config.rewind_hist_len { - return Ok(cp_count - self.config.rewind_hist_len); - } - Ok(0) - } - - // Reconstruct the base MMR using the shared checkpoints. The base MMR contains the state from the the first - // checkpoint to the checkpoint tip minus the minimum history length. - fn create_base_mmr(&mut self) -> Result<(), MerkleMountainRangeError> { - self.base_mmr.clear()?; - self.base_cp_index = self.calculate_base_cp_index()?; - for cp_index in 0..=self.base_cp_index { - if let Some(cp) = self - .checkpoints - .get(cp_index) - .map_err(|e| MerkleMountainRangeError::BackendError(e.to_string()))? - { - cp.apply(&mut self.base_mmr)?; - } - } - Ok(()) - } - - // Reconstruct the current MMR from the next checkpoint after the base MMR to the last checkpoints. - fn create_curr_mmr(&mut self) -> Result<(), MerkleMountainRangeError> { - self.curr_cp_index = self - .checkpoints - .len() - .map_err(|e| MerkleMountainRangeError::BackendError(e.to_string()))?; - self.curr_mmr = prune_mutable_mmr::(&self.base_mmr)?; - for cp_index in self.base_cp_index + 1..self.curr_cp_index { - if let Some(cp) = self - .checkpoints - .get(cp_index) - .map_err(|e| MerkleMountainRangeError::BackendError(e.to_string()))? - { - cp.apply(&mut self.curr_mmr)?; - } - } - Ok(()) - } - - // An update to the checkpoints have been detected, update the base MMR to the correct position. - fn update_base_mmr(&mut self) -> Result<(), MerkleMountainRangeError> { - let prev_cp_index = self.base_cp_index; - self.base_cp_index = self.calculate_base_cp_index()?; - if prev_cp_index < self.base_cp_index { - for cp_index in prev_cp_index + 1..=self.base_cp_index { - if let Some(cp) = self - .checkpoints - .get(cp_index) - .map_err(|e| MerkleMountainRangeError::BackendError(e.to_string()))? - { - cp.apply(&mut self.base_mmr)?; - } - } - } else { - self.create_base_mmr()?; - } - Ok(()) - } - - /// Inform the MmrCache that the first N checkpoints have been merged to allow the base and current indices to be - /// updated. - pub fn checkpoints_merged(&mut self, num_merged: usize) -> Result<(), MerkleMountainRangeError> { - if let Some(num_reverse) = num_merged.checked_sub(1) { - self.base_cp_index = self.base_cp_index.saturating_sub(num_reverse); - self.curr_cp_index = self.curr_cp_index.saturating_sub(num_reverse); - } - self.update() - } - - /// This function updates the state of the MMR cache based on the current state of the shared checkpoints. - pub fn update(&mut self) -> Result<(), MerkleMountainRangeError> { - let cp_count = self - .checkpoints - .len() - .map_err(|e| MerkleMountainRangeError::BackendError(e.to_string()))?; - if cp_count <= self.base_cp_index { - // Checkpoint before or the same as the base MMR index, this will require a full reconstruction of the - // cache. - self.create_base_mmr()?; - self.create_curr_mmr()?; - } else if cp_count < self.curr_cp_index { - // A short checkpoint reorg has occurred, and requires the current MMR to be reconstructed. - self.create_curr_mmr()?; - } else if cp_count > self.curr_cp_index { - // The cache has fallen behind and needs to update to the new checkpoint state. - self.update_base_mmr()?; - self.create_curr_mmr()?; - } else { - //we should not update anything - } - Ok(()) - } - - /// Reset the MmrCache and rebuild the base and current MMR state. - pub fn reset(&mut self) -> Result<(), MerkleMountainRangeError> { - self.create_base_mmr()?; - self.create_curr_mmr() - } - - /// Returns the hash of the leaf index provided, as well as its deletion status. The node has been marked for - /// deletion if the boolean value is true. - pub fn fetch_mmr_node(&self, leaf_index: LeafIndex) -> Result<(Option, bool), MerkleMountainRangeError> { - let (base_hash, base_deleted) = self.base_mmr.get_leaf_status(leaf_index)?; - let (curr_hash, curr_deleted) = self.curr_mmr.get_leaf_status(leaf_index)?; - if let Some(base_hash) = base_hash { - return Ok((Some(base_hash), base_deleted | curr_deleted)); - } - Ok((curr_hash, base_deleted | curr_deleted)) - } - - /// Search for the leaf index of the given hash in the nodes of the current and base MMR. - pub fn find_leaf_index(&self, hash: &[u8]) -> Result, MerkleMountainRangeError> { - let mut index = self.base_mmr.find_leaf_index(hash)?; - if index.is_none() { - index = self.curr_mmr.find_leaf_index(hash)?; - } - Ok(index) - } -} - -impl Deref for MmrCache -where - D: Digest + DomainDigest, - BaseBackend: ArrayLike, -{ - type Target = PrunedMutableMmr; - - fn deref(&self) -> &Self::Target { - &self.curr_mmr - } -} diff --git a/base_layer/mmr/src/mutable_mmr.rs b/base_layer/mmr/src/mutable_mmr.rs deleted file mode 100644 index 6c50122879..0000000000 --- a/base_layer/mmr/src/mutable_mmr.rs +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2019. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use std::convert::TryFrom; - -use croaring::Bitmap; -use digest::Digest; -use tari_common::DomainDigest; - -use crate::{ - backend::ArrayLike, - common::{node_index, LeafIndex}, - error::MerkleMountainRangeError, - mutable_mmr_leaf_nodes::MutableMmrLeafNodes, - Hash, - MerkleMountainRange, -}; - -/// Unlike a pure MMR, which is append-only, in `MutableMmr`, leaf nodes can be marked as deleted. -/// -/// In `MutableMmr` a roaring bitmap tracks which data have been marked as deleted, and the merklish root is modified -/// to include the hash of the roaring bitmap. -/// -/// The `MutableMmr` API maps nearly 1:1 to that of MerkleMountainRange so that you should be able to use it as a -/// drop-in replacement for the latter in most cases. -#[derive(Debug)] -pub struct MutableMmr -where - D: Digest + DomainDigest, - B: ArrayLike, -{ - pub(crate) mmr: MerkleMountainRange, - pub(crate) deleted: Bitmap, - pub(crate) size: u32, -} - -impl MutableMmr -where - D: Digest + DomainDigest, - B: ArrayLike, -{ - /// Create a new mutable MMR using the backend provided - pub fn new(mmr_backend: B, deleted: Bitmap) -> Result, MerkleMountainRangeError> { - let mmr = MerkleMountainRange::new(mmr_backend); - Ok(MutableMmr { - size: u32::try_from(mmr.get_leaf_count()?).map_err(|_|MerkleMountainRangeError::InvalidMmrSize)?, - mmr, - deleted, - }) - } - - /// Clear the MutableMmr and assign the MMR state from the set of leaf_hashes and deleted nodes given in `state`. - pub fn assign(&mut self, state: MutableMmrLeafNodes) -> Result<(), MerkleMountainRangeError> { - - self.mmr.assign(state.leaf_hashes)?; - self.deleted = state.deleted; - self.size = u32::try_from(self.mmr.get_leaf_count()?).map_err(|_|MerkleMountainRangeError::InvalidMmrSize)?; - Ok(()) - } - - /// Return the number of leaf nodes in the `MutableMmr` that have not been marked as deleted. - /// - /// NB: This is semantically different to `MerkleMountainRange::len()`. The latter returns the total number of - /// nodes in the MMR, while this function returns the number of leaf nodes minus the number of nodes marked for - /// deletion. - #[allow(clippy::len_without_is_empty)] - pub fn len(&self) -> Result { - let deleted_size = u32::try_from(self.deleted.cardinality()).map_err(|_|MerkleMountainRangeError::InvalidMmrSize)?; - let result = self.size.checked_sub(deleted_size).ok_or(MerkleMountainRangeError::InvalidMmrSize)?; - Ok(result ) - } - - /// Returns true if the the MMR contains no nodes, OR all nodes have been marked for deletion - pub fn is_empty(&self) -> Result { - Ok(self.mmr.is_empty()? || self.deleted.cardinality() == u64::from(self.size)) - } - - /// This function returns the hash of the leaf index provided, indexed from 0. If the hash does not exist, or if it - /// has been marked for deletion, `None` is returned. - pub fn get_leaf_hash(&self, leaf_index: LeafIndex) -> Result, MerkleMountainRangeError> { - let leaf_index_value = u32::try_from(leaf_index.0).map_err(|_| MerkleMountainRangeError::InvalidLeafIndex)?; - if self.deleted.contains(leaf_index_value) { - return Ok(None); - } - self.mmr.get_node_hash(node_index(leaf_index)) - } - - /// Returns the hash of the leaf index provided, as well as its deletion status. The node has been marked for - /// deletion if the boolean value is true. - pub fn get_leaf_status(&self, leaf_index: LeafIndex) -> Result<(Option, bool), MerkleMountainRangeError> { - let hash = self.mmr.get_node_hash(node_index(leaf_index))?; - let leaf_index_value = u32::try_from(leaf_index.0).map_err(|_| MerkleMountainRangeError::InvalidLeafIndex)?; - let deleted = self.deleted.contains(leaf_index_value); - Ok((hash, deleted)) - } - - /// Returns the number of leaf nodes in the MMR. - pub fn get_leaf_count(&self) -> usize { - self.size as usize - } - - /// Returns a merkle(ish) root for this merkle set. - /// - /// The root is calculated by concatenating the MMR merkle root with the compressed serialisation of the bitmap - /// and then hashing the result. - pub fn get_merkle_root(&self) -> Result { - // Note that two MutableMmrs could both return true for `is_empty()`, but have different merkle roots by - // virtue of the fact that the underlying MMRs could be different, but all elements are marked as deleted in - // both sets. - let mmr_root = self.mmr.get_merkle_root()?; - - // To avoid requiring mutability, we compress a clone of the bitmap - let mut bitmap = self.deleted.clone(); - bitmap.run_optimize(); - let bitmap_ser = bitmap.serialize(); - - // Include the compressed bitmap in the root hash - let mut hasher = D::new(); - Digest::update(&mut hasher, &mmr_root); - Digest::update(&mut hasher, &bitmap_ser); - - Ok(hasher.finalize().to_vec()) - } - - /// Returns only the MMR merkle root without the compressed serialisation of the bitmap - pub fn get_mmr_only_root(&self) -> Result { - self.mmr.get_merkle_root() - } - - /// See [MerkleMountainRange::find_node_index] - pub fn find_node_index(&self, hash: &[u8]) -> Result, MerkleMountainRangeError> { - self.mmr.find_node_index(hash) - } - - /// See [MerkleMountainRange::find_leaf_index] - pub fn find_leaf_index(&self, hash: &[u8]) -> Result, MerkleMountainRangeError> { - self.mmr.find_leaf_index(hash) - } - - /// Push a new element into the MMR. Computes new related peaks at the same time if applicable. - /// Returns the new number of leaf nodes (regardless of deleted state) in the mutable MMR - pub fn push(&mut self, hash: Hash) -> Result { - if self.size == u32::MAX { - return Err(MerkleMountainRangeError::MaximumSizeReached); - } - self.mmr.push(hash)?; - self.size += 1; - Ok(self.size as usize) - } - - /// Mark a node for deletion and optionally compress the deletion bitmap. Don't call this function unless you're - /// in a tight loop and want to eke out some extra performance by delaying the bitmap compression until after the - /// batch deletion. - /// - /// Note that this function doesn't actually delete anything (the underlying MMR structure is immutable), but marks - /// the leaf node as deleted. Once a leaf node has been marked for deletion: - /// * `get_leaf_hash(n)` will return None, - /// * `len()` will not count this node anymore - /// - /// **NB**: You should call compress before calling `get_merkle_root()`. If you don't, the merkle root will be - /// incorrect. - /// - /// # Parameters - /// * `leaf_node_index`: The index of the leaf node to mark for deletion, zero-based. - /// - /// # Return - /// The function returns true if a node was actually marked for deletion. If the index is out of bounds, or was - /// already deleted, the function returns false. - pub fn delete(&mut self, leaf_index: u32) -> bool { - if (leaf_index >= self.size) || self.deleted.contains(leaf_index) { - return false; - } - self.deleted.add(leaf_index); - true - } - - /// Compress the roaring bitmap mapping deleted nodes. You never have to call this method unless you have been - /// calling [delete_and_compress] with `compress` set to `false` ahead of a call to [get_merkle_root]. - pub fn compress(&mut self) -> bool { - self.deleted.run_optimize() - } - - /// Walks the nodes in the MMR and validates all parent hashes - /// - /// This just calls through to the underlying MMR's validate method. There's nothing we can do to check whether - /// the roaring bitmap represents all the leaf nodes that we want to delete. Note: A struct that uses - /// `MutableMmr` and links it to actual data should be able to do this though. - pub fn validate(&self) -> Result<(), MerkleMountainRangeError> { - self.mmr.validate() - } - - // Returns a bitmap with only the deleted nodes for the specified region in the MMR. - fn get_sub_bitmap(&self, leaf_index: LeafIndex, count: usize) -> Result { - let mut deleted = self.deleted.clone(); - if leaf_index.0 > 0 { - let remove_range = leaf_index.0.checked_sub(1).ok_or(MerkleMountainRangeError::InvalidMmrSize)?; - deleted.remove_range(0..u32::try_from(remove_range).map_err(|_|MerkleMountainRangeError::InvalidMmrSize)?) - } - let leaf_count = self.mmr.get_leaf_count()?; - if leaf_count > 1 { - let last_index = leaf_index.0.checked_add(count).ok_or(MerkleMountainRangeError::InvalidMmrSize)?.checked_sub(1).ok_or(MerkleMountainRangeError::InvalidMmrSize)?; - //Overflow not possible here as leaf_count will always be greater than 0, min > 1-1=0 - if last_index < leaf_count - 1 { - let remove_range = last_index.checked_add(1).ok_or(MerkleMountainRangeError::InvalidMmrSize)?; - deleted.remove_range(u32::try_from(remove_range).map_err(|_|MerkleMountainRangeError::InvalidMmrSize)?..u32::try_from(leaf_count).map_err(|_|MerkleMountainRangeError::InvalidMmrSize)?); - } - } - Ok(deleted) - } - - /// Returns the state of the MMR that consists of the leaf hashes and the deleted nodes. - pub fn to_leaf_nodes( - &self, - leaf_index: LeafIndex, - count: usize, - ) -> Result { - Ok(MutableMmrLeafNodes { - leaf_hashes: self.mmr.get_leaf_hashes(leaf_index, count)?, - deleted: self.get_sub_bitmap(leaf_index, count)?, - }) - } - - /// Expose the MerkleMountainRange for verifying proofs - pub fn mmr(&self) -> &MerkleMountainRange { - &self.mmr - } - - /// Return a reference to the bitmap of deleted nodes - pub fn deleted(&self) -> &Bitmap { - &self.deleted - } - - pub fn set_deleted(&mut self, deleted: Bitmap) { - self.deleted = deleted; - } - - pub fn clear(&mut self) -> Result<(), MerkleMountainRangeError> { - self.mmr.clear()?; - self.deleted = Bitmap::create(); - self.size = 0; - Ok(()) - } -} - -impl PartialEq> for MutableMmr -where - D: Digest + DomainDigest, - B: ArrayLike, - B2: ArrayLike, -{ - fn eq(&self, other: &MutableMmr) -> bool { - self.get_merkle_root() == other.get_merkle_root() - } -} diff --git a/base_layer/mmr/src/mutable_mmr_leaf_nodes.rs b/base_layer/mmr/src/mutable_mmr_leaf_nodes.rs deleted file mode 100644 index 848a7b16f0..0000000000 --- a/base_layer/mmr/src/mutable_mmr_leaf_nodes.rs +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2019. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use crate::Hash; -use croaring::Bitmap; -use serde::{ - de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor}, - ser::{Serialize, SerializeStruct, Serializer}, -}; -use std::fmt; - -/// The MutableMmrLeafNodes is used to create and share a restorable state for an MMR. -#[derive(Debug, Clone, PartialEq)] -pub struct MutableMmrLeafNodes { - pub leaf_hashes: Vec, - pub deleted: Bitmap, -} - -impl MutableMmrLeafNodes { - /// Create a new MutableMmrLeafNodes using the set of leaf hashes and deleted nodes. - pub fn new(leaf_hashes: Vec, deleted: Bitmap) -> Self { - Self { leaf_hashes, deleted } - } - - /// Merge the current state with the next state bundle - pub fn combine(&mut self, next_state: MutableMmrLeafNodes) { - let MutableMmrLeafNodes { - mut leaf_hashes, - deleted, - } = next_state; - self.leaf_hashes.append(&mut leaf_hashes); - self.deleted.or_inplace(&deleted); - } -} - -impl From> for MutableMmrLeafNodes { - fn from(leaf_hashes: Vec) -> Self { - Self { - leaf_hashes, - deleted: Bitmap::create(), - } - } -} - -impl Serialize for MutableMmrLeafNodes { - fn serialize(&self, serializer: S) -> Result - where S: Serializer { - let mut state = serializer.serialize_struct("MutableMmrLeafNodes", 2)?; - state.serialize_field("leaf_hashes", &self.leaf_hashes)?; - state.serialize_field("deleted", &self.deleted.serialize())?; - state.end() - } -} - -impl<'de> Deserialize<'de> for MutableMmrLeafNodes { - fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { - enum Field { - LeafHashes, - Deleted, - } - - impl<'de> Deserialize<'de> for Field { - fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> { - struct FieldVisitor; - - impl<'de> Visitor<'de> for FieldVisitor { - type Value = Field; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("`leaf_hashes` or `deleted`") - } - - fn visit_str(self, value: &str) -> Result - where E: de::Error { - match value { - "leaf_hashes" => Ok(Field::LeafHashes), - "deleted" => Ok(Field::Deleted), - _ => Err(de::Error::unknown_field(value, FIELDS)), - } - } - } - - deserializer.deserialize_identifier(FieldVisitor) - } - } - - struct MutableMmrLeafNodesVisitor; - - impl<'de> Visitor<'de> for MutableMmrLeafNodesVisitor { - type Value = MutableMmrLeafNodes; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("struct MutableMmrLeafNodes") - } - - fn visit_seq(self, mut seq: V) -> Result - where V: SeqAccess<'de> { - let leaf_hashes = seq.next_element()?.ok_or_else(|| de::Error::invalid_length(0, &self))?; - let deleted_buf: Vec = seq.next_element()?.ok_or_else(|| de::Error::invalid_length(1, &self))?; - let deleted: Bitmap = Bitmap::deserialize(&deleted_buf); - Ok(MutableMmrLeafNodes::new(leaf_hashes, deleted)) - } - - fn visit_map(self, mut map: V) -> Result - where V: MapAccess<'de> { - let mut leaf_hashes = None; - let mut deleted = None; - while let Some(key) = map.next_key()? { - match key { - Field::LeafHashes => { - if leaf_hashes.is_some() { - return Err(de::Error::duplicate_field("nodes_added")); - } - leaf_hashes = Some(map.next_value()?); - }, - Field::Deleted => { - if deleted.is_some() { - return Err(de::Error::duplicate_field("nodes_deleted")); - } - let deleted_buf: Vec = map.next_value()?; - deleted = Some(Bitmap::deserialize(&deleted_buf)); - }, - } - } - let leaf_hashes = leaf_hashes.ok_or_else(|| de::Error::missing_field("leaf_hashes"))?; - let deleted = deleted.ok_or_else(|| de::Error::missing_field("deleted"))?; - Ok(MutableMmrLeafNodes::new(leaf_hashes, deleted)) - } - } - - const FIELDS: &[&str] = &["leaf_hashes", "deleted"]; - deserializer.deserialize_struct("MutableMmrLeafNodes", FIELDS, MutableMmrLeafNodesVisitor) - } -} diff --git a/base_layer/mmr/tests/support/mod.rs b/base_layer/mmr/tests/support/mod.rs index 87da4632d1..6314ac3ca6 100644 --- a/base_layer/mmr/tests/support/mod.rs +++ b/base_layer/mmr/tests/support/mod.rs @@ -22,15 +22,13 @@ // use blake2::Blake2b; -use croaring::Bitmap; use digest::{consts::U32, Digest}; use tari_crypto::{hash_domain, hashing::DomainSeparatedHasher}; -use tari_mmr::{Hash, HashSlice, MerkleMountainRange, MutableMmr}; +use tari_mmr::{Hash, HashSlice, MerkleMountainRange}; hash_domain!(MmrTestHashDomain, "com.tari.test.base_layer.core.kernel_mmr", 1); pub type MmrTestHasherBlake256 = DomainSeparatedHasher, MmrTestHashDomain>; pub type TestMmr = MerkleMountainRange>; -pub type MutableTestMmr = MutableMmr>; pub fn create_mmr(size: usize) -> TestMmr { let mut mmr = TestMmr::new(Vec::default()); @@ -41,15 +39,6 @@ pub fn create_mmr(size: usize) -> TestMmr { mmr } -pub fn create_mutable_mmr(size: usize) -> MutableTestMmr { - let mut mmr = MutableTestMmr::new(Vec::default(), Bitmap::create()).unwrap(); - for i in 0..size { - let hash = int_to_hash(i); - assert!(mmr.push(hash).is_ok()); - } - mmr -} - pub fn int_to_hash(n: usize) -> Vec { MmrTestHasherBlake256::new().digest(&n.to_le_bytes()).as_ref().to_vec() } diff --git a/base_layer/mmr/tests/tests/merkle_checkpoint.rs b/base_layer/mmr/tests/tests/merkle_checkpoint.rs deleted file mode 100644 index 1570ef482a..0000000000 --- a/base_layer/mmr/tests/tests/merkle_checkpoint.rs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2022. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use croaring::Bitmap; -use tari_mmr::MerkleCheckPoint; - -use crate::support; - -#[test] -fn serialize_deserialize() { - let h1 = support::int_to_hash(1); - let mut checkpoint = MerkleCheckPoint::new(vec![h1.clone()], Bitmap::create(), 0); - checkpoint.push_deletion(1); - let bytes = bincode::serialize(&checkpoint).unwrap(); - let der_checkpoint = bincode::deserialize::(&bytes).unwrap(); - assert_eq!(der_checkpoint.nodes_added(), &[h1]); - assert!(der_checkpoint.nodes_deleted().contains(1)); - assert_eq!(der_checkpoint.accumulated_nodes_added_count(), 1); -} diff --git a/base_layer/mmr/tests/tests/mmr_cache.rs b/base_layer/mmr/tests/tests/mmr_cache.rs deleted file mode 100644 index e7e79deb0b..0000000000 --- a/base_layer/mmr/tests/tests/mmr_cache.rs +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright 2022. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use croaring::Bitmap; -use tari_mmr::{ArrayLike, ArrayLikeExt, MemBackendVec, MerkleCheckPoint, MmrCache, MmrCacheConfig}; - -use crate::support::{combine_hashes, int_to_hash, MmrTestHasherBlake256}; - -#[test] -fn create_cache_update_and_rewind() { - let config = MmrCacheConfig { rewind_hist_len: 2 }; - let mut checkpoint_db = MemBackendVec::::new(); - let mut mmr_cache = - MmrCache::::new(Vec::new(), checkpoint_db.clone(), config).unwrap(); - - let h1 = int_to_hash(1); - let h2 = int_to_hash(2); - let h3 = int_to_hash(3); - let h4 = int_to_hash(4); - let h5 = int_to_hash(5); - let h6 = int_to_hash(6); - let h7 = int_to_hash(7); - let h8 = int_to_hash(8); - let ha = combine_hashes(&[&h1, &h2]); - let hb = combine_hashes(&[&h3, &h4]); - let hc = combine_hashes(&[&h5, &h6]); - let hd = combine_hashes(&[&h7, &h8]); - let hahb = combine_hashes(&[&ha, &hb]); - let hchd = combine_hashes(&[&hc, &hd]); - let cp1_mmr_only_root = combine_hashes(&[&ha]); - let cp2_mmr_only_root = combine_hashes(&[&hahb]); - let cp3_mmr_only_root = combine_hashes(&[&hahb, &hc]); - let cp4_mmr_only_root = combine_hashes(&[&combine_hashes(&[&hahb, &hchd])]); - - checkpoint_db - .push(MerkleCheckPoint::new(vec![h1, h2], Bitmap::create(), 0)) - .unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp1_mmr_only_root.clone())); - - checkpoint_db - .push(MerkleCheckPoint::new(vec![h3, h4], Bitmap::create(), 0)) - .unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp2_mmr_only_root)); - - // Two checkpoint update - checkpoint_db - .push(MerkleCheckPoint::new(vec![h5, h6], Bitmap::create(), 0)) - .unwrap(); - checkpoint_db - .push(MerkleCheckPoint::new(vec![h7, h8], Bitmap::create(), 0)) - .unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp4_mmr_only_root.clone())); - - // No rewind - checkpoint_db.truncate(4).unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp4_mmr_only_root)); - - // Only current MMR update - checkpoint_db.truncate(3).unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp3_mmr_only_root)); - - // Full cache update - checkpoint_db.truncate(1).unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp1_mmr_only_root)); -} - -#[test] -fn multiple_rewinds() { - let config = MmrCacheConfig { rewind_hist_len: 2 }; - let mut checkpoint_db = MemBackendVec::::new(); - let mut mmr_cache = - MmrCache::::new(Vec::new(), checkpoint_db.clone(), config).unwrap(); - - // Add h1, h2, h3 and h4 checkpoints - let h1 = int_to_hash(1); - let h2 = int_to_hash(2); - let h3 = int_to_hash(3); - let h4 = int_to_hash(4); - let h5 = int_to_hash(5); - checkpoint_db - .push(MerkleCheckPoint::new(vec![h1.clone()], Bitmap::create(), 0)) - .unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(combine_hashes(&[&h1]))); - - checkpoint_db - .push(MerkleCheckPoint::new(vec![h2.clone()], Bitmap::create(), 0)) - .unwrap(); - assert!(mmr_cache.update().is_ok()); - let h1h2 = combine_hashes(&[&h1, &h2]); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(combine_hashes(&[&h1h2]))); - - checkpoint_db - .push(MerkleCheckPoint::new(vec![h3.clone()], Bitmap::create(), 0)) - .unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(combine_hashes(&[&h1h2, &h3]))); - - checkpoint_db - .push(MerkleCheckPoint::new(vec![h4.clone()], Bitmap::create(), 0)) - .unwrap(); - assert!(mmr_cache.update().is_ok()); - let h3h4 = combine_hashes(&[&h3, &h4]); - let h1h2h3h4 = combine_hashes(&[&h1h2, &h3h4]); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(combine_hashes(&[&h1h2h3h4]))); - assert_eq!(checkpoint_db.len(), Ok(4)); - - // Remove h4 checkpoint - checkpoint_db.truncate(3).unwrap(); - assert_eq!(checkpoint_db.len(), Ok(3)); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(combine_hashes(&[&h1h2, &h3]))); - - // Add h5 checkpoint - checkpoint_db - .push(MerkleCheckPoint::new(vec![h5.clone()], Bitmap::create(), 0)) - .unwrap(); - assert!(mmr_cache.update().is_ok()); - let h3h5 = combine_hashes(&[&h3, &h5]); - let h1h2h3h5 = combine_hashes(&[&h1h2, &h3h5]); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(combine_hashes(&[&h1h2h3h5]))); - - // Remove h5 checkpoint - checkpoint_db.truncate(3).unwrap(); - assert_eq!(checkpoint_db.len(), Ok(3)); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(combine_hashes(&[&h1h2, &h3]))); - - // Remove h3 checkpoint - checkpoint_db.truncate(2).unwrap(); - assert_eq!(checkpoint_db.len(), Ok(2)); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(combine_hashes(&[&h1h2]))); -} - -#[test] -fn checkpoint_merging() { - let config = MmrCacheConfig { rewind_hist_len: 2 }; - let mut checkpoint_db = MemBackendVec::::new(); - let mut mmr_cache = - MmrCache::::new(Vec::new(), checkpoint_db.clone(), config).unwrap(); - - let h1 = int_to_hash(1); - let h2 = int_to_hash(2); - let h3 = int_to_hash(3); - let h4 = int_to_hash(4); - let h5 = int_to_hash(5); - let h6 = int_to_hash(6); - let ha = combine_hashes(&[&h1, &h2]); - let hb = combine_hashes(&[&h3, &h4]); - let hc = combine_hashes(&[&h5, &h6]); - let hahb = combine_hashes(&[&ha, &hb]); - let cp4_mmr_only_root = combine_hashes(&[&hahb]); - let cp6_mmr_only_root = combine_hashes(&[&hahb, &hc]); - let cp1 = MerkleCheckPoint::new(vec![h1], Bitmap::create(), 0); - let cp2 = MerkleCheckPoint::new(vec![h2], Bitmap::create(), 0); - let cp3 = MerkleCheckPoint::new(vec![h3], Bitmap::create(), 0); - let cp4 = MerkleCheckPoint::new(vec![h4], Bitmap::create(), 0); - let cp5 = MerkleCheckPoint::new(vec![h5], Bitmap::create(), 0); - let cp6 = MerkleCheckPoint::new(vec![h6], Bitmap::create(), 0); - - checkpoint_db.push(cp1).unwrap(); - assert!(mmr_cache.update().is_ok()); - checkpoint_db.push(cp2).unwrap(); - assert!(mmr_cache.update().is_ok()); - checkpoint_db.push(cp3).unwrap(); - assert!(mmr_cache.update().is_ok()); - checkpoint_db.push(cp4).unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp4_mmr_only_root.clone())); - - let mut merged_cp = checkpoint_db.get(0).unwrap().unwrap(); - merged_cp.append(checkpoint_db.get(1).unwrap().unwrap()); - assert!(checkpoint_db.shift(2).is_ok()); - assert!(checkpoint_db.push_front(merged_cp).is_ok()); - assert_eq!(checkpoint_db.len(), Ok(3)); - assert!(mmr_cache.checkpoints_merged(2).is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp4_mmr_only_root)); - - checkpoint_db.push(cp5).unwrap(); - assert!(mmr_cache.update().is_ok()); - checkpoint_db.push(cp6).unwrap(); - assert!(mmr_cache.update().is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp6_mmr_only_root.clone())); - - let mut merged_cp = checkpoint_db.get(0).unwrap().unwrap(); - merged_cp.append(checkpoint_db.get(1).unwrap().unwrap()); - merged_cp.append(checkpoint_db.get(2).unwrap().unwrap()); - assert!(checkpoint_db.shift(3).is_ok()); - assert!(checkpoint_db.push_front(merged_cp).is_ok()); - assert_eq!(checkpoint_db.len(), Ok(3)); - assert!(mmr_cache.checkpoints_merged(3).is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp6_mmr_only_root.clone())); - - // Recreate the MmrCache from the altered checkpoints - let mut mmr_cache = - MmrCache::::new(Vec::new(), checkpoint_db.clone(), config).unwrap(); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp6_mmr_only_root.clone())); - - // Replace all existing checkpoints with a single accumulated checkpoint - let mut merged_cp = checkpoint_db.get(0).unwrap().unwrap(); - merged_cp.append(checkpoint_db.get(1).unwrap().unwrap()); - merged_cp.append(checkpoint_db.get(2).unwrap().unwrap()); - assert!(checkpoint_db.shift(3).is_ok()); - assert!(checkpoint_db.push_front(merged_cp).is_ok()); - assert_eq!(checkpoint_db.len(), Ok(1)); - assert!(mmr_cache.checkpoints_merged(3).is_ok()); - assert_eq!(mmr_cache.get_mmr_only_root(), Ok(cp6_mmr_only_root)); -} diff --git a/base_layer/mmr/tests/tests/mod.rs b/base_layer/mmr/tests/tests/mod.rs index 276227adfe..fbf1e969b5 100644 --- a/base_layer/mmr/tests/tests/mod.rs +++ b/base_layer/mmr/tests/tests/mod.rs @@ -21,10 +21,7 @@ // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mod mem_backend_vec; -mod merkle_checkpoint; mod merkle_mountain_range; mod merkle_proof; -mod mmr_cache; -mod mutable_mmr; mod pruned_mmr; mod with_blake512_hash; diff --git a/base_layer/mmr/tests/tests/mutable_mmr.rs b/base_layer/mmr/tests/tests/mutable_mmr.rs deleted file mode 100644 index 81b8727847..0000000000 --- a/base_layer/mmr/tests/tests/mutable_mmr.rs +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2022. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use croaring::Bitmap; -use digest::Digest; -use tari_mmr::{common::LeafIndex, Hash, HashSlice}; -use tari_utilities::hex::Hex; - -use crate::support::{create_mmr, int_to_hash, MmrTestHasherBlake256, MutableTestMmr}; - -fn hash_with_bitmap(hash: &HashSlice, bitmap: &mut Bitmap) -> Hash { - bitmap.run_optimize(); - let hasher = MmrTestHasherBlake256::new(); - hasher - .chain_update(hash) - .chain_update(bitmap.serialize()) - .finalize() - .as_ref() - .to_vec() -} - -/// MMRs with no elements should provide sane defaults. The merkle root must be the hash of an empty string, b"". -#[test] -fn zero_length_mmr() { - let mmr = MutableTestMmr::new(Vec::default(), Bitmap::create()).unwrap(); - assert_eq!(mmr.len().unwrap(), 0); - assert_eq!(mmr.is_empty(), Ok(true)); - let empty_hash = MmrTestHasherBlake256::new().digest(b"").as_ref().to_vec(); - assert_eq!( - mmr.get_merkle_root(), - Ok(hash_with_bitmap(&empty_hash, &mut Bitmap::create())) - ); -} - -#[test] -// Note the hardcoded hashes are only valid when using MutableTestMmr -fn delete() { - let mut mmr = MutableTestMmr::new(Vec::default(), Bitmap::create()).unwrap(); - assert_eq!(mmr.is_empty(), Ok(true)); - for i in 0..5 { - assert!(mmr.push(int_to_hash(i)).is_ok()); - } - assert_eq!(mmr.len().unwrap(), 5); - let root = mmr.get_merkle_root().unwrap(); - assert_eq!( - &root.to_hex(), - "23affc8202916d88fdb991b22c6975ce4bdb69661c40257580a0c4f6147925d7" - ); - // Can't delete past bounds - assert!(!mmr.delete(5)); - assert_eq!(mmr.len().unwrap(), 5); - assert_eq!(mmr.is_empty(), Ok(false)); - assert_eq!(mmr.get_merkle_root(), Ok(root)); - // Delete some nodes - assert!(mmr.push(int_to_hash(5)).is_ok()); - assert!(mmr.delete(0)); - assert!(mmr.delete(2)); - assert!(mmr.delete(4)); - let root = mmr.get_merkle_root().unwrap(); - assert_eq!( - &root.to_hex(), - "9418eecd5f30ae1d892024e068c18013bb4f79f584c9aa5fdba818f9bd40da1e" - ); - assert_eq!(mmr.len().unwrap(), 3); - assert_eq!(mmr.is_empty(), Ok(false)); - // Can't delete that which has already been deleted - assert!(!mmr.delete(0,)); - assert!(!mmr.delete(2,)); - assert!(!mmr.delete(0,)); - // .. or beyond bounds of MMR - assert!(!mmr.delete(9)); - assert_eq!(mmr.len().unwrap(), 3); - assert_eq!(mmr.is_empty(), Ok(false)); - // Merkle root should not have changed: - assert_eq!(mmr.get_merkle_root(), Ok(root)); - assert!(mmr.delete(1)); - assert!(mmr.delete(5)); - assert!(mmr.delete(3)); - assert_eq!(mmr.len().unwrap(), 0); - assert_eq!(mmr.is_empty(), Ok(true)); - mmr.compress(); - let root = mmr.get_merkle_root().unwrap(); - assert_eq!( - &root.to_hex(), - "8bdcad274c1677d94037137492185541d3ae259a863df8a4d71ac665644b78ef" - ); -} - -/// Successively build up an MMR and check that the roots, heights and indices are all correct. -#[test] -fn build_mmr() { - // Check the mutable MMR against a standard MMR and a roaring bitmap. Create one with 5 leaf nodes *8 MMR nodes) - let mmr_check = create_mmr(5); - assert_eq!(mmr_check.len(), Ok(8)); - let mut bitmap = Bitmap::create(); - // Create a small mutable MMR - let mut mmr = MutableTestMmr::new(Vec::default(), Bitmap::create()).unwrap(); - for i in 0..5 { - assert!(mmr.push(int_to_hash(i)).is_ok()); - } - // MutableMmr::len gives the size in terms of leaf nodes: - assert_eq!(mmr.len().unwrap(), 5); - let mmr_root = mmr_check.get_merkle_root().unwrap(); - let root_check = hash_with_bitmap(&mmr_root, &mut bitmap); - assert_eq!(mmr.get_merkle_root(), Ok(root_check)); - // Delete a node - assert!(mmr.delete(3)); - bitmap.add(3); - let root_check = hash_with_bitmap(&mmr_root, &mut bitmap); - assert_eq!(mmr.get_merkle_root(), Ok(root_check)); -} - -#[test] -fn equality_check() { - let mut ma = MutableTestMmr::new(Vec::default(), Bitmap::create()).unwrap(); - let mut mb = MutableTestMmr::new(Vec::default(), Bitmap::create()).unwrap(); - assert!(ma == mb); - assert!(ma.push(int_to_hash(1)).is_ok()); - assert!(ma != mb); - assert!(mb.push(int_to_hash(1)).is_ok()); - assert!(ma == mb); - assert!(ma.push(int_to_hash(2)).is_ok()); - assert!(ma != mb); - assert!(ma.delete(1)); - // Even though the two trees have the same apparent elements, they're still not equal, because we don't actually - // delete anything - assert!(ma != mb); - // Add the same hash to mb and then delete it - assert!(mb.push(int_to_hash(2)).is_ok()); - assert!(mb.delete(1)); - // Now they're equal! - assert!(ma == mb); -} - -#[test] -fn restore_from_leaf_nodes() { - let mut mmr = MutableTestMmr::new(Vec::default(), Bitmap::create()).unwrap(); - for i in 0..12 { - assert!(mmr.push(int_to_hash(i)).is_ok()); - } - assert!(mmr.delete(2)); - assert!(mmr.delete(4)); - assert!(mmr.delete(5)); - - // Request state of MMR with single call - let leaf_count = mmr.get_leaf_count(); - let mmr_state1 = mmr.to_leaf_nodes(LeafIndex(0), leaf_count).unwrap(); - - // Request state of MMR with multiple calls - let mut mmr_state2 = mmr.to_leaf_nodes(LeafIndex(0), 3).unwrap(); - mmr_state2.combine(mmr.to_leaf_nodes(LeafIndex(3), 3).unwrap()); - mmr_state2.combine(mmr.to_leaf_nodes(LeafIndex(6), leaf_count - 6).unwrap()); - assert_eq!(mmr_state1, mmr_state2); - - // Change the state more before the restore - let mmr_root = mmr.get_merkle_root(); - assert!(mmr.push(int_to_hash(7)).is_ok()); - assert!(mmr.push(int_to_hash(8)).is_ok()); - assert!(mmr.delete(3)); - - // Restore from compact state - assert!(mmr.assign(mmr_state1).is_ok()); - assert_eq!(mmr.get_merkle_root(), mmr_root); - let restored_mmr_state = mmr.to_leaf_nodes(LeafIndex(0), mmr.get_leaf_count()).unwrap(); - assert_eq!(restored_mmr_state, mmr_state2); -} diff --git a/base_layer/mmr/tests/tests/pruned_mmr.rs b/base_layer/mmr/tests/tests/pruned_mmr.rs index 8073639391..1a1cf51249 100644 --- a/base_layer/mmr/tests/tests/pruned_mmr.rs +++ b/base_layer/mmr/tests/tests/pruned_mmr.rs @@ -28,11 +28,11 @@ use rand::{ }; use tari_mmr::{ common::LeafIndex, - functions::{calculate_mmr_root, calculate_pruned_mmr_root, prune_mmr}, + functions::{calculate_mmr_root, prune_mmr}, Hash, }; -use crate::support::{create_mmr, create_mutable_mmr, int_to_hash}; +use crate::support::{create_mmr, int_to_hash}; #[test] fn pruned_mmr_empty() { @@ -81,27 +81,6 @@ fn get_changes() -> (usize, Vec, Vec) { (src_size, additions, deletions) } -/// Create a random-sized MMR. Add a random number of additions and deletions; and check the new root against the -/// result of `calculate_pruned_mmr_root` -#[test] -pub fn calculate_pruned_mmr_roots() { - let (src_size, additions, deletions) = get_changes(); - let mut src = create_mutable_mmr(src_size); - let src_root = src.get_merkle_root().expect("Did not get source root"); - let root = - calculate_pruned_mmr_root(&src, additions.clone(), deletions.clone()).expect("Did not calculate new root"); - assert_ne!(src_root, root); - // Double check - additions.into_iter().for_each(|h| { - src.push(h).unwrap(); - }); - deletions.iter().for_each(|i| { - src.delete(*i); - }); - let new_root = src.get_merkle_root().expect("Did not calculate new root"); - assert_eq!(root, new_root); -} - /// Create a random-sized MMR. Add a random number of additions; and check the new root against the /// result of `calculate_mmr_root` #[test] From 24a19eb163a7eee4509cbc7be69c8a5327e51486 Mon Sep 17 00:00:00 2001 From: SW van Heerden Date: Wed, 15 Nov 2023 08:54:41 +0200 Subject: [PATCH 2/2] cleanup smt benches --- base_layer/mmr/Cargo.toml | 3 -- base_layer/mmr/benches/smt.rs | 56 ++++++++++++++++++++- base_layer/mmr/benches/smt_vs_mmr.rs | 73 ---------------------------- 3 files changed, 54 insertions(+), 78 deletions(-) delete mode 100644 base_layer/mmr/benches/smt_vs_mmr.rs diff --git a/base_layer/mmr/Cargo.toml b/base_layer/mmr/Cargo.toml index a357a18d20..863a3618b9 100644 --- a/base_layer/mmr/Cargo.toml +++ b/base_layer/mmr/Cargo.toml @@ -39,9 +39,6 @@ harness = false name = "smt" harness = false -[[bench]] -name = "smt_vs_mmr" -harness = false [[test]] name = "tari_mmr_integration_tests" diff --git a/base_layer/mmr/benches/smt.rs b/base_layer/mmr/benches/smt.rs index c6557c2986..0676e58401 100644 --- a/base_layer/mmr/benches/smt.rs +++ b/base_layer/mmr/benches/smt.rs @@ -19,7 +19,7 @@ fn create_smt() -> SparseMerkleTree> { SparseMerkleTree::>::new() } -pub fn benchmark_sparse_merkle_trees(c: &mut Criterion) { +pub fn benchmark_smt_insert(c: &mut Criterion) { let sizes = [100, 10_000]; for size in sizes { c.bench_function(&format!("SMT: Insert {size} keys"), move |b| { @@ -38,5 +38,57 @@ pub fn benchmark_sparse_merkle_trees(c: &mut Criterion) { } } -criterion_group!(smt, benchmark_sparse_merkle_trees); +fn insert_into_smt(keys: &[NodeKey], tree: &mut SparseMerkleTree>) { + keys.iter().for_each(|key| { + tree.upsert(key.clone(), ValueHash::default()).unwrap(); + }); +} + +fn delete_from_smt(keys: &[NodeKey], tree: &mut SparseMerkleTree>) { + keys.iter().for_each(|key| { + tree.delete(key).unwrap(); + }); +} + +fn time_function(header: &str, f: impl FnOnce()) -> std::time::Duration { + println!("Starting: {header}"); + let now = std::time::Instant::now(); + f(); + let t = now.elapsed(); + println!("Finished: {header} - {t:?}"); + t +} + +pub fn root_hash(_c: &mut Criterion) { + let size = 1_000_000; + let half_size = size / 2; + let keys = get_keys(size); + let mut tree = create_smt(); + time_function(&format!("SMT: Inserting {size} keys"), || { + insert_into_smt(&keys, &mut tree); + }); + time_function("SMT: Calculating root hash", || { + let size = tree.size(); + let hash = tree.hash(); + println!("Tree size: {size}. Root hash: {hash:x}"); + }); + time_function(&format!("SMT: Deleting {half_size} keys"), || { + delete_from_smt(&keys[0..half_size], &mut tree); + }); + time_function("SMT: Calculating root hash", || { + let size = tree.size(); + let hash = tree.hash(); + println!("Tree size: {size}. Root hash: {hash:x}"); + }); + time_function(&format!("SMT: Deleting another {half_size} keys"), || { + delete_from_smt(&keys[half_size..], &mut tree); + }); + time_function("SMT: Calculating root hash", || { + let size = tree.size(); + let hash = tree.hash(); + println!("Tree size: {size}. Root hash: {hash:x}"); + }); +} + +criterion_group!(smt, benchmark_smt_insert, root_hash); criterion_main!(smt); diff --git a/base_layer/mmr/benches/smt_vs_mmr.rs b/base_layer/mmr/benches/smt_vs_mmr.rs deleted file mode 100644 index a14cd89580..0000000000 --- a/base_layer/mmr/benches/smt_vs_mmr.rs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2023. The Tari Project -// SPDX-License-Identifier: BSD-3-Clause - -use std::convert::TryFrom; - -use blake2::Blake2b; -use digest::consts::U32; -use tari_mmr::sparse_merkle_tree::{NodeKey, SparseMerkleTree, ValueHash}; - -fn random_key() -> NodeKey { - let key = rand::random::<[u8; 32]>(); - NodeKey::from(key) -} - -fn get_keys(n: usize) -> Vec { - (0..n).map(|_| random_key()).collect() -} - -fn create_smt() -> SparseMerkleTree> { - SparseMerkleTree::>::new() -} - -fn insert_into_smt(keys: &[NodeKey], tree: &mut SparseMerkleTree>) { - keys.iter().for_each(|key| { - tree.upsert(key.clone(), ValueHash::default()).unwrap(); - }); -} - -fn delete_from_smt(keys: &[NodeKey], tree: &mut SparseMerkleTree>) { - keys.iter().for_each(|key| { - tree.delete(key).unwrap(); - }); -} - -fn time_function(header: &str, f: impl FnOnce()) -> std::time::Duration { - println!("Starting: {header}"); - let now = std::time::Instant::now(); - f(); - let t = now.elapsed(); - println!("Finished: {header} - {t:?}"); - t -} - -fn main() { - let size = 1_000_000; - let half_size = size / 2; - let keys = get_keys(size); - let mut tree = create_smt(); - time_function(&format!("SMT: Inserting {size} keys"), || { - insert_into_smt(&keys, &mut tree); - }); - time_function("SMT: Calculating root hash", || { - let size = tree.size(); - let hash = tree.hash(); - println!("Tree size: {size}. Root hash: {hash:x}"); - }); - time_function(&format!("SMT: Deleting {half_size} keys"), || { - delete_from_smt(&keys[0..half_size], &mut tree); - }); - time_function("SMT: Calculating root hash", || { - let size = tree.size(); - let hash = tree.hash(); - println!("Tree size: {size}. Root hash: {hash:x}"); - }); - time_function(&format!("SMT: Deleting another {half_size} keys"), || { - delete_from_smt(&keys[half_size..], &mut tree); - }); - time_function("SMT: Calculating root hash", || { - let size = tree.size(); - let hash = tree.hash(); - println!("Tree size: {size}. Root hash: {hash:x}"); - }); -}