From 7c220e2eb2013799ec42a0b8098c2687685a8fe1 Mon Sep 17 00:00:00 2001 From: Bruno Deferrari Date: Mon, 25 Mar 2024 13:48:09 -0300 Subject: [PATCH] feat(ledgers): When dumping and loading the genesis ledgers, include the hashes matrix to speed up the process --- ledger/src/mask/mask.rs | 11 ++++++++++ ledger/src/mask/mask_impl.rs | 20 +++++++++++++++++ ledger/src/tree.rs | 18 +++++++++++++++ .../transition_frontier_genesis_config.rs | 22 ++++++++++++++++++- tools/ledger-tool/src/main.rs | 10 ++++++++- 5 files changed, 79 insertions(+), 2 deletions(-) diff --git a/ledger/src/mask/mask.rs b/ledger/src/mask/mask.rs index d5815f78a..9af366983 100644 --- a/ledger/src/mask/mask.rs +++ b/ledger/src/mask/mask.rs @@ -268,6 +268,17 @@ impl Mask { self.with(|this| this.validate_inner_hashes()) } + /// Returns a vector of tuples of (index, hash) of all the hashes contained + /// in this mask. + pub fn get_raw_inner_hashes(&self) -> Vec<(u64, Fp)> { + self.with(|this| this.get_raw_inner_hashes()) + } + + /// Sets the contents of this mask's hash matrix using raw data. + pub fn set_raw_inner_hashes(&self, hashes: Vec<(u64, Fp)>) { + self.with(|this| this.set_raw_inner_hashes(hashes)) + } + /// For tests only, check if the address is in the mask, without checking parent #[cfg(test)] fn test_is_in_mask(&self, addr: &Address) -> bool { diff --git a/ledger/src/mask/mask_impl.rs b/ledger/src/mask/mask_impl.rs index 6e13b9ffd..4f4908cc3 100644 --- a/ledger/src/mask/mask_impl.rs +++ b/ledger/src/mask/mask_impl.rs @@ -972,6 +972,26 @@ impl MaskImpl { Ok(()) } + pub fn get_raw_inner_hashes(&self) -> Vec<(u64, Fp)> { + match self { + Root { database, .. } => { + database.with(|this| this.hashes_matrix.get_raw_inner_hashes()) + } + Attached { hashes, .. } => hashes.clone().get_raw_inner_hashes(), + Unattached { hashes, .. } => hashes.clone().get_raw_inner_hashes(), + } + } + + pub fn set_raw_inner_hashes(&self, raw_hashes: Vec<(u64, Fp)>) { + match self { + Root { database, .. } => { + database.with(|this| this.hashes_matrix.set_raw_inner_hashes(raw_hashes)) + } + Attached { hashes, .. } => hashes.clone().set_raw_inner_hashes(raw_hashes), + Unattached { hashes, .. } => hashes.clone().set_raw_inner_hashes(raw_hashes), + } + } + /// For tests only, check if the address is in the mask, without checking parent #[cfg(test)] pub fn test_is_in_mask(&self, addr: &Address) -> bool { diff --git a/ledger/src/tree.rs b/ledger/src/tree.rs index 1784cd8d3..3637705df 100644 --- a/ledger/src/tree.rs +++ b/ledger/src/tree.rs @@ -106,6 +106,14 @@ impl HashesMatrix { self.nhashes += 1; } + /// Do not use directly. Used to forcefully reconstructing the hashes + /// matrix from raw data. + pub fn set_raw_index(&mut self, idx: u64, hash: Fp) { + let old = self.matrix.insert(idx, hash); + assert!(old.is_none()); + self.nhashes += 1; + } + pub fn remove(&mut self, addr: &Address) { let linear = addr.to_linear_index(); self.remove_at_index(linear); @@ -194,6 +202,16 @@ impl HashesMatrix { nhashes: *nhashes, } } + + pub fn get_raw_inner_hashes(&self) -> Vec<(u64, Fp)> { + self.matrix.clone().into_iter().collect() + } + + pub fn set_raw_inner_hashes(&mut self, hashes: Vec<(u64, Fp)>) { + for (idx, hash) in hashes { + self.set_raw_index(idx, hash); + } + } } static HASH_EMPTIES: Lazy>> = Lazy::new(|| { diff --git a/node/src/transition_frontier/genesis/transition_frontier_genesis_config.rs b/node/src/transition_frontier/genesis/transition_frontier_genesis_config.rs index f41139f75..1e03b415f 100644 --- a/node/src/transition_frontier/genesis/transition_frontier_genesis_config.rs +++ b/node/src/transition_frontier/genesis/transition_frontier_genesis_config.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use ledger::{scan_state::currency::Balance, BaseLedger}; +use mina_hasher::Fp; use mina_p2p_messages::{binprot::BinProtRead, v2}; use openmina_core::constants::CONSTRAINT_CONSTANTS; use serde::{Deserialize, Serialize}; @@ -100,9 +101,14 @@ impl GenesisConfig { Self::AccountsBinProt { bytes, constants } => { let mut bytes = bytes.as_ref(); let expected_hash = Option::::binprot_read(&mut bytes)?; + let hashes = Vec::<(u64, v2::LedgerHash)>::binprot_read(&mut bytes)? + .into_iter() + .map(|(idx, hash)| (idx, hash.0.to_field())) + .collect(); let accounts = Vec::::binprot_read(&mut bytes)?; - let (mut mask, total_currency) = Self::build_ledger_from_accounts(accounts); + let (mut mask, total_currency) = + Self::build_ledger_from_accounts_and_hashes(accounts, hashes); let ledger_hash = ledger_hash(&mut mask); if let Some(expected_hash) = expected_hash.filter(|h| h != &ledger_hash) { anyhow::bail!("ledger hash mismatch after building the mask! expected: '{expected_hash}', got '{ledger_hash}'"); @@ -169,8 +175,22 @@ impl GenesisConfig { mask.get_or_create_account(account_id, account).unwrap(); (mask, total_currency) }); + (mask, v2::CurrencyAmountStableV1(total_currency.into())) } + + fn build_ledger_from_accounts_and_hashes( + accounts: impl IntoIterator, + hashes: Vec<(u64, Fp)>, + ) -> (ledger::Mask, v2::CurrencyAmountStableV1) { + let (mask, total_currency) = Self::build_ledger_from_accounts(accounts); + + // Must happen after the accounts have been set to avoid + // cache invalidations. + mask.set_raw_inner_hashes(hashes); + + (mask, total_currency) + } } fn ledger_hash(mask: &mut ledger::Mask) -> v2::LedgerHash { diff --git a/tools/ledger-tool/src/main.rs b/tools/ledger-tool/src/main.rs index c20a363a0..b9206a68c 100644 --- a/tools/ledger-tool/src/main.rs +++ b/tools/ledger-tool/src/main.rs @@ -154,11 +154,19 @@ fn main() -> anyhow::Result<()> { } let root = mask.merkle_root(); - let top_hash = v2::LedgerHash::from(v2::MinaBaseLedgerHash0StableV1(root.into())); + let top_hash = v2::LedgerHash::from_fp(root); println!("{top_hash}"); + let mut hashes = vec![]; + + for (idx, hash) in mask.get_raw_inner_hashes() { + let hash = v2::LedgerHash::from_fp(hash); + hashes.push((idx, hash)); + } + let mut output = File::create(&output)?; Some(top_hash).binprot_write(&mut output)?; + hashes.binprot_write(&mut output)?; accounts.binprot_write(&mut output)?; Ok(())