From f8902ddd8e0044441e62ea029c7477b2cc4d2451 Mon Sep 17 00:00:00 2001 From: Steve Myers Date: Mon, 24 Aug 2020 21:18:44 -0700 Subject: [PATCH 1/2] Convert ChainDB to a trait implemented by Hammersbald struct. Make hammersbald default optional dependency. --- .gitignore | 1 + Cargo.toml | 7 +- src/chaindb.rs | 216 +++++++----------------------------------- src/constructor.rs | 9 +- src/hammersbald.rs | 231 +++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 6 files changed, 279 insertions(+), 186 deletions(-) create mode 100644 src/hammersbald.rs diff --git a/.gitignore b/.gitignore index 35bf9d4..d5f1ae6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ target *.iml Cargo.lock +client.db.* diff --git a/Cargo.toml b/Cargo.toml index 226ac1f..ec33754 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,9 @@ keywords = [ "bitcoin" ] readme = "README.md" edition = "2018" +[features] +default = ["hammersbald"] + [lib] name = "murmel" path = "src/lib.rs" @@ -19,7 +22,6 @@ path = "src/lib.rs" lightning = { version ="0.0.9", optional=true } bitcoin = { version= "0.21", features=["use-serde"]} bitcoin_hashes = "0.7" -hammersbald = { version= "2.4", features=["bitcoin_support"]} mio = "0.6" rand = "0.7" log = "0.4" @@ -31,6 +33,9 @@ futures-timer = "0.3" serde="1" serde_derive="1" +## optional +hammersbald = { version= "2.4", features=["bitcoin_support"], optional=true } + [dev-dependencies] rustc-serialize = "0.3" hex = "0.3" diff --git a/src/chaindb.rs b/src/chaindb.rs index 5173788..dfe0212 100644 --- a/src/chaindb.rs +++ b/src/chaindb.rs @@ -14,176 +14,69 @@ // limitations under the License. // //! -//! # Blockchain DB for a node +//! # Blockchain DB API for a node //! use std::sync::{Arc, RwLock}; -use std::path::Path; -use bitcoin::{BitcoinHash, Network}; +use bitcoin::BitcoinHash; use bitcoin::blockdata::block::BlockHeader; -use bitcoin::blockdata::constants::genesis_block; use bitcoin_hashes::sha256d; -use hammersbald::{BitcoinAdaptor, HammersbaldAPI, persistent, transient}; use crate::error::Error; -use crate::headercache::{CachedHeader, HeaderCache}; -use log::{debug, info, warn, error}; +use crate::headercache::CachedHeader; + use serde_derive::{Serialize, Deserialize}; /// Shared handle to a database storing the block chain /// protected by an RwLock -pub type SharedChainDB = Arc>; - -/// Database storing the block chain -pub struct ChainDB { - db: BitcoinAdaptor, - headercache: HeaderCache, - network: Network, -} - -impl ChainDB { - /// Create an in-memory database instance - pub fn mem(network: Network) -> Result { - info!("working with in memory chain db"); - let db = BitcoinAdaptor::new(transient(2)?); - let headercache = HeaderCache::new(network); - Ok(ChainDB { db, network, headercache }) - } +pub type SharedChainDB = Arc>>; - /// Create or open a persistent database instance identified by the path - pub fn new(path: &Path, network: Network) -> Result { - let basename = path.to_str().unwrap().to_string(); - let db = BitcoinAdaptor::new(persistent((basename.clone()).as_str(), 100, 2)?); - let headercache = HeaderCache::new(network); - Ok(ChainDB { db, network, headercache }) - } +/// Blockchain DB API for a client node. +pub trait ChainDB: Send + Sync { - /// Initialize caches - pub fn init(&mut self) -> Result<(), Error> { - self.init_headers()?; - Ok(()) - } + /// Initialize caches. + fn init(&mut self) -> Result<(), Error>; /// Batch updates. Updates are permanent after finishing a batch. - pub fn batch(&mut self) -> Result<(), Error> { - self.db.batch()?; - Ok(()) - } + fn batch(&mut self) -> Result<(), Error>; - fn init_headers(&mut self) -> Result<(), Error> { - if let Some(tip) = self.fetch_header_tip()? { - info!("reading stored header chain from tip {}", tip); - if self.fetch_header(&tip)?.is_some() { - let mut h = tip; - while let Some(stored) = self.fetch_header(&h)? { - debug!("read stored header {}", &stored.bitcoin_hash()); - self.headercache.add_header_unchecked(&h, &stored); - if stored.header.prev_blockhash != sha256d::Hash::default() { - h = stored.header.prev_blockhash; - } else { - break; - } - } - self.headercache.reverse_trunk(); - info!("read {} headers", self.headercache.len()); - } else { - warn!("unable to read header for tip {}", tip); - self.init_to_genesis()?; - } - } else { - info!("no header tip found"); - self.init_to_genesis()?; - } - Ok(()) - } + /// Store a header. + fn add_header(&mut self, header: &BlockHeader) -> Result>, Option>)>, Error>; - fn init_to_genesis(&mut self) -> Result<(), Error> { - let genesis = genesis_block(self.network).header; - if let Some((cached, _, _)) = self.headercache.add_header(&genesis)? { - info!("initialized with genesis header {}", genesis.bitcoin_hash()); - self.db.put_hash_keyed(&cached.stored)?; - self.db.batch()?; - self.store_header_tip(&cached.bitcoin_hash())?; - self.db.batch()?; - } else { - error!("failed to initialize with genesis header"); - return Err(Error::NoTip); - } - Ok(()) - } + /// Return position of hash on trunk if hash is on trunk. + fn pos_on_trunk(&self, hash: &sha256d::Hash) -> Option; - /// Store a header - pub fn add_header(&mut self, header: &BlockHeader) -> Result>, Option>)>, Error> { - if let Some((cached, unwinds, forward)) = self.headercache.add_header(header)? { - self.db.put_hash_keyed(&cached.stored)?; - if let Some(forward) = forward.clone() { - if forward.len() > 0 { - self.store_header_tip(forward.last().unwrap())?; - } - } - return Ok(Some((cached.stored, unwinds, forward))); - } - Ok(None) - } + /// Iterate trunk [from .. tip]. + fn iter_trunk<'a>(&'a self, from: u32) -> Box + 'a>; - /// return position of hash on trunk if hash is on trunk - pub fn pos_on_trunk(&self, hash: &sha256d::Hash) -> Option { - self.headercache.pos_on_trunk(hash) - } + /// Iterate trunk [genesis .. from] in reverse order from is the tip if not specified. + fn iter_trunk_rev<'a>(&'a self, from: Option) -> Box + 'a>; - /// iterate trunk [from .. tip] - pub fn iter_trunk<'a>(&'a self, from: u32) -> impl Iterator + 'a { - self.headercache.iter_trunk(from) - } + /// Retrieve the id of the block/header with most work. + fn header_tip(&self) -> Option; - /// iterate trunk [genesis .. from] in reverse order from is the tip if not specified - pub fn iter_trunk_rev<'a>(&'a self, from: Option) -> impl Iterator + 'a { - self.headercache.iter_trunk_rev(from) - } + /// Fetch a header by its id from cache. + fn get_header(&self, id: &sha256d::Hash) -> Option; - /// retrieve the id of the block/header with most work - pub fn header_tip(&self) -> Option { - self.headercache.tip() - } + /// Fetch a header by its id from cache. + fn get_header_for_height(&self, height: u32) -> Option; - /// Fetch a header by its id from cache - pub fn get_header(&self, id: &sha256d::Hash) -> Option { - self.headercache.get_header(id) - } + /// Locator for getheaders message. + fn header_locators(&self) -> Vec; - /// Fetch a header by its id from cache - pub fn get_header_for_height(&self, height: u32) -> Option { - self.headercache.get_header_for_height(height) - } + /// Store the header id with most work. + fn store_header_tip(&mut self, tip: &sha256d::Hash) -> Result<(), Error>; - /// locator for getheaders message - pub fn header_locators(&self) -> Vec { - self.headercache.locator_hashes() - } + /// Find header id with most work. + fn fetch_header_tip(&self) -> Result, Error>; - /// Store the header id with most work - pub fn store_header_tip(&mut self, tip: &sha256d::Hash) -> Result<(), Error> { - self.db.put_keyed_encodable(HEADER_TIP_KEY, tip)?; - Ok(()) - } + /// Read header from the DB. + fn fetch_header(&self, id: &sha256d::Hash) -> Result, Error>; - /// Find header id with most work - pub fn fetch_header_tip(&self) -> Result, Error> { - Ok(self.db.get_keyed_decodable::(HEADER_TIP_KEY)?.map(|(_, h)| h.clone())) - } - - /// Read header from the DB - pub fn fetch_header(&self, id: &sha256d::Hash) -> Result, Error> { - Ok(self.db.get_hash_keyed::(id)?.map(|(_, header)| header)) - } - - /// Shutdown db - pub fn shutdown(&mut self) { - self.db.shutdown(); - debug!("shutdown chain db") - } + /// Shutdown the DB. + fn shutdown(&mut self); } /// A header enriched with information about its position on the blockchain @@ -204,45 +97,4 @@ impl BitcoinHash for StoredHeader { } } -const HEADER_TIP_KEY: &[u8] = &[0u8; 1]; - -#[cfg(test)] -mod test { - use bitcoin::{Network, BitcoinHash}; - use bitcoin_hashes::sha256d::Hash; - use bitcoin::blockdata::constants::genesis_block; - - use crate::chaindb::ChainDB; - - #[test] - fn init_tip_header() { - let network = Network::Testnet; - let genesis_header = genesis_block(network).header; - - let mut chaindb = ChainDB::mem(network).unwrap(); - chaindb.init().unwrap(); - chaindb.init().unwrap(); - - let header_tip = chaindb.header_tip(); - assert!(header_tip.is_some(), "failed to get header for tip"); - assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) - } - - #[test] - fn init_recover_if_missing_tip_header() { - let network = Network::Testnet; - let genesis_header = genesis_block(network).header; - - let mut chaindb = ChainDB::mem(network).unwrap(); - let missing_tip_header_hash: Hash = "6cfb35868c4465b7c289d7d5641563aa973db6a929655282a7bf95c8257f53ef".parse().unwrap(); - chaindb.store_header_tip(&missing_tip_header_hash).unwrap(); - - chaindb.init().unwrap(); - - let header_tip = chaindb.header_tip(); - assert!(header_tip.is_some(), "failed to get header for tip"); - assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) - } -} - diff --git a/src/constructor.rs b/src/constructor.rs index 748240f..316fff9 100644 --- a/src/constructor.rs +++ b/src/constructor.rs @@ -24,7 +24,7 @@ use bitcoin::{ constants::Network } }; -use crate::chaindb::{ChainDB, SharedChainDB}; +use crate::hammersbald::Hammersbald; use crate::dispatcher::Dispatcher; use crate::dns::dns_seed; use crate::error::Error; @@ -55,6 +55,7 @@ use bitcoin::network::message::NetworkMessage; use bitcoin::network::message::RawNetworkMessage; use crate::p2p::BitcoinP2PConfig; use std::time::Duration; +use crate::chaindb::SharedChainDB; const MAX_PROTOCOL_VERSION: u32 = 70001; const USER_AGENT: &'static str = concat!("/Murmel:", env!("CARGO_PKG_VERSION"), '/'); @@ -71,9 +72,11 @@ impl Constructor { pub fn open_db(path: Option<&Path>, network: Network, _birth: u64) -> Result { let mut chaindb = if let Some(path) = path { - ChainDB::new(path, network)? + #[cfg(feature = "default")] + Hammersbald::new(path, network)? } else { - ChainDB::mem(network)? + #[cfg(feature = "default")] + Hammersbald::mem(network)? }; chaindb.init()?; Ok(Arc::new(RwLock::new(chaindb))) diff --git a/src/hammersbald.rs b/src/hammersbald.rs new file mode 100644 index 0000000..2690354 --- /dev/null +++ b/src/hammersbald.rs @@ -0,0 +1,231 @@ +// +// Copyright 2018-2019 Tamas Blummer +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//! +//! # Blockchain DB for a node +//! + +use std::path::Path; + +use bitcoin::{BitcoinHash, Network}; +use bitcoin::blockdata::block::BlockHeader; +use bitcoin::blockdata::constants::genesis_block; + +use bitcoin_hashes::sha256d; +use hammersbald::{BitcoinAdaptor, HammersbaldAPI, persistent, transient}; + +use crate::error::Error; +use crate::headercache::{CachedHeader, HeaderCache}; +use log::{debug, info, warn, error}; +use crate::chaindb::StoredHeader; +use crate::chaindb::ChainDB; + +/// Database storing the block chain +pub struct Hammersbald { + db: BitcoinAdaptor, + headercache: HeaderCache, + network: Network, +} + + +impl Hammersbald { + + /// Create an in-memory database instance + pub fn mem(network: Network) -> Result, Error> { + info!("working with in memory chain db"); + let db = BitcoinAdaptor::new(transient(2)?); + let headercache = HeaderCache::new(network); + Ok(Box::from(Hammersbald { db, network, headercache })) + } + + /// Create or open a persistent database instance identified by the path + pub fn new(path: &Path, network: Network) -> Result, Error> { + let basename = path.to_str().unwrap().to_string(); + let db = BitcoinAdaptor::new(persistent((basename.clone()).as_str(), 100, 2)?); + let headercache = HeaderCache::new(network); + Ok(Box::from(Hammersbald { db, network, headercache })) + } + + fn init_headers(&mut self) -> Result<(), Error> { + if let Some(tip) = self.fetch_header_tip()? { + info!("reading stored header chain from tip {}", tip); + if self.fetch_header(&tip)?.is_some() { + let mut h = tip; + while let Some(stored) = self.fetch_header(&h)? { + debug!("read stored header {}", &stored.bitcoin_hash()); + self.headercache.add_header_unchecked(&h, &stored); + if stored.header.prev_blockhash != sha256d::Hash::default() { + h = stored.header.prev_blockhash; + } else { + break; + } + } + self.headercache.reverse_trunk(); + info!("read {} headers", self.headercache.len()); + } else { + warn!("unable to read header for tip {}", tip); + self.init_to_genesis()?; + } + } else { + info!("no header tip found"); + self.init_to_genesis()?; + } + Ok(()) + } + + fn init_to_genesis(&mut self) -> Result<(), Error> { + let genesis = genesis_block(self.network).header; + if let Some((cached, _, _)) = self.headercache.add_header(&genesis)? { + info!("initialized with genesis header {}", genesis.bitcoin_hash()); + self.db.put_hash_keyed(&cached.stored)?; + self.db.batch()?; + self.store_header_tip(&cached.bitcoin_hash())?; + self.db.batch()?; + } else { + error!("failed to initialize with genesis header"); + return Err(Error::NoTip); + } + Ok(()) + } +} + +impl ChainDB for Hammersbald { + + /// Initialize caches + fn init(&mut self) -> Result<(), Error> { + self.init_headers()?; + Ok(()) + } + + /// Batch updates. Updates are permanent after finishing a batch. + fn batch(&mut self) -> Result<(), Error> { + self.db.batch()?; + Ok(()) + } + + /// Store a header + fn add_header(&mut self, header: &BlockHeader) -> Result>, Option>)>, Error> { + if let Some((cached, unwinds, forward)) = self.headercache.add_header(header)? { + self.db.put_hash_keyed(&cached.stored)?; + if let Some(forward) = forward.clone() { + if forward.len() > 0 { + self.store_header_tip(forward.last().unwrap())?; + } + } + return Ok(Some((cached.stored, unwinds, forward))); + } + Ok(None) + } + + /// return position of hash on trunk if hash is on trunk + fn pos_on_trunk(&self, hash: &sha256d::Hash) -> Option { + self.headercache.pos_on_trunk(hash) + } + + /// iterate trunk [from .. tip] + fn iter_trunk<'a>(&'a self, from: u32) -> Box +'a> { + self.headercache.iter_trunk(from) + } + + /// iterate trunk [genesis .. from] in reverse order from is the tip if not specified + fn iter_trunk_rev<'a>(&'a self, from: Option) -> Box +'a> { + self.headercache.iter_trunk_rev(from) + } + + /// retrieve the id of the block/header with most work + fn header_tip(&self) -> Option { + self.headercache.tip() + } + + /// Fetch a header by its id from cache + fn get_header(&self, id: &sha256d::Hash) -> Option { + self.headercache.get_header(id) + } + + /// Fetch a header by its id from cache + fn get_header_for_height(&self, height: u32) -> Option { + self.headercache.get_header_for_height(height) + } + + /// locator for getheaders message + fn header_locators(&self) -> Vec { + self.headercache.locator_hashes() + } + + /// Store the header id with most work + fn store_header_tip(&mut self, tip: &sha256d::Hash) -> Result<(), Error> { + self.db.put_keyed_encodable(HEADER_TIP_KEY, tip)?; + Ok(()) + } + + /// Find header id with most work + fn fetch_header_tip(&self) -> Result, Error> { + Ok(self.db.get_keyed_decodable::(HEADER_TIP_KEY)?.map(|(_, h)| h.clone())) + } + + /// Read header from the DB + fn fetch_header(&self, id: &sha256d::Hash) -> Result, Error> { + Ok(self.db.get_hash_keyed::(id)?.map(|(_, header)| header)) + } + + /// Shutdown db + fn shutdown(&mut self) { + self.db.shutdown(); + debug!("shutdown chain db") + } +} + +const HEADER_TIP_KEY: &[u8] = &[0u8; 1]; + +#[cfg(test)] +mod test { + use bitcoin::{Network, BitcoinHash}; + use bitcoin_hashes::sha256d::Hash; + use bitcoin::blockdata::constants::genesis_block; + + use crate::hammersbald::Hammersbald; + + #[test] + fn init_tip_header() { + let network = Network::Testnet; + let genesis_header = genesis_block(network).header; + + let mut chaindb = Hammersbald::mem(network).unwrap(); + chaindb.init().unwrap(); + chaindb.init().unwrap(); + + let header_tip = chaindb.header_tip(); + assert!(header_tip.is_some(), "failed to get header for tip"); + assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) + } + + #[test] + fn init_recover_if_missing_tip_header() { + let network = Network::Testnet; + let genesis_header = genesis_block(network).header; + + let mut chaindb = Hammersbald::mem(network).unwrap(); + let missing_tip_header_hash: Hash = "6cfb35868c4465b7c289d7d5641563aa973db6a929655282a7bf95c8257f53ef".parse().unwrap(); + chaindb.store_header_tip(&missing_tip_header_hash).unwrap(); + + chaindb.init().unwrap(); + + let header_tip = chaindb.header_tip(); + assert!(header_tip.is_some(), "failed to get header for tip"); + assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) + } +} + + diff --git a/src/lib.rs b/src/lib.rs index c5da572..1be378b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,7 @@ pub mod dispatcher; pub mod p2p; pub mod error; pub mod chaindb; +#[cfg(feature = "default")] pub mod hammersbald; pub mod constructor; pub use error::Error; \ No newline at end of file From a9253e1992b033d58a949d28400fdacdb6813b42 Mon Sep 17 00:00:00 2001 From: Steve Myers Date: Wed, 26 Aug 2020 13:25:28 -0700 Subject: [PATCH 2/2] Remove unneeded ChainDB shutdown function. --- src/chaindb.rs | 3 --- src/hammersbald.rs | 6 ------ 2 files changed, 9 deletions(-) diff --git a/src/chaindb.rs b/src/chaindb.rs index dfe0212..f4b3e27 100644 --- a/src/chaindb.rs +++ b/src/chaindb.rs @@ -74,9 +74,6 @@ pub trait ChainDB: Send + Sync { /// Read header from the DB. fn fetch_header(&self, id: &sha256d::Hash) -> Result, Error>; - - /// Shutdown the DB. - fn shutdown(&mut self); } /// A header enriched with information about its position on the blockchain diff --git a/src/hammersbald.rs b/src/hammersbald.rs index 2690354..d13624e 100644 --- a/src/hammersbald.rs +++ b/src/hammersbald.rs @@ -179,12 +179,6 @@ impl ChainDB for Hammersbald { fn fetch_header(&self, id: &sha256d::Hash) -> Result, Error> { Ok(self.db.get_hash_keyed::(id)?.map(|(_, header)| header)) } - - /// Shutdown db - fn shutdown(&mut self) { - self.db.shutdown(); - debug!("shutdown chain db") - } } const HEADER_TIP_KEY: &[u8] = &[0u8; 1];