diff --git a/.gitignore b/.gitignore index 35bf9d4..1a7535a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ target *.iml Cargo.lock +client.db.* +client.db \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 226ac1f..6138ef1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,19 +7,21 @@ homepage = "https://github.com/rust-bitcoin/murmel/" repository = "https://github.com/rust-bitcoin/murmel/" documentation = "https://github.com/rust-bitcoin/murmel/" description = "Murmel Bitcoin node" -keywords = [ "bitcoin" ] +keywords = ["bitcoin"] readme = "README.md" edition = "2018" +[features] +default = ["hammersbald"] + [lib] name = "murmel" path = "src/lib.rs" [dependencies] -lightning = { version ="0.0.9", optional=true } -bitcoin = { version= "0.21", features=["use-serde"]} +lightning = { version = "0.0.9", optional = true } +bitcoin = { version = "0.21", features = ["use-serde"] } bitcoin_hashes = "0.7" -hammersbald = { version= "2.4", features=["bitcoin_support"]} mio = "0.6" rand = "0.7" log = "0.4" @@ -28,8 +30,13 @@ byteorder = "1.2" lru-cache = "0.1.1" futures-preview = "=0.3.0-alpha.18" futures-timer = "0.3" -serde="1" -serde_derive="1" +serde = "1" +serde_derive = "1" +serde_cbor = "0.10" + +## optional +hammersbald = { version = "2.4", features = ["bitcoin_support"], optional = true } +rocksdb = { version = "0.15.0", default-features = false, features = ["lz4"], optional = true } [dev-dependencies] rustc-serialize = "0.3" diff --git a/src/chaindb.rs b/src/chaindb.rs index 5173788..f4b3e27 100644 --- a/src/chaindb.rs +++ b/src/chaindb.rs @@ -14,176 +14,66 @@ // limitations under the License. // //! -//! # Blockchain DB for a node +//! # Blockchain DB API for a node //! use std::sync::{Arc, RwLock}; -use std::path::Path; -use bitcoin::{BitcoinHash, Network}; +use bitcoin::BitcoinHash; use bitcoin::blockdata::block::BlockHeader; -use bitcoin::blockdata::constants::genesis_block; use bitcoin_hashes::sha256d; -use hammersbald::{BitcoinAdaptor, HammersbaldAPI, persistent, transient}; use crate::error::Error; -use crate::headercache::{CachedHeader, HeaderCache}; -use log::{debug, info, warn, error}; +use crate::headercache::CachedHeader; + use serde_derive::{Serialize, Deserialize}; /// Shared handle to a database storing the block chain /// protected by an RwLock -pub type SharedChainDB = Arc>; - -/// Database storing the block chain -pub struct ChainDB { - db: BitcoinAdaptor, - headercache: HeaderCache, - network: Network, -} - -impl ChainDB { - /// Create an in-memory database instance - pub fn mem(network: Network) -> Result { - info!("working with in memory chain db"); - let db = BitcoinAdaptor::new(transient(2)?); - let headercache = HeaderCache::new(network); - Ok(ChainDB { db, network, headercache }) - } +pub type SharedChainDB = Arc>>; - /// Create or open a persistent database instance identified by the path - pub fn new(path: &Path, network: Network) -> Result { - let basename = path.to_str().unwrap().to_string(); - let db = BitcoinAdaptor::new(persistent((basename.clone()).as_str(), 100, 2)?); - let headercache = HeaderCache::new(network); - Ok(ChainDB { db, network, headercache }) - } +/// Blockchain DB API for a client node. +pub trait ChainDB: Send + Sync { - /// Initialize caches - pub fn init(&mut self) -> Result<(), Error> { - self.init_headers()?; - Ok(()) - } + /// Initialize caches. + fn init(&mut self) -> Result<(), Error>; /// Batch updates. Updates are permanent after finishing a batch. - pub fn batch(&mut self) -> Result<(), Error> { - self.db.batch()?; - Ok(()) - } + fn batch(&mut self) -> Result<(), Error>; - fn init_headers(&mut self) -> Result<(), Error> { - if let Some(tip) = self.fetch_header_tip()? { - info!("reading stored header chain from tip {}", tip); - if self.fetch_header(&tip)?.is_some() { - let mut h = tip; - while let Some(stored) = self.fetch_header(&h)? { - debug!("read stored header {}", &stored.bitcoin_hash()); - self.headercache.add_header_unchecked(&h, &stored); - if stored.header.prev_blockhash != sha256d::Hash::default() { - h = stored.header.prev_blockhash; - } else { - break; - } - } - self.headercache.reverse_trunk(); - info!("read {} headers", self.headercache.len()); - } else { - warn!("unable to read header for tip {}", tip); - self.init_to_genesis()?; - } - } else { - info!("no header tip found"); - self.init_to_genesis()?; - } - Ok(()) - } + /// Store a header. + fn add_header(&mut self, header: &BlockHeader) -> Result>, Option>)>, Error>; - fn init_to_genesis(&mut self) -> Result<(), Error> { - let genesis = genesis_block(self.network).header; - if let Some((cached, _, _)) = self.headercache.add_header(&genesis)? { - info!("initialized with genesis header {}", genesis.bitcoin_hash()); - self.db.put_hash_keyed(&cached.stored)?; - self.db.batch()?; - self.store_header_tip(&cached.bitcoin_hash())?; - self.db.batch()?; - } else { - error!("failed to initialize with genesis header"); - return Err(Error::NoTip); - } - Ok(()) - } + /// Return position of hash on trunk if hash is on trunk. + fn pos_on_trunk(&self, hash: &sha256d::Hash) -> Option; - /// Store a header - pub fn add_header(&mut self, header: &BlockHeader) -> Result>, Option>)>, Error> { - if let Some((cached, unwinds, forward)) = self.headercache.add_header(header)? { - self.db.put_hash_keyed(&cached.stored)?; - if let Some(forward) = forward.clone() { - if forward.len() > 0 { - self.store_header_tip(forward.last().unwrap())?; - } - } - return Ok(Some((cached.stored, unwinds, forward))); - } - Ok(None) - } + /// Iterate trunk [from .. tip]. + fn iter_trunk<'a>(&'a self, from: u32) -> Box + 'a>; - /// return position of hash on trunk if hash is on trunk - pub fn pos_on_trunk(&self, hash: &sha256d::Hash) -> Option { - self.headercache.pos_on_trunk(hash) - } + /// Iterate trunk [genesis .. from] in reverse order from is the tip if not specified. + fn iter_trunk_rev<'a>(&'a self, from: Option) -> Box + 'a>; - /// iterate trunk [from .. tip] - pub fn iter_trunk<'a>(&'a self, from: u32) -> impl Iterator + 'a { - self.headercache.iter_trunk(from) - } - - /// iterate trunk [genesis .. from] in reverse order from is the tip if not specified - pub fn iter_trunk_rev<'a>(&'a self, from: Option) -> impl Iterator + 'a { - self.headercache.iter_trunk_rev(from) - } - - /// retrieve the id of the block/header with most work - pub fn header_tip(&self) -> Option { - self.headercache.tip() - } - - /// Fetch a header by its id from cache - pub fn get_header(&self, id: &sha256d::Hash) -> Option { - self.headercache.get_header(id) - } + /// Retrieve the id of the block/header with most work. + fn header_tip(&self) -> Option; - /// Fetch a header by its id from cache - pub fn get_header_for_height(&self, height: u32) -> Option { - self.headercache.get_header_for_height(height) - } + /// Fetch a header by its id from cache. + fn get_header(&self, id: &sha256d::Hash) -> Option; - /// locator for getheaders message - pub fn header_locators(&self) -> Vec { - self.headercache.locator_hashes() - } + /// Fetch a header by its id from cache. + fn get_header_for_height(&self, height: u32) -> Option; - /// Store the header id with most work - pub fn store_header_tip(&mut self, tip: &sha256d::Hash) -> Result<(), Error> { - self.db.put_keyed_encodable(HEADER_TIP_KEY, tip)?; - Ok(()) - } + /// Locator for getheaders message. + fn header_locators(&self) -> Vec; - /// Find header id with most work - pub fn fetch_header_tip(&self) -> Result, Error> { - Ok(self.db.get_keyed_decodable::(HEADER_TIP_KEY)?.map(|(_, h)| h.clone())) - } + /// Store the header id with most work. + fn store_header_tip(&mut self, tip: &sha256d::Hash) -> Result<(), Error>; - /// Read header from the DB - pub fn fetch_header(&self, id: &sha256d::Hash) -> Result, Error> { - Ok(self.db.get_hash_keyed::(id)?.map(|(_, header)| header)) - } + /// Find header id with most work. + fn fetch_header_tip(&self) -> Result, Error>; - /// Shutdown db - pub fn shutdown(&mut self) { - self.db.shutdown(); - debug!("shutdown chain db") - } + /// Read header from the DB. + fn fetch_header(&self, id: &sha256d::Hash) -> Result, Error>; } /// A header enriched with information about its position on the blockchain @@ -204,45 +94,4 @@ impl BitcoinHash for StoredHeader { } } -const HEADER_TIP_KEY: &[u8] = &[0u8; 1]; - -#[cfg(test)] -mod test { - use bitcoin::{Network, BitcoinHash}; - use bitcoin_hashes::sha256d::Hash; - use bitcoin::blockdata::constants::genesis_block; - - use crate::chaindb::ChainDB; - - #[test] - fn init_tip_header() { - let network = Network::Testnet; - let genesis_header = genesis_block(network).header; - - let mut chaindb = ChainDB::mem(network).unwrap(); - chaindb.init().unwrap(); - chaindb.init().unwrap(); - - let header_tip = chaindb.header_tip(); - assert!(header_tip.is_some(), "failed to get header for tip"); - assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) - } - - #[test] - fn init_recover_if_missing_tip_header() { - let network = Network::Testnet; - let genesis_header = genesis_block(network).header; - - let mut chaindb = ChainDB::mem(network).unwrap(); - let missing_tip_header_hash: Hash = "6cfb35868c4465b7c289d7d5641563aa973db6a929655282a7bf95c8257f53ef".parse().unwrap(); - chaindb.store_header_tip(&missing_tip_header_hash).unwrap(); - - chaindb.init().unwrap(); - - let header_tip = chaindb.header_tip(); - assert!(header_tip.is_some(), "failed to get header for tip"); - assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) - } -} - diff --git a/src/constructor.rs b/src/constructor.rs index 748240f..6d55e89 100644 --- a/src/constructor.rs +++ b/src/constructor.rs @@ -24,7 +24,7 @@ use bitcoin::{ constants::Network } }; -use crate::chaindb::{ChainDB, SharedChainDB}; + use crate::dispatcher::Dispatcher; use crate::dns::dns_seed; use crate::error::Error; @@ -55,6 +55,7 @@ use bitcoin::network::message::NetworkMessage; use bitcoin::network::message::RawNetworkMessage; use crate::p2p::BitcoinP2PConfig; use std::time::Duration; +use crate::chaindb::{SharedChainDB, ChainDB}; const MAX_PROTOCOL_VERSION: u32 = 70001; const USER_AGENT: &'static str = concat!("/Murmel:", env!("CARGO_PKG_VERSION"), '/'); @@ -67,18 +68,35 @@ pub struct Constructor { } impl Constructor { - /// open DBs + /// open DB pub fn open_db(path: Option<&Path>, network: Network, _birth: u64) -> Result { - let mut chaindb = - if let Some(path) = path { - ChainDB::new(path, network)? - } else { - ChainDB::mem(network)? - }; + let mut chaindb = Constructor::new_db(path, network)?; chaindb.init()?; Ok(Arc::new(RwLock::new(chaindb))) } + /// new Hammersbald DB + #[cfg(feature = "default")] + fn new_db(path: Option<&Path>, network:Network) -> Result, Error> { + use crate::hammersbald::Hammersbald; + if let Some(path) = path { + Hammersbald::new(path, network) + } else { + Hammersbald::mem(network) + } + } + + /// new RocksDB + #[cfg(feature = "rocksdb")] + fn new_db(path: Option<&Path>, network:Network) -> Result, Error> { + use crate::rocksdb::RocksDB; + if let Some(path) = path { + RocksDB::new(path, network) + } else { + Err(Error::RocksDB("RocksDB doesn't support mem DB, path is required.".to_string())) + } + } + /// Construct the stack pub fn new(network: Network, listen: Vec, chaindb: SharedChainDB) -> Result { const BACK_PRESSURE: usize = 10; diff --git a/src/error.rs b/src/error.rs index e35956c..4846f05 100644 --- a/src/error.rs +++ b/src/error.rs @@ -22,7 +22,13 @@ use bitcoin::consensus::encode; use bitcoin::util; use bitcoin::util::bip158; + +#[cfg(feature="default")] use hammersbald; + +#[cfg(feature="rocksdb")] +use rocksdb; + use std::convert; use std::fmt; use std::io; @@ -52,7 +58,10 @@ pub enum Error { /// Bitcoin serialize error Serialize(encode::Error), /// Hammersbald error + #[cfg(feature="default")] Hammersbald(hammersbald::Error), + #[cfg(feature="rocksdb")] + RocksDB(String), /// Handshake failure Handshake, /// lost connection @@ -76,7 +85,10 @@ impl std::error::Error for Error { Error::BadMerkleRoot => None, Error::IO(ref err) => Some(err), Error::Util(ref err) => Some(err), + #[cfg(feature="default")] Error::Hammersbald(ref err) => Some(err), + #[cfg(feature="rocksdb")] + Error::RocksDB(_) => None, Error::Serialize(ref err) => Some(err), Error::Handshake => None, Error::Lost(_) => None @@ -101,7 +113,10 @@ impl fmt::Display for Error { // The underlying errors already impl `Display`, so we defer to their implementations. Error::IO(ref err) => write!(f, "IO error: {}", err), Error::Util(ref err) => write!(f, "Util error: {}", err), + #[cfg(feature="default")] Error::Hammersbald(ref err) => write!(f, "Hammersbald error: {}", err), + #[cfg(feature="rocksdb")] + Error::RocksDB(ref err) => write!(f, "RocksDB error: {}", err), Error::Serialize(ref err) => write!(f, "Serialize error: {}", err), } } @@ -137,12 +152,20 @@ impl convert::From for Error { } } +#[cfg(feature="default")] impl convert::From for Error { fn from(err: hammersbald::Error) -> Error { Error::Hammersbald(err) } } +#[cfg(feature="rocksdb")] +impl convert::From for Error { + fn from(err: rocksdb::Error) -> Error { + Error::RocksDB(err.into_string()) + } +} + impl convert::From for Error { fn from(err: encode::Error) -> Error { Error::Serialize(err) diff --git a/src/hammersbald.rs b/src/hammersbald.rs new file mode 100644 index 0000000..8ee87cd --- /dev/null +++ b/src/hammersbald.rs @@ -0,0 +1,245 @@ +// +// Copyright 2018-2019 Tamas Blummer +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//! +//! # Blockchain DB for a node +//! + +use std::path::Path; + +use bitcoin::{BitcoinHash, Network}; +use bitcoin::blockdata::block::BlockHeader; +use bitcoin::blockdata::constants::genesis_block; + +use bitcoin_hashes::sha256d; +use hammersbald::{BitcoinAdaptor, HammersbaldAPI, persistent, transient}; + +use crate::error::Error; +use crate::headercache::{CachedHeader, HeaderCache}; +use log::{debug, info, warn, error}; +use crate::chaindb::StoredHeader; +use crate::chaindb::ChainDB; + +/// Database storing the block chain +pub struct Hammersbald { + db: BitcoinAdaptor, + headercache: HeaderCache, + network: Network, +} + + +impl Hammersbald { + + /// Create an in-memory database instance + pub fn mem(network: Network) -> Result, Error> { + info!("working with in memory chain db"); + let db = BitcoinAdaptor::new(transient(2)?); + let headercache = HeaderCache::new(network); + Ok(Box::from(Hammersbald { db, network, headercache })) + } + + /// Create or open a persistent database instance identified by the path + pub fn new(path: &Path, network: Network) -> Result, Error> { + let basename = path.to_str().unwrap().to_string(); + let db = BitcoinAdaptor::new(persistent((basename.clone()).as_str(), 100, 2)?); + let headercache = HeaderCache::new(network); + Ok(Box::from(Hammersbald { db, network, headercache })) + } + + fn init_headers(&mut self) -> Result<(), Error> { + if let Some(tip) = self.fetch_header_tip()? { + info!("reading stored header chain from tip {}", tip); + if self.fetch_header(&tip)?.is_some() { + let mut h = tip; + while let Some(stored) = self.fetch_header(&h)? { + debug!("read stored header {}", &stored.bitcoin_hash()); + self.headercache.add_header_unchecked(&h, &stored); + if stored.header.prev_blockhash != sha256d::Hash::default() { + h = stored.header.prev_blockhash; + } else { + break; + } + } + self.headercache.reverse_trunk(); + info!("read {} headers", self.headercache.len()); + } else { + warn!("unable to read header for tip {}", tip); + self.init_to_genesis()?; + } + } else { + info!("no header tip found"); + self.init_to_genesis()?; + } + Ok(()) + } + + fn init_to_genesis(&mut self) -> Result<(), Error> { + let genesis = genesis_block(self.network).header; + if let Some((cached, _, _)) = self.headercache.add_header(&genesis)? { + info!("initialized with genesis header {}", genesis.bitcoin_hash()); + self.db.put_hash_keyed(&cached.stored)?; + self.db.batch()?; + self.store_header_tip(&cached.bitcoin_hash())?; + self.db.batch()?; + } else { + error!("failed to initialize with genesis header"); + return Err(Error::NoTip); + } + Ok(()) + } +} + +impl ChainDB for Hammersbald { + + /// Initialize caches + fn init(&mut self) -> Result<(), Error> { + self.init_headers()?; + Ok(()) + } + + /// Batch updates. Updates are permanent after finishing a batch. + fn batch(&mut self) -> Result<(), Error> { + self.db.batch()?; + Ok(()) + } + + /// Store a header + fn add_header(&mut self, header: &BlockHeader) -> Result>, Option>)>, Error> { + if let Some((cached, unwinds, forward)) = self.headercache.add_header(header)? { + self.db.put_hash_keyed(&cached.stored)?; + if let Some(forward) = forward.clone() { + if forward.len() > 0 { + self.store_header_tip(forward.last().unwrap())?; + } + } + return Ok(Some((cached.stored, unwinds, forward))); + } + Ok(None) + } + + /// return position of hash on trunk if hash is on trunk + fn pos_on_trunk(&self, hash: &sha256d::Hash) -> Option { + self.headercache.pos_on_trunk(hash) + } + + /// iterate trunk [from .. tip] + fn iter_trunk<'a>(&'a self, from: u32) -> Box +'a> { + self.headercache.iter_trunk(from) + } + + /// iterate trunk [genesis .. from] in reverse order from is the tip if not specified + fn iter_trunk_rev<'a>(&'a self, from: Option) -> Box +'a> { + self.headercache.iter_trunk_rev(from) + } + + /// retrieve the id of the block/header with most work + fn header_tip(&self) -> Option { + self.headercache.tip() + } + + /// Fetch a header by its id from cache + fn get_header(&self, id: &sha256d::Hash) -> Option { + self.headercache.get_header(id) + } + + /// Fetch a header by its id from cache + fn get_header_for_height(&self, height: u32) -> Option { + self.headercache.get_header_for_height(height) + } + + /// locator for getheaders message + fn header_locators(&self) -> Vec { + self.headercache.locator_hashes() + } + + /// Store the header id with most work + fn store_header_tip(&mut self, tip: &sha256d::Hash) -> Result<(), Error> { + self.db.put_keyed_encodable(HEADER_TIP_KEY, tip)?; + Ok(()) + } + + /// Find header id with most work + fn fetch_header_tip(&self) -> Result, Error> { + Ok(self.db.get_keyed_decodable::(HEADER_TIP_KEY)?.map(|(_, h)| h.clone())) + } + + /// Read header from the DB + fn fetch_header(&self, id: &sha256d::Hash) -> Result, Error> { + Ok(self.db.get_hash_keyed::(id)?.map(|(_, header)| header)) + } +} + +const HEADER_TIP_KEY: &[u8] = &[0u8; 1]; + +#[cfg(test)] +mod test { + use bitcoin::{Network, BitcoinHash}; + use bitcoin_hashes::sha256d::Hash; + use bitcoin::blockdata::constants::genesis_block; + + use log::debug; + + use crate::hammersbald::Hammersbald; + + use std::sync::Once; + + static INIT: Once = Once::new(); + + /// Setup function that is only run once, even if called multiple times. + fn setup() { + INIT.call_once(|| { + simple_logger::init().unwrap(); + }); + } + + #[test] + fn init_tip_header() { + setup(); + + let network = Network::Testnet; + let genesis_header = genesis_block(network).header; + + //let path = Path::new("test"); + let mut chaindb = Hammersbald::mem(network).unwrap(); + debug!("init 1"); + chaindb.init().unwrap(); + debug!("init 2"); + chaindb.init().unwrap(); + + let header_tip = chaindb.header_tip(); + assert!(header_tip.is_some(), "failed to get header for tip"); + assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) + } + + #[test] + fn init_recover_if_missing_tip_header() { + setup(); + + let network = Network::Testnet; + let genesis_header = genesis_block(network).header; + + let mut chaindb = Hammersbald::mem(network).unwrap(); + let missing_tip_header_hash: Hash = "6cfb35868c4465b7c289d7d5641563aa973db6a929655282a7bf95c8257f53ef".parse().unwrap(); + chaindb.store_header_tip(&missing_tip_header_hash).unwrap(); + + chaindb.init().unwrap(); + + let header_tip = chaindb.header_tip(); + assert!(header_tip.is_some(), "failed to get header for tip"); + assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) + } +} + + diff --git a/src/headercache.rs b/src/headercache.rs index f154157..339c9c1 100644 --- a/src/headercache.rs +++ b/src/headercache.rs @@ -34,6 +34,8 @@ use std::{ collections::HashMap }; +use log::debug; + #[derive(Clone)] pub struct CachedHeader { pub stored : StoredHeader, @@ -137,6 +139,7 @@ impl HeaderCache { pub fn add_header(&mut self, header: &BlockHeader) -> Result>, Option>)>, Error> { if self.headers.get(&header.bitcoin_hash()).is_some() { // ignore already known header + debug!("ignore already known header"); return Ok(None); } if header.prev_blockhash != Sha256dHash::default() { diff --git a/src/lib.rs b/src/lib.rs index c5da572..f8cae6a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,5 +39,7 @@ pub mod p2p; pub mod error; pub mod chaindb; pub mod constructor; +#[cfg(feature = "hammersbald")] pub mod hammersbald; +#[cfg(feature = "rocksdb")] pub mod rocksdb; pub use error::Error; \ No newline at end of file diff --git a/src/rocksdb.rs b/src/rocksdb.rs new file mode 100644 index 0000000..b74ad0f --- /dev/null +++ b/src/rocksdb.rs @@ -0,0 +1,309 @@ +// +// Copyright 2018-2019 Tamas Blummer +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//! +//! # Blockchain DB for a node +//! + +use std::path::Path; + +use bitcoin::{BitcoinHash, Network}; +use bitcoin::blockdata::block::BlockHeader; +use bitcoin::blockdata::constants::genesis_block; + +use bitcoin_hashes::sha256d; + +use crate::error::Error; +use crate::headercache::{CachedHeader, HeaderCache}; +use log::{debug, info, warn, error}; +use crate::chaindb::StoredHeader; +use crate::chaindb::ChainDB; +use rocksdb::DB; + +/// Database storing the block chain +pub struct RocksDB { + db: DB, + header_cache: HeaderCache, + network: Network, +} + +impl RocksDB { + + /// Create or open a persistent database instance identified by the path + pub fn new(path: &Path, network: Network) -> Result, Error> { + info!("working with chain db: {}", &path.to_str().unwrap()); + let db = rocksdb::DB::open_default(path).unwrap(); // TODO convert rocksdb::error to murmel::Error + let header_cache = HeaderCache::new(network); + Ok(Box::from(RocksDB { db, header_cache, network })) + } + + fn init_headers(&mut self) -> Result<(), Error> { + if let Some(tip) = self.fetch_header_tip()? { + info!("reading stored header chain from tip {}", tip); + if self.fetch_header(&tip)?.is_some() { + let mut h = tip; + while let Some(stored) = self.fetch_header(&h)? { + debug!("read stored header {}", &stored.bitcoin_hash()); + self.header_cache.add_header_unchecked(&h, &stored); + if stored.header.prev_blockhash != sha256d::Hash::default() { + h = stored.header.prev_blockhash; + } else { + break; + } + } + self.header_cache.reverse_trunk(); + info!("read {} headers", self.header_cache.len()); + } else { + warn!("unable to read header for tip {}", tip); + self.init_to_genesis()?; + } + } else { + info!("no header tip found"); + self.init_to_genesis()?; + } + Ok(()) + } + + fn init_to_genesis(&mut self) -> Result<(), Error> { + let genesis = genesis_block(self.network).header; + if let Some((cached, _, _)) = self.header_cache.add_header(&genesis)? { + info!("initialized with genesis header {}", genesis.bitcoin_hash()); + self.db.put(&cached.stored.bitcoin_hash()[..], serde_cbor::to_vec(&cached.stored).unwrap().as_slice()).unwrap(); + self.store_header_tip(&cached.bitcoin_hash())?; + } else { + error!("failed to initialize with genesis header"); + return Err(Error::NoTip); + } + Ok(()) + } +} + +impl ChainDB for RocksDB { + /// Initialize caches + fn init(&mut self) -> Result<(), Error> { + self.init_headers()?; + Ok(()) + } + + /// Batch updates. Updates are permanent after finishing a batch. + fn batch(&mut self) -> Result<(), Error> { + self.db.flush().unwrap(); // TODO convert rocksdb::error to murmel::Error + Ok(()) + } + + /// Store a header + fn add_header(&mut self, header: &BlockHeader) -> Result>, Option>)>, Error> { + if let Some((cached, unwinds, forward)) = self.header_cache.add_header(header)? { + // TODO convert serde_cbor::error::Error and rocksdb::error to murmel::Error + self.db.put(&cached.stored.bitcoin_hash()[..], serde_cbor::to_vec(&cached.stored).unwrap().as_slice()).unwrap(); + if let Some(forward) = forward.clone() { + if forward.len() > 0 { + self.store_header_tip(forward.last().unwrap())?; + } + } + return Ok(Some((cached.stored, unwinds, forward))); + } + Ok(None) + } + + /// return position of hash on trunk if hash is on trunk + fn pos_on_trunk(&self, hash: &sha256d::Hash) -> Option { + self.header_cache.pos_on_trunk(hash) + } + + /// iterate trunk [from .. tip] + fn iter_trunk<'a>(&'a self, from: u32) -> Box + 'a> { + self.header_cache.iter_trunk(from) + } + + /// iterate trunk [genesis .. from] in reverse order from is the tip if not specified + fn iter_trunk_rev<'a>(&'a self, from: Option) -> Box + 'a> { + self.header_cache.iter_trunk_rev(from) + } + + /// retrieve the id of the block/header with most work + fn header_tip(&self) -> Option { + self.header_cache.tip() + } + + /// Fetch a header by its id from cache + fn get_header(&self, id: &sha256d::Hash) -> Option { + self.header_cache.get_header(id) + } + + /// Fetch a header by its id from cache + fn get_header_for_height(&self, height: u32) -> Option { + self.header_cache.get_header_for_height(height) + } + + /// locator for getheaders message + fn header_locators(&self) -> Vec { + self.header_cache.locator_hashes() + } + + /// Store the header id with most work + fn store_header_tip(&mut self, tip: &sha256d::Hash) -> Result<(), Error> { + // TODO convert serde_cbor::error::Error and rocksdb::error to murmel::Error + self.db.put(HEADER_TIP_KEY, serde_cbor::to_vec(&tip).unwrap().as_slice()).unwrap(); + Ok(()) + } + + /// Find header id with most work + fn fetch_header_tip(&self) -> Result, Error> { + // TODO convert serde_cbor::error::Error and rocksdb::error to murmel::Error + if let Some(value) = self.db.get(HEADER_TIP_KEY).unwrap() { + Ok(serde_cbor::from_slice(value.as_slice()).unwrap()) + } else { + Ok(None) + } + } + + /// Read header from the DB + fn fetch_header(&self, id: &sha256d::Hash) -> Result, Error> { + // TODO convert serde_cbor::error::Error and rocksdb::error to murmel::Error + if let Some(value) = self.db.get(id.to_vec()).unwrap() { + Ok(serde_cbor::from_slice(value.as_slice()).unwrap()) + } else { + Ok(None) + } + } +} + +const HEADER_TIP_KEY:&[u8] = b"HEADER_TIP"; + +#[cfg(test)] +mod test { + use bitcoin::{Network, BitcoinHash}; + use bitcoin_hashes::sha256d::Hash; + use bitcoin::blockdata::constants::genesis_block; + + use log::debug; + + use crate::rocksdb::RocksDB; + use std::path::{Path, PathBuf}; + use rocksdb::{Options, DB}; + + use std::sync::Once; + + static INIT: Once = Once::new(); + + /// Setup function that is only run once, even if called multiple times. + fn setup() { + INIT.call_once(|| { + simple_logger::init().unwrap(); + }); + } + + #[test] + fn add_fetch_header() { + setup(); + + let network = Network::Testnet; + let genesis_header = genesis_block(network).header; + + let db_path = DBPath::new("_add_fetch_header_test"); + let path = db_path.path.as_path(); + let mut chaindb = RocksDB::new(path, network).unwrap(); + + let genesis = genesis_block(network).header; + let (stored_header, _unwinds, _forward) = chaindb.add_header(&genesis).unwrap().unwrap(); + + let fetched_header = chaindb.fetch_header(&stored_header.bitcoin_hash()).unwrap().unwrap(); + assert_eq!(fetched_header.header, genesis_header); + assert_eq!(fetched_header.height, 0); + assert_eq!(fetched_header.log2work, 32.00002201394726); + } + + #[test] + fn init_tip_header() { + setup(); + + let network = Network::Testnet; + let genesis_header = genesis_block(network).header; + + let db_path = DBPath::new("_init_tip_header_test"); + let path = db_path.path.as_path(); + let mut chaindb = RocksDB::new(path, network).unwrap(); + debug!("init 1"); + chaindb.init().unwrap(); + debug!("init 2"); + chaindb.init().unwrap(); + + let header_tip = chaindb.header_tip(); + assert!(header_tip.is_some(), "failed to get header for tip"); + assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) + } + + #[test] + fn init_recover_if_missing_tip_header() { + setup(); + + let network = Network::Testnet; + let genesis_header = genesis_block(network).header; + + let db_path = DBPath::new("_init_recover_if_missing_tip_header_test"); + let path = db_path.path.as_path(); + let mut chaindb = RocksDB::new(path, network).unwrap(); + let missing_tip_header_hash: Hash = "6cfb35868c4465b7c289d7d5641563aa973db6a929655282a7bf95c8257f53ef".parse().unwrap(); + chaindb.store_header_tip(&missing_tip_header_hash).unwrap(); + + chaindb.init().unwrap(); + + let header_tip = chaindb.header_tip(); + assert!(header_tip.is_some(), "failed to get header for tip"); + assert!(header_tip.unwrap().stored.bitcoin_hash().eq(&genesis_header.bitcoin_hash())) + } + + // below is from rust-rocksdb tests/util mod + + /// Temporary database path which calls DB::Destroy when DBPath is dropped. + pub struct DBPath { + #[allow(dead_code)] + dir: tempfile::TempDir, // kept for cleaning up during drop + path: PathBuf, + } + + impl DBPath { + /// Produces a fresh (non-existent) temporary path which will be DB::destroy'ed automatically. + pub fn new(prefix: &str) -> DBPath { + let dir = tempfile::Builder::new() + .prefix(prefix) + .tempdir() + .expect("Failed to create temporary path for db."); + let path = dir.path().join("db"); + + DBPath { dir, path } + } + } + + impl Drop for DBPath { + fn drop(&mut self) { + let opts = Options::default(); + DB::destroy(&opts, &self.path).expect("Failed to destroy temporary DB"); + } + } + + /// Convert a DBPath ref to a Path ref. + /// We don't implement this for DBPath values because we want them to + /// exist until the end of their scope, not get passed in to functions and + /// dropped early. + impl AsRef for &DBPath { + fn as_ref(&self) -> &Path { + &self.path + } + } +} + +