feat: add get-blockchain-db-stats command

Adds `get-db-stats` command. This returns the LMDB entry stats and
the total entry sizes for each internal blockchain db.

applications/tari_base_node/src/command_handler.rs

@@ -48,6 +48,7 @@ use tari_core::{
         LocalNodeCommsInterface,
     },
     blocks::BlockHeader,
+    chain_storage,
     chain_storage::{async_db::AsyncBlockchainDb, ChainHeader, LMDBDatabase},
     consensus::ConsensusManager,
     mempool::service::LocalMempoolService,
@@ -503,7 +504,7 @@ impl CommandHandler {
                             info_str,
                         ]);
                     }
-                    table.print_std();
+                    table.print_stdout();
 
                     println!("{} peer(s) known by this node", num_peers);
                 },
@@ -677,7 +678,7 @@ impl CommandHandler {
                         ]);
                     }
 
-                    table.print_std();
+                    table.print_stdout();
 
                     println!("{} active connection(s)", num_connections);
                 },
@@ -1054,6 +1055,76 @@ impl CommandHandler {
     pub(crate) fn get_software_updater(&self) -> SoftwareUpdaterHandle {
         self.software_updater.clone()
     }
+
+    pub fn get_blockchain_db_stats(&self) -> () {
+        let db = self.blockchain_db.clone();
+
+        fn stat_to_row(stat: &chain_storage::DbStat) -> Vec<String> {
+            row![
+                stat.name,
+                stat.entries,
+                stat.depth,
+                stat.branch_pages,
+                stat.leaf_pages,
+                stat.overflow_pages,
+            ]
+        }
+
+        self.executor.spawn(async move {
+            match db.get_stats().await {
+                Ok(stats) => {
+                    let mut table = Table::new();
+                    table.set_titles(vec![
+                        "Name",
+                        "Entries",
+                        "Depth",
+                        "Branch Pages",
+                        "Leaf Pages",
+                        "Overflow Pages",
+                    ]);
+                    table.add_row(stat_to_row(stats.global()));
+                    stats.db_stats().iter().for_each(|stat| {
+                        table.add_row(stat_to_row(stat));
+                    });
+                    table.print_stdout();
+                    println!();
+                    println!("Database page size: {} bytes", stats.global().psize);
+                },
+                Err(err) => {
+                    println!("{}", err);
+                    return;
+                },
+            }
+
+            println!();
+            println!("Totalling DB entry sizes. This may take a few seconds...");
+            println!();
+            match db.fetch_total_size_stats().await {
+                Ok(stats) => {
+                    println!();
+                    let mut table = Table::new();
+                    table.set_titles(vec!["Name", "Entries", "Total Size", "Avg. Size/Entry", "% of total"]);
+                    let total_db_size = stats.sizes().iter().map(|s| s.total()).sum::<u64>();
+                    stats.sizes().iter().for_each(|size| {
+                        let total = size.total() as f32 / 1024.0 / 1024.0;
+                        table.add_row(row![
+                            size.name,
+                            size.num_entries,
+                            format!("{:.2} MiB", total),
+                            format!("{} bytes", size.avg_bytes_per_entry()),
+                            format!("{:.2}%", (size.total() as f32 / total_db_size as f32) * 100.0)
+                        ])
+                    });
+                    table.print_stdout();
+                    println!();
+                    println!("Total data size: {:.2} MiB", total_db_size as f32 / 1024.0 / 1024.0);
+                },
+                Err(err) => {
+                    println!("{}", err);
+                },
+            }
+        });
+    }
 }
 
 async fn fetch_banned_peers(pm: &PeerManager) -> Result<Vec<Peer>, PeerManagerError> {

applications/tari_base_node/src/main.rs

@@ -27,6 +27,7 @@
 #![deny(unused_must_use)]
 #![deny(unreachable_patterns)]
 #![deny(unknown_lints)]
+#![allow(dead_code)]
 
 /// ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣠⣶⣿⣿⣿⣿⣶⣦⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
 /// ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣤⣾⣿⡿⠋⠀⠀⠀⠀⠉⠛⠿⣿⣿⣶⣤⣀⠀⠀⠀⠀⠀⠀⢰⣿⣾⣾⣾⣾⣾⣾⣾⣾⣾⣿⠀⠀⠀⣾⣾⣾⡀⠀⠀⠀⠀⢰⣾⣾⣾⣾⣿⣶⣶⡀⠀⠀⠀⢸⣾⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀

applications/tari_base_node/src/parser.rs

@@ -57,6 +57,7 @@ pub enum BaseNodeCommand {
     CheckForUpdates,
     Status,
     GetChainMetadata,
+    GetDbStats,
     GetPeer,
     ListPeers,
     DialPeer,
@@ -188,6 +189,9 @@ impl Parser {
             GetChainMetadata => {
                 self.command_handler.get_chain_meta();
             },
+            GetDbStats => {
+                self.command_handler.get_blockchain_db_stats();
+            },
             DialPeer => {
                 self.process_dial_peer(args);
             },
@@ -289,6 +293,9 @@ impl Parser {
             GetChainMetadata => {
                 println!("Gets your base node chain meta data");
             },
+            GetDbStats => {
+                println!("Gets your base node database stats");
+            },
             DialPeer => {
                 println!("Attempt to connect to a known peer");
             },

applications/tari_base_node/src/table.rs

@@ -48,6 +48,8 @@ impl<'t, 's> Table<'t, 's> {
 
     pub fn render<T: Write>(&self, out: &mut T) -> io::Result<()> {
         self.render_titles(out)?;
+        out.write_all(b"\n")?;
+        self.render_separator(out)?;
         if !self.rows.is_empty() {
             out.write_all(b"\n")?;
             self.render_rows(out)?;
@@ -56,7 +58,7 @@ impl<'t, 's> Table<'t, 's> {
         Ok(())
     }
 
-    pub fn print_std(&self) {
+    pub fn print_stdout(&self) {
         self.render(&mut io::stdout()).unwrap();
     }
 
@@ -106,6 +108,23 @@ impl<'t, 's> Table<'t, 's> {
         }
         Ok(())
     }
+
+    fn render_separator<T: Write>(&self, out: &mut T) -> io::Result<()> {
+        if let Some(rows_len) = self.rows.first().map(|r| r.len()) {
+            for i in 0..rows_len {
+                let width = self.col_width(i);
+                let pad_left = if i == 0 { "" } else { " " };
+                out.write_all(pad_left.as_bytes())?;
+                let sep = "-".repeat(width);
+                out.write_all(sep.as_bytes())?;
+                out.write_all(" ".as_bytes())?;
+                if i < rows_len - 1 {
+                    out.write_all(self.delim_str.as_bytes())?;
+                }
+            }
+        }
+        Ok(())
+    }
 }
 
 macro_rules! row {

base_layer/core/src/chain_storage/async_db.rs

@@ -32,6 +32,8 @@ use crate::{
         ChainHeader,
         ChainStorageError,
         CompleteDeletedBitmap,
+        DbBasicStats,
+        DbTotalSizeStats,
         DbTransaction,
         HistoricalBlock,
         HorizonData,
@@ -226,6 +228,10 @@ impl<B: BlockchainBackend + 'static> AsyncBlockchainDb<B> {
     make_async_fn!(fetch_block_hashes_from_header_tip(n: usize, offset: usize) -> Vec<HashOutput>, "fetch_block_hashes_from_header_tip");
 
     make_async_fn!(fetch_complete_deleted_bitmap_at(hash: HashOutput) -> CompleteDeletedBitmap, "fetch_deleted_bitmap");
+
+    make_async_fn!(get_stats() -> DbBasicStats, "get_stats");
+
+    make_async_fn!(fetch_total_size_stats() -> DbTotalSizeStats, "fetch_total_size_stats");
 }
 
 impl<B: BlockchainBackend + 'static> From<BlockchainDatabase<B>> for AsyncBlockchainDb<B> {

base_layer/core/src/chain_storage/blockchain_backend.rs

@@ -8,7 +8,9 @@ use crate::{
         ChainBlock,
         ChainHeader,
         ChainStorageError,
+        DbBasicStats,
         DbKey,
+        DbTotalSizeStats,
         DbTransaction,
         DbValue,
         HorizonData,
@@ -158,4 +160,11 @@ pub trait BlockchainBackend: Send + Sync {
     fn fetch_monero_seed_first_seen_height(&self, seed: &[u8]) -> Result<u64, ChainStorageError>;
 
     fn fetch_horizon_data(&self) -> Result<Option<HorizonData>, ChainStorageError>;
+
+    /// Returns basic database stats for each internal database, such as number of entries and page sizes. This call may
+    /// not apply to every database implementation.
+    fn get_stats(&self) -> Result<DbBasicStats, ChainStorageError>;
+    /// Returns total size information about each internal database. This call may be very slow and will obtain a read
+    /// lock for the duration.
+    fn fetch_total_size_stats(&self) -> Result<DbTotalSizeStats, ChainStorageError>;
 }

base_layer/core/src/chain_storage/blockchain_database.rs

@@ -35,6 +35,8 @@ use crate::{
         BlockchainBackend,
         ChainBlock,
         ChainHeader,
+        DbBasicStats,
+        DbTotalSizeStats,
         HistoricalBlock,
         HorizonData,
         MmrTree,
@@ -903,6 +905,18 @@ where B: BlockchainBackend
             chain_metadata.best_block().clone(),
         ))
     }
+
+    pub fn get_stats(&self) -> Result<DbBasicStats, ChainStorageError> {
+        let lock = self.db_read_access()?;
+        lock.get_stats()
+    }
+
+    /// Returns total size information about each internal database. This call may be very slow and will obtain a read
+    /// lock for the duration.
+    pub fn fetch_total_size_stats(&self) -> Result<DbTotalSizeStats, ChainStorageError> {
+        let lock = self.db_read_access()?;
+        lock.fetch_total_size_stats()
+    }
 }
 
 fn unexpected_result<T>(req: DbKey, res: DbValue) -> Result<T, ChainStorageError> {

base_layer/core/src/chain_storage/lmdb_db/lmdb.rs

@@ -77,6 +77,7 @@ where
     V: Serialize + Debug,
 {
     let val_buf = serialize(val)?;
+    trace!(target: LOG_TARGET, "LMDB: {} bytes inserted", val_buf.len());
     txn.access().put(&db, key, &val_buf, put::NOOVERWRITE).map_err(|e| {
         error!(
             target: LOG_TARGET,
@@ -387,3 +388,19 @@ where
     }
     Ok(result)
 }
+
+/// Fetches all the size of all key/values in the given DB. Returns the number of entries, the total size of all the
+/// keys and values in bytes.
+pub fn fetch_db_entry_sizes(txn: &ConstTransaction<'_>, db: &Database) -> Result<(u64, u64, u64), ChainStorageError> {
+    let access = txn.access();
+    let mut cursor = txn.cursor(db)?;
+    let mut num_entries = 0;
+    let mut total_key_size = 0;
+    let mut total_value_size = 0;
+    while let Some((key, value)) = cursor.next::<[u8], [u8]>(&access).to_opt()? {
+        num_entries += 1;
+        total_key_size += key.len() as u64;
+        total_value_size += value.len() as u64;
+    }
+    Ok((num_entries, total_key_size, total_value_size))
+}

base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs

@@ -27,6 +27,7 @@ use crate::{
         error::{ChainStorageError, OrNotFound},
         lmdb_db::{
             lmdb::{
+                fetch_db_entry_sizes,
                 lmdb_delete,
                 lmdb_delete_key_value,
                 lmdb_delete_keys_starting_with,
@@ -65,9 +66,12 @@ use crate::{
             LMDB_DB_UTXO_COMMITMENT_INDEX,
             LMDB_DB_UTXO_MMR_SIZE_INDEX,
         },
+        stats::DbTotalSizeStats,
         BlockchainBackend,
         ChainBlock,
         ChainHeader,
+        DbBasicStats,
+        DbSize,
         HorizonData,
         MmrTree,
         PrunedOutput,
@@ -361,6 +365,33 @@ impl LMDBDatabase {
         Ok(())
     }
 
+    fn all_dbs(&self) -> [(&'static str, &DatabaseRef); 19] {
+        [
+            ("metadata_db", &self.metadata_db),
+            ("headers_db", &self.headers_db),
+            ("header_accumulated_data_db", &self.header_accumulated_data_db),
+            ("block_accumulated_data_db", &self.block_accumulated_data_db),
+            ("block_hashes_db", &self.block_hashes_db),
+            ("utxos_db", &self.utxos_db),
+            ("inputs_db", &self.inputs_db),
+            ("txos_hash_to_index_db", &self.txos_hash_to_index_db),
+            ("kernels_db", &self.kernels_db),
+            ("kernel_excess_index", &self.kernel_excess_index),
+            ("kernel_excess_sig_index", &self.kernel_excess_sig_index),
+            ("kernel_mmr_size_index", &self.kernel_mmr_size_index),
+            ("output_mmr_size_index", &self.output_mmr_size_index),
+            ("utxo_commitment_index", &self.utxo_commitment_index),
+            ("orphans_db", &self.orphans_db),
+            (
+                "orphan_header_accumulated_data_db",
+                &self.orphan_header_accumulated_data_db,
+            ),
+            ("monero_seed_height_db", &self.monero_seed_height_db),
+            ("orphan_chain_tips_db", &self.orphan_chain_tips_db),
+            ("orphan_parent_map_index", &self.orphan_parent_map_index),
+        ]
+    }
+
     fn prune_output(
         &self,
         txn: &WriteTransaction<'_>,
@@ -2044,6 +2075,34 @@ impl BlockchainBackend for LMDBDatabase {
         let txn = self.read_transaction()?;
         fetch_horizon_data(&txn, &self.metadata_db)
     }
+
+    fn get_stats(&self) -> Result<DbBasicStats, ChainStorageError> {
+        let global = self.env.stat()?;
+
+        let txn = self.read_transaction()?;
+        let db_stats = self
+            .all_dbs()
+            .iter()
+            .map(|(name, db)| txn.db_stat(db).map(|s| (*name, s)))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        Ok(DbBasicStats::new(global, db_stats))
+    }
+
+    fn fetch_total_size_stats(&self) -> Result<DbTotalSizeStats, ChainStorageError> {
+        let txn = self.read_transaction()?;
+        self.all_dbs()
+            .iter()
+            .map(|(name, db)| {
+                fetch_db_entry_sizes(&txn, db).map(|(num_entries, total_key_size, total_value_size)| DbSize {
+                    name,
+                    num_entries,
+                    total_key_size,
+                    total_value_size,
+                })
+            })
+            .collect()
+    }
 }
 
 // Fetch the chain metadata

base_layer/core/src/chain_storage/mod.rs

@@ -92,5 +92,8 @@ pub use lmdb_db::{
     LMDB_DB_UTXOS,
 };
 
+mod stats;
+pub use stats::{DbBasicStats, DbSize, DbStat, DbTotalSizeStats};
+
 mod target_difficulties;
 pub use target_difficulties::TargetDifficulties;