feat: add get-db-stats command (#3274)

Description --- Adds `get-db-stats` command. This returns the LMDB entry stats and the total entry sizes for each internal blockchain db. Motivation and Context --- Useful in debugging database sizes. At height 26215 ``` >> get-db-stats Name | Entries | Depth | Branch Pages | Leaf Pages | Overflow Pages --------------------------------- | ------- | ----- | ------------ | ---------- | -------------- metadata_db | 5 | 1 | 0 | 1 | 23 headers_db | 26218 | 3 | 19 | 4057 | 0 header_accumulated_data_db | 26218 | 3 | 6 | 1010 | 0 block_accumulated_data_db | 26218 | 3 | 99 | 21775 | 1087 block_hashes_db | 26218 | 3 | 9 | 468 | 0 utxos_db | 747509 | 5 | 15022 | 373736 | 0 inputs_db | 560784 | 5 | 5928 | 99085 | 0 txos_hash_to_index_db | 747509 | 4 | 533 | 35242 | 0 kernels_db | 262572 | 5 | 2556 | 39712 | 0 kernel_excess_index | 262572 | 4 | 172 | 11999 | 0 kernel_excess_sig_index | 262572 | 4 | 432 | 15393 | 0 kernel_mmr_size_index | 26218 | 2 | 1 | 170 | 0 output_mmr_size_index | 26218 | 3 | 3 | 438 | 0 utxo_commitment_index | 186725 | 4 | 119 | 6889 | 0 orphans_db | 720 | 3 | 9 | 554 | 1674 orphan_header_accumulated_data_db | 718 | 2 | 1 | 47 | 0 monero_seed_height_db | 1 | 1 | 0 | 1 | 0 orphan_chain_tips_db | 16 | 1 | 0 | 1 | 0 orphan_parent_map_index | 720 | 2 | 1 | 22 | 0 19 databases, page size: 4096 bytes Totalling DB entry sizes. This may take a few seconds... >> Name | Entries | Total Size | Avg. Size/Entry | % of total --------------------------------- | ------- | ---------- | --------------- | ---------- metadata_db | 5 | 0.09 MiB | 18859 bytes | 0.01% headers_db | 26218 | 12.34 MiB | 493 bytes | 0.90% header_accumulated_data_db | 26218 | 3.40 MiB | 136 bytes | 0.25% block_accumulated_data_db | 26218 | 39.25 MiB | 1569 bytes | 2.86% block_hashes_db | 26218 | 1.00 MiB | 40 bytes | 0.07% utxos_db | 747509 | 789.72 MiB | 1107 bytes | 57.51% inputs_db | 560784 | 263.80 MiB | 493 bytes | 19.21% txos_hash_to_index_db | 747509 | 84.83 MiB | 119 bytes | 6.18% kernels_db | 262572 | 90.40 MiB | 361 bytes | 6.58% kernel_excess_index | 262572 | 29.05 MiB | 116 bytes | 2.12% kernel_excess_sig_index | 262572 | 37.06 MiB | 148 bytes | 2.70% kernel_mmr_size_index | 26218 | 0.40 MiB | 16 bytes | 0.03% output_mmr_size_index | 26218 | 1.40 MiB | 56 bytes | 0.10% utxo_commitment_index | 186725 | 12.82 MiB | 72 bytes | 0.93% orphans_db | 720 | 7.51 MiB | 10932 bytes | 0.55% orphan_header_accumulated_data_db | 718 | 0.11 MiB | 160 bytes | 0.01% monero_seed_height_db | 1 | 0.00 MiB | 40 bytes | 0.00% orphan_chain_tips_db | 16 | 0.00 MiB | 72 bytes | 0.00% orphan_parent_map_index | 720 | 0.05 MiB | 72 bytes | 0.00% Total data size: 1373.23 MiB ``` How Has This Been Tested? --- Manually by running `get-db-stats`
tari-project · Sep 2, 2021 · d785f4f · d785f4f
1 parent 9837703
commit d785f4f
Show file tree

Hide file tree

Showing 17 changed files with 655 additions and 112 deletions.
diff --git a/applications/tari_base_node/src/command_handler.rs b/applications/tari_base_node/src/command_handler.rs
@@ -504,7 +504,7 @@ impl CommandHandler {
                             info_str,
                         ]);
                     }
-                    table.print_std();
+                    table.print_stdout();
 
                     println!("{} peer(s) known by this node", num_peers);
                 },
@@ -678,7 +678,7 @@ impl CommandHandler {
                         ]);
                     }
 
-                    table.print_std();
+                    table.print_stdout();
 
                     println!("{} active connection(s)", num_connections);
                 },
@@ -1055,6 +1055,97 @@ impl CommandHandler {
     pub(crate) fn get_software_updater(&self) -> SoftwareUpdaterHandle {
         self.software_updater.clone()
     }
+
+    pub fn get_blockchain_db_stats(&self) {
+        const BYTES_PER_MB: usize = 1024 * 1024;
+
+        let db = self.blockchain_db.clone();
+
+        self.executor.spawn(async move {
+            let total_db_size = match db.get_stats().await {
+                Ok(stats) => {
+                    let mut table = Table::new();
+                    table.set_titles(vec![
+                        "Name",
+                        "Entries",
+                        "Depth",
+                        "Branch Pages",
+                        "Leaf Pages",
+                        "Overflow Pages",
+                        "Est. Size (MiB)",
+                        "% of total",
+                    ]);
+                    let total_db_size = stats.db_stats().iter().map(|s| s.total_page_size()).sum::<usize>();
+                    stats.db_stats().iter().for_each(|stat| {
+                        table.add_row(row![
+                            stat.name,
+                            stat.entries,
+                            stat.depth,
+                            stat.branch_pages,
+                            stat.leaf_pages,
+                            stat.overflow_pages,
+                            format!("{:.2}", stat.total_page_size() as f32 / BYTES_PER_MB as f32),
+                            format!("{:.2}%", (stat.total_page_size() as f32 / total_db_size as f32) * 100.0)
+                        ]);
+                    });
+
+                    table.print_stdout();
+                    println!();
+                    println!(
+                        "{} databases, {:.2} MiB used ({:.2}%), page size: {} bytes, env_info = ({})",
+                        stats.root().entries,
+                        total_db_size as f32 / BYTES_PER_MB as f32,
+                        (total_db_size as f32 / stats.env_info().mapsize as f32) * 100.0,
+                        stats.root().psize as usize,
+                        stats.env_info()
+                    );
+                    total_db_size
+                },
+                Err(err) => {
+                    println!("{}", err);
+                    return;
+                },
+            };
+
+            println!();
+            println!("Totalling DB entry sizes. This may take a few seconds...");
+            println!();
+            match db.fetch_total_size_stats().await {
+                Ok(stats) => {
+                    println!();
+                    let mut table = Table::new();
+                    table.set_titles(vec![
+                        "Name",
+                        "Entries",
+                        "Total Size (MiB)",
+                        "Avg. Size/Entry (bytes)",
+                        "% of total",
+                    ]);
+                    let total_data_size = stats.sizes().iter().map(|s| s.total()).sum::<u64>();
+                    stats.sizes().iter().for_each(|size| {
+                        let total = size.total() as f32 / BYTES_PER_MB as f32;
+                        table.add_row(row![
+                            size.name,
+                            size.num_entries,
+                            format!("{:.2}", total),
+                            format!("{}", size.avg_bytes_per_entry()),
+                            format!("{:.2}%", (size.total() as f32 / total_data_size as f32) * 100.0)
+                        ])
+                    });
+                    table.print_stdout();
+                    println!();
+                    println!(
+                        "Total blockchain data size: {:.2} MiB ({:.2} % of LMDB map size)",
+                        total_data_size as f32 / BYTES_PER_MB as f32,
+                        (total_data_size as f32 / total_db_size as f32) * 100.0
+                    );
+                },
+                Err(err) => {
+                    println!("{}", err);
+                },
+            }
+        });
+    }
 }
 
 async fn fetch_banned_peers(pm: &PeerManager) -> Result<Vec<Peer>, PeerManagerError> {

diff --git a/applications/tari_base_node/src/parser.rs b/applications/tari_base_node/src/parser.rs
@@ -53,6 +53,7 @@ pub enum BaseNodeCommand {
     CheckForUpdates,
     Status,
     GetChainMetadata,
+    GetDbStats,
     GetPeer,
     ListPeers,
     DialPeer,
@@ -184,6 +185,9 @@ impl Parser {
             GetChainMetadata => {
                 self.command_handler.get_chain_meta();
             },
+            GetDbStats => {
+                self.command_handler.get_blockchain_db_stats();
+            },
             DialPeer => {
                 self.process_dial_peer(args);
             },
@@ -285,6 +289,9 @@ impl Parser {
             GetChainMetadata => {
                 println!("Gets your base node chain meta data");
             },
+            GetDbStats => {
+                println!("Gets your base node database stats");
+            },
             DialPeer => {
                 println!("Attempt to connect to a known peer");
             },

diff --git a/applications/tari_base_node/src/table.rs b/applications/tari_base_node/src/table.rs
@@ -48,6 +48,8 @@ impl<'t, 's> Table<'t, 's> {
 
     pub fn render<T: Write>(&self, out: &mut T) -> io::Result<()> {
         self.render_titles(out)?;
+        out.write_all(b"\n")?;
+        self.render_separator(out)?;
         if !self.rows.is_empty() {
             out.write_all(b"\n")?;
             self.render_rows(out)?;
@@ -56,7 +58,7 @@ impl<'t, 's> Table<'t, 's> {
         Ok(())
     }
 
-    pub fn print_std(&self) {
+    pub fn print_stdout(&self) {
         self.render(&mut io::stdout()).unwrap();
     }
 
@@ -106,6 +108,23 @@ impl<'t, 's> Table<'t, 's> {
         }
         Ok(())
     }
+
+    fn render_separator<T: Write>(&self, out: &mut T) -> io::Result<()> {
+        if let Some(rows_len) = self.rows.first().map(|r| r.len()) {
+            for i in 0..rows_len {
+                let width = self.col_width(i);
+                let pad_left = if i == 0 { "" } else { " " };
+                out.write_all(pad_left.as_bytes())?;
+                let sep = "-".repeat(width);
+                out.write_all(sep.as_bytes())?;
+                out.write_all(" ".as_bytes())?;
+                if i < rows_len - 1 {
+                    out.write_all(self.delim_str.as_bytes())?;
+                }
+            }
+        }
+        Ok(())
+    }
 }
 
 macro_rules! row {
@@ -126,7 +145,7 @@ mod test {
         table.render(&mut buf).unwrap();
         assert_eq!(
             String::from_utf8_lossy(&buf.into_inner()),
-            "Hello | World | Bonjour | Le | Monde "
+            "Hello | World | Bonjour | Le | Monde \n"
         );
     }
 
@@ -141,7 +160,8 @@ mod test {
         table.render(&mut buf).unwrap();
         assert_eq!(
             String::from_utf8_lossy(&buf.into_inner()),
-            "Name    | Age | Telephone Number | Favourite Headwear \nTrevor  | 132 | +123 12323223    | Pith Helmet        \n\nHatless | 2   \n"
+            "Name    | Age | Telephone Number | Favourite Headwear \n------- | --- | ---------------- | \
+             ------------------ \nTrevor  | 132 | +123 12323223    | Pith Helmet        \n\nHatless | 2   \n"
         );
     }
 }
diff --git a/base_layer/core/src/chain_storage/async_db.rs b/base_layer/core/src/chain_storage/async_db.rs
@@ -32,6 +32,8 @@ use crate::{
         ChainHeader,
         ChainStorageError,
         CompleteDeletedBitmap,
+        DbBasicStats,
+        DbTotalSizeStats,
         DbTransaction,
         HistoricalBlock,
         HorizonData,
@@ -226,6 +228,10 @@ impl<B: BlockchainBackend + 'static> AsyncBlockchainDb<B> {
     make_async_fn!(fetch_block_hashes_from_header_tip(n: usize, offset: usize) -> Vec<HashOutput>, "fetch_block_hashes_from_header_tip");
 
     make_async_fn!(fetch_complete_deleted_bitmap_at(hash: HashOutput) -> CompleteDeletedBitmap, "fetch_deleted_bitmap");
+
+    make_async_fn!(get_stats() -> DbBasicStats, "get_stats");
+
+    make_async_fn!(fetch_total_size_stats() -> DbTotalSizeStats, "fetch_total_size_stats");
 }
 
 impl<B: BlockchainBackend + 'static> From<BlockchainDatabase<B>> for AsyncBlockchainDb<B> {

diff --git a/base_layer/core/src/chain_storage/blockchain_backend.rs b/base_layer/core/src/chain_storage/blockchain_backend.rs
@@ -8,7 +8,9 @@ use crate::{
         ChainBlock,
         ChainHeader,
         ChainStorageError,
+        DbBasicStats,
         DbKey,
+        DbTotalSizeStats,
         DbTransaction,
         DbValue,
         HorizonData,
@@ -158,4 +160,11 @@ pub trait BlockchainBackend: Send + Sync {
     fn fetch_monero_seed_first_seen_height(&self, seed: &[u8]) -> Result<u64, ChainStorageError>;
 
     fn fetch_horizon_data(&self) -> Result<Option<HorizonData>, ChainStorageError>;
+
+    /// Returns basic database stats for each internal database, such as number of entries and page sizes. This call may
+    /// not apply to every database implementation.
+    fn get_stats(&self) -> Result<DbBasicStats, ChainStorageError>;
+    /// Returns total size information about each internal database. This call may be very slow and will obtain a read
+    /// lock for the duration.
+    fn fetch_total_size_stats(&self) -> Result<DbTotalSizeStats, ChainStorageError>;
 }
diff --git a/base_layer/core/src/chain_storage/blockchain_database.rs b/base_layer/core/src/chain_storage/blockchain_database.rs
@@ -35,6 +35,8 @@ use crate::{
         BlockchainBackend,
         ChainBlock,
         ChainHeader,
+        DbBasicStats,
+        DbTotalSizeStats,
         HistoricalBlock,
         HorizonData,
         MmrTree,
@@ -903,6 +905,18 @@ where B: BlockchainBackend
             chain_metadata.best_block().clone(),
         ))
     }
+
+    pub fn get_stats(&self) -> Result<DbBasicStats, ChainStorageError> {
+        let lock = self.db_read_access()?;
+        lock.get_stats()
+    }
+
+    /// Returns total size information about each internal database. This call may be very slow and will obtain a read
+    /// lock for the duration.
+    pub fn fetch_total_size_stats(&self) -> Result<DbTotalSizeStats, ChainStorageError> {
+        let lock = self.db_read_access()?;
+        lock.fetch_total_size_stats()
+    }
 }
 
 fn unexpected_result<T>(req: DbKey, res: DbValue) -> Result<T, ChainStorageError> {

diff --git a/base_layer/core/src/chain_storage/db_transaction.rs b/base_layer/core/src/chain_storage/db_transaction.rs
@@ -255,10 +255,6 @@ impl DbTransaction {
         &self.operations
     }
 
-    pub(crate) fn into_operations(self) -> Vec<WriteOperation> {
-        self.operations
-    }
-
     /// This will store the seed key with the height. This is called when a block is accepted into the main chain.
     /// This will only update the hieght of the seed, if its lower then currently stored.
     pub fn insert_monero_seed_height(&mut self, monero_seed: Vec<u8>, height: u64) {

diff --git a/base_layer/core/src/chain_storage/error.rs b/base_layer/core/src/chain_storage/error.rs
@@ -21,6 +21,7 @@
 // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 use crate::{chain_storage::MmrTree, proof_of_work::PowError, validation::ValidationError};
+use lmdb_zero::error;
 use tari_mmr::{error::MerkleMountainRangeError, MerkleProofError};
 use tari_storage::lmdb_store::LMDBError;
 use thiserror::Error;
@@ -109,6 +110,8 @@ pub enum ChainStorageError {
     CannotCalculateNonTipMmr(String),
     #[error("Key {key} in {table_name} already exists")]
     KeyExists { table_name: &'static str, key: String },
+    #[error("Database resize required")]
+    DbResizeRequired,
 }
 
 impl ChainStorageError {
@@ -131,11 +134,12 @@ impl From<lmdb_zero::Error> for ChainStorageError {
     fn from(err: lmdb_zero::Error) -> Self {
         use lmdb_zero::Error::*;
         match err {
-            Code(c) if c == lmdb_zero::error::NOTFOUND => ChainStorageError::ValueNotFound {
+            Code(error::NOTFOUND) => ChainStorageError::ValueNotFound {
                 entity: "<unspecified entity>",
                 field: "<unknown>",
                 value: "<unknown>".to_string(),
             },
+            Code(error::MAP_FULL) => ChainStorageError::DbResizeRequired,
             _ => ChainStorageError::AccessError(err.to_string()),
         }
     }

diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs
@@ -77,6 +77,7 @@ where
     V: Serialize + Debug,
 {
     let val_buf = serialize(val)?;
+    trace!(target: LOG_TARGET, "LMDB: {} bytes inserted", val_buf.len());
     txn.access().put(&db, key, &val_buf, put::NOOVERWRITE).map_err(|e| {
         error!(
             target: LOG_TARGET,
@@ -86,16 +87,17 @@ where
             val,
             e,
         );
-
         if let lmdb_zero::Error::Code(code) = &e {
             if *code == lmdb_zero::error::KEYEXIST {
                 return ChainStorageError::KeyExists {
                     table_name,
                     key: to_hex(key.as_lmdb_bytes()),
                 };
             }
+            if *code == lmdb_zero::error::MAP_FULL {
+                return ChainStorageError::DbResizeRequired;
+            }
         }
-
         ChainStorageError::InsertError {
             table: table_name,
             error: e.to_string(),
@@ -120,6 +122,11 @@ where
             target: LOG_TARGET,
             "Could not insert value into lmdb transaction: {:?}", e
         );
+        if let lmdb_zero::Error::Code(code) = &e {
+            if *code == lmdb_zero::error::MAP_FULL {
+                return ChainStorageError::DbResizeRequired;
+            }
+        }
         ChainStorageError::AccessError(e.to_string())
     })
 }
@@ -387,3 +394,19 @@ where
     }
     Ok(result)
 }
+
+/// Fetches all the size of all key/values in the given DB. Returns the number of entries, the total size of all the
+/// keys and values in bytes.
+pub fn fetch_db_entry_sizes(txn: &ConstTransaction<'_>, db: &Database) -> Result<(u64, u64, u64), ChainStorageError> {
+    let access = txn.access();
+    let mut cursor = txn.cursor(db)?;
+    let mut num_entries = 0;
+    let mut total_key_size = 0;
+    let mut total_value_size = 0;
+    while let Some((key, value)) = cursor.next::<[u8], [u8]>(&access).to_opt()? {
+        num_entries += 1;
+        total_key_size += key.len() as u64;
+        total_value_size += value.len() as u64;
+    }
+    Ok((num_entries, total_key_size, total_value_size))
+}