feat: lazily evaluate for new random_x template (#6170)

Description
---
- Changed the merge mining proxy to evaluate RandomX block template
changes lazily, similar to the strategy employed for the SHA3 miner.
Evaluating this often is very expensive, especially during busy network
times with a highly volatile mempool.
- Removed _new template_ spam from logging
- Fixed a bug where the command-line network argument `igor` specified
for the merge mining proxy was not recognized in the network-aware
'DomainSeparatedConsensusHasher'
- Updated the merge mining proxy `config.toml` to be in line with core

Motivation and Context
---
- With recent stress tests on `nextnet` that lasted multiple hours,
RandomX mining virtually ground to a halt during the stress test,
whereas SHAR3 mining was able to continue.
- One merge mining proxy that was monitored for 45 min went up to 4.5 GB
in memory usage. During that time the connected base node only advanced
one block, while the merge mining proxy requested 428 new block
templates from the base node, all of them unique. This could account for
~1 GB in memory usage as block templates are discarded after 20 minutes.

How Has This Been Tested?
---
- System-level testing during low network activity (_on `igor`_)
- System-level stress testing:
  - coin split stress test (_on `igor`_)
  - one-side stealth transaction stress test (_on `igor`_)

What process can a PR reviewer use to test or verify this change?
---
Code walk-through, system-level testing 

<!-- Checklist -->
<!-- 1. Is the title of your PR in the form that would make nice release
notes? The title, excluding the conventional commit
tag, will be included exactly as is in the CHANGELOG, so please think
about it carefully. -->


Breaking Changes
---

- [x] None
- [ ] Requires data directory on base node to be deleted
- [ ] Requires hard fork
- [ ] Other - Please specify

<!-- Does this include a breaking change? If so, include this line as a
footer -->
<!-- BREAKING CHANGE: Description what the user should do, e.g. delete a
database, resync the chain -->

---------

Co-authored-by: SW van Heerden <swvheerden@gmail.com>

applications/minotari_app_utilities/Cargo.toml

@@ -9,7 +9,6 @@ license = "BSD-3-Clause"
 tari_common = { path = "../../common" }
 tari_common_types = { path = "../../base_layer/common_types" }
 tari_comms = { path = "../../comms/core" }
-tari_features = { path = "../../common/tari_features", version = "1.0.0-pre.9" }
 tari_utilities = { version = "0.7" }
 minotari_app_grpc = { path = "../minotari_app_grpc", optional = true }
 

applications/minotari_app_utilities/src/common_cli_args.rs

@@ -105,7 +105,7 @@ impl CommonCliArgs {
 }
 
 impl ConfigOverrideProvider for CommonCliArgs {
-    fn get_config_property_overrides(&self, _default_network: Network) -> Vec<(String, String)> {
+    fn get_config_property_overrides(&self, _network: &mut Network) -> Vec<(String, String)> {
         let mut overrides = self.config_property_overrides.clone();
         overrides.push((
             "common.base_path".to_string(),

applications/minotari_app_utilities/src/lib.rs

@@ -22,7 +22,6 @@
 
 pub mod common_cli_args;
 pub mod identity_management;
-pub mod network_check;
 #[cfg(feature = "miner_input")]
 pub mod parse_miner_input;
 pub mod utilities;

applications/minotari_console_wallet/src/cli.rs

@@ -91,9 +91,9 @@ pub struct Cli {
 }
 
 impl ConfigOverrideProvider for Cli {
-    fn get_config_property_overrides(&self, default_network: Network) -> Vec<(String, String)> {
-        let mut overrides = self.common.get_config_property_overrides(default_network);
-        let network = self.common.network.unwrap_or(default_network);
+    fn get_config_property_overrides(&self, network: &mut Network) -> Vec<(String, String)> {
+        let mut overrides = self.common.get_config_property_overrides(network);
+        *network = self.common.network.unwrap_or(*network);
         overrides.push(("wallet.network".to_string(), network.to_string()));
         overrides.push(("wallet.override_from".to_string(), network.to_string()));
         overrides.push(("p2p.seeds.override_from".to_string(), network.to_string()));

applications/minotari_console_wallet/src/lib.rs

@@ -48,7 +48,7 @@ pub use cli::{
 };
 use init::{change_password, get_base_node_peer_config, init_wallet, start_wallet, tari_splash_screen, WalletBoot};
 use log::*;
-use minotari_app_utilities::{common_cli_args::CommonCliArgs, consts, network_check::set_network_if_choice_valid};
+use minotari_app_utilities::{common_cli_args::CommonCliArgs, consts};
 use minotari_wallet::transaction_service::config::TransactionRoutingMechanism;
 use recovery::{get_seed_from_seed_words, prompt_private_key_from_seed_words};
 use tari_common::{
@@ -117,8 +117,6 @@ pub fn run_wallet_with_cli(
         consts::APP_VERSION
     );
 
-    set_network_if_choice_valid(config.wallet.network)?;
-
     let password = get_password(config, &cli);
 
     if password.is_none() {

applications/minotari_merge_mining_proxy/Cargo.toml

@@ -11,33 +11,33 @@ edition = "2018"
 default = []
 
 [dependencies]
+minotari_app_grpc = { path = "../minotari_app_grpc" }
+minotari_app_utilities = { path = "../minotari_app_utilities", features = ["miner_input"] }
+minotari_node_grpc_client = { path = "../../clients/rust/base_node_grpc_client" }
+minotari_wallet_grpc_client = { path = "../../clients/rust/wallet_grpc_client" }
 tari_common = { path = "../../common" }
 tari_common_types = { path = "../../base_layer/common_types" }
 tari_comms = { path = "../../comms/core" }
 tari_core = { path = "../../base_layer/core", default-features = false, features = ["transactions"] }
-minotari_app_utilities = { path = "../minotari_app_utilities", features = ["miner_input"] }
-tari_utilities = { version = "0.7" }
-minotari_node_grpc_client = { path = "../../clients/rust/base_node_grpc_client" }
-minotari_wallet_grpc_client = { path = "../../clients/rust/wallet_grpc_client" }
-minotari_app_grpc = { path = "../minotari_app_grpc" }
 tari_key_manager = {  path = "../../base_layer/key_manager", features = ["key_manager_service"] }
+tari_utilities = { version = "0.7" }
 
 anyhow = "1.0.53"
-crossterm = { version = "0.25.0" }
 bincode = "1.3.1"
 borsh = "1.2"
 bytes = "1.1"
-chrono = { version = "0.4.6", default-features = false }
+chrono = { version = "0.4.19", default-features = false }
 clap = { version = "3.2", features = ["derive", "env"] }
 config = { version = "0.13.0" }
-futures = "0.3.5"
+crossterm = { version = "0.25.0" }
+futures = { version = "^0.3.16", features = ["async-await"] }
 hex = "0.4.2"
 hyper = "0.14.12"
 jsonrpc = "0.12.0"
 log = { version = "0.4.8", features = ["std"] }
 monero = { version = "0.20.0" }
 reqwest = { version = "0.11.4", features = ["json"] }
-serde = { version = "1.0.106", features = ["derive"] }
+serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.57"
 thiserror = "1.0.26"
 tokio = { version = "1.23", features = ["macros"] }

applications/minotari_merge_mining_proxy/src/block_template_data.rs

@@ -22,7 +22,7 @@
 
 //! Provides methods for for building template data and storing them with timestamps.
 
-use std::{collections::HashMap, sync::Arc};
+use std::{collections::HashMap, convert::TryFrom, sync::Arc};
 
 #[cfg(not(test))]
 use chrono::Duration;
@@ -33,7 +33,7 @@ use tari_core::proof_of_work::monero_rx::FixedByteArray;
 use tokio::sync::RwLock;
 use tracing::trace;
 
-use crate::error::MmProxyError;
+use crate::{block_template_protocol::FinalBlockTemplateData, error::MmProxyError};
 
 const LOG_TARGET: &str = "minotari_mm_proxy::xmrig";
 
@@ -46,13 +46,13 @@ pub struct BlockTemplateRepository {
 /// Structure holding [BlockTemplateData] along with a timestamp.
 #[derive(Debug, Clone)]
 pub struct BlockTemplateRepositoryItem {
-    pub data: BlockTemplateData,
+    pub data: FinalBlockTemplateData,
     datetime: DateTime<Utc>,
 }
 
 impl BlockTemplateRepositoryItem {
     /// Create new [Self] with current time in UTC.
-    pub fn new(block_template: BlockTemplateData) -> Self {
+    pub fn new(block_template: FinalBlockTemplateData) -> Self {
         Self {
             data: block_template,
             datetime: Utc::now(),
@@ -73,7 +73,7 @@ impl BlockTemplateRepository {
     }
 
     /// Return [BlockTemplateData] with the associated hash. None if the hash is not stored.
-    pub async fn get<T: AsRef<[u8]>>(&self, hash: T) -> Option<BlockTemplateData> {
+    pub async fn get<T: AsRef<[u8]>>(&self, hash: T) -> Option<FinalBlockTemplateData> {
         trace!(
             target: LOG_TARGET,
             "Retrieving blocktemplate with merge mining hash: {:?}",
@@ -83,16 +83,28 @@ impl BlockTemplateRepository {
         b.get(hash.as_ref()).map(|item| item.data.clone())
     }
 
-    /// Store [BlockTemplateData] at the hash value.
-    pub async fn save(&self, hash: Vec<u8>, block_template: BlockTemplateData) {
-        trace!(
-            target: LOG_TARGET,
-            "Saving blocktemplate with merge mining hash: {:?}",
-            hex::encode(&hash)
-        );
+    /// Store [BlockTemplateData] at the hash value if the key does not exist.
+    pub async fn save_if_key_unique(&self, hash: Vec<u8>, block_template: FinalBlockTemplateData) {
         let mut b = self.blocks.write().await;
-        let repository_item = BlockTemplateRepositoryItem::new(block_template);
-        b.insert(hash, repository_item);
+        b.entry(hash.clone()).or_insert_with(|| {
+            trace!(
+                target: LOG_TARGET,
+                "Saving blocktemplate with merge mining hash: {:?}",
+                hex::encode(&hash)
+            );
+            BlockTemplateRepositoryItem::new(block_template)
+        });
+    }
+
+    /// Check if the repository contains a block template with best_previous_block_hash
+    pub async fn contains(&self, current_best_block_hash: FixedHash) -> Option<FinalBlockTemplateData> {
+        let b = self.blocks.read().await;
+        b.values()
+            .find(|item| {
+                let header = item.data.template.new_block_template.header.clone().unwrap_or_default();
+                FixedHash::try_from(header.prev_hash).unwrap_or(FixedHash::default()) == current_best_block_hash
+            })
+            .map(|val| val.data.clone())
     }
 
     /// Remove any data that is older than 20 minutes.
@@ -126,8 +138,9 @@ pub struct BlockTemplateData {
     pub tari_miner_data: grpc::MinerData,
     pub monero_difficulty: u64,
     pub tari_difficulty: u64,
-    pub tari_hash: FixedHash,
+    pub tari_merge_mining_hash: FixedHash,
     pub aux_chain_hashes: Vec<monero::Hash>,
+    pub new_block_template: grpc::NewBlockTemplate,
 }
 
 impl BlockTemplateData {}
@@ -140,8 +153,9 @@ pub struct BlockTemplateDataBuilder {
     tari_miner_data: Option<grpc::MinerData>,
     monero_difficulty: Option<u64>,
     tari_difficulty: Option<u64>,
-    tari_hash: Option<FixedHash>,
+    tari_merge_mining_hash: Option<FixedHash>,
     aux_chain_hashes: Vec<monero::Hash>,
+    new_block_template: Option<grpc::NewBlockTemplate>,
 }
 
 impl BlockTemplateDataBuilder {
@@ -174,8 +188,8 @@ impl BlockTemplateDataBuilder {
         self
     }
 
-    pub fn tari_hash(mut self, hash: FixedHash) -> Self {
-        self.tari_hash = Some(hash);
+    pub fn tari_merge_mining_hash(mut self, hash: FixedHash) -> Self {
+        self.tari_merge_mining_hash = Some(hash);
         self
     }
 
@@ -184,6 +198,11 @@ impl BlockTemplateDataBuilder {
         self
     }
 
+    pub fn new_block_template(mut self, template: grpc::NewBlockTemplate) -> Self {
+        self.new_block_template = Some(template);
+        self
+    }
+
     /// Build a new [BlockTemplateData], all the values have to be set.
     ///
     /// # Errors
@@ -205,21 +224,25 @@ impl BlockTemplateDataBuilder {
         let tari_difficulty = self
             .tari_difficulty
             .ok_or_else(|| MmProxyError::MissingDataError("tari_difficulty not provided".to_string()))?;
-        let tari_hash = self
-            .tari_hash
+        let tari_merge_mining_hash = self
+            .tari_merge_mining_hash
             .ok_or_else(|| MmProxyError::MissingDataError("tari_hash not provided".to_string()))?;
         if self.aux_chain_hashes.is_empty() {
             return Err(MmProxyError::MissingDataError("aux chain hashes are empty".to_string()));
         };
+        let new_block_template = self
+            .new_block_template
+            .ok_or_else(|| MmProxyError::MissingDataError("new_block_template not provided".to_string()))?;
 
         Ok(BlockTemplateData {
             monero_seed,
             tari_block,
             tari_miner_data,
             monero_difficulty,
             tari_difficulty,
-            tari_hash,
+            tari_merge_mining_hash,
             aux_chain_hashes: self.aux_chain_hashes,
+            new_block_template,
         })
     }
 }
@@ -230,12 +253,14 @@ pub mod test {
 
     use tari_core::{
         blocks::{Block, BlockHeader},
+        proof_of_work::Difficulty,
         transactions::aggregated_body::AggregateBody,
     };
+    use tari_utilities::ByteArray;
 
     use super::*;
 
-    fn create_block_template_data() -> BlockTemplateData {
+    fn create_block_template_data() -> FinalBlockTemplateData {
         let header = BlockHeader::new(100);
         let body = AggregateBody::empty();
         let block = Block::new(header, body);
@@ -246,15 +271,25 @@ pub mod test {
             total_fees: 100,
             algo: Some(grpc::PowAlgo { pow_algo: 0 }),
         };
+        let new_block_template = grpc::NewBlockTemplate::default();
         let btdb = BlockTemplateDataBuilder::new()
             .monero_seed(FixedByteArray::new())
             .tari_block(block.try_into().unwrap())
             .tari_miner_data(miner_data)
             .monero_difficulty(123456)
             .tari_difficulty(12345)
-            .tari_hash(hash)
-            .aux_hashes(vec![monero::Hash::from_slice(hash.as_slice())]);
-        btdb.build().unwrap()
+            .tari_merge_mining_hash(hash)
+            .aux_hashes(vec![monero::Hash::from_slice(hash.as_slice())])
+            .new_block_template(new_block_template);
+        let block_template_data = btdb.build().unwrap();
+        FinalBlockTemplateData {
+            template: block_template_data,
+            target_difficulty: Difficulty::from_u64(12345).unwrap(),
+            blockhashing_blob: "no blockhashing_blob data".to_string(),
+            blocktemplate_blob: "no blocktemplate_blob data".to_string(),
+            aux_chain_hashes: vec![monero::Hash::from_slice(hash.as_slice())],
+            aux_chain_mr: hash.to_vec(),
+        }
     }
 
     #[tokio::test]
@@ -264,8 +299,8 @@ pub mod test {
         let hash2 = vec![2; 32];
         let hash3 = vec![3; 32];
         let block_template = create_block_template_data();
-        btr.save(hash1.clone(), block_template.clone()).await;
-        btr.save(hash2.clone(), block_template).await;
+        btr.save_if_key_unique(hash1.clone(), block_template.clone()).await;
+        btr.save_if_key_unique(hash2.clone(), block_template).await;
         assert!(btr.get(hash1.clone()).await.is_some());
         assert!(btr.get(hash2.clone()).await.is_some());
         assert!(btr.get(hash3.clone()).await.is_none());
@@ -323,10 +358,13 @@ pub mod test {
     #[test]
     pub fn ok_block_template_data_builder() {
         let build = create_block_template_data();
-        assert!(build.monero_seed.is_empty());
-        assert_eq!(build.tari_block.header.unwrap().version, 100);
-        assert_eq!(build.tari_miner_data.target_difficulty, 600000);
-        assert_eq!(build.monero_difficulty, 123456);
-        assert_eq!(build.tari_difficulty, 12345);
+        assert!(build.template.monero_seed.is_empty());
+        assert_eq!(build.template.tari_block.header.unwrap().version, 100);
+        assert_eq!(build.template.tari_miner_data.target_difficulty, 600000);
+        assert_eq!(build.template.monero_difficulty, 123456);
+        assert_eq!(build.template.tari_difficulty, 12345);
+        assert_eq!(build.blockhashing_blob, "no blockhashing_blob data".to_string());
+        assert_eq!(build.blocktemplate_blob, "no blocktemplate_blob data".to_string());
+        assert_eq!(build.target_difficulty, Difficulty::from_u64(12345).unwrap());
     }
 }