From b7efd1c4237511be12348e5678541213835b1b78 Mon Sep 17 00:00:00 2001 From: mrnaveira <47919901+mrnaveira@users.noreply.github.com> Date: Thu, 22 Sep 2022 12:58:41 -0600 Subject: [PATCH 1/5] add grpc method to get templates --- .../tari_app_grpc/proto/base_node.proto | 9 ++- .../src/grpc/base_node_grpc_server.rs | 76 ++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/applications/tari_app_grpc/proto/base_node.proto b/applications/tari_app_grpc/proto/base_node.proto index 96dad7eb33..af5e3cc648 100644 --- a/applications/tari_app_grpc/proto/base_node.proto +++ b/applications/tari_app_grpc/proto/base_node.proto @@ -25,6 +25,7 @@ import "types.proto"; import "transaction.proto"; import "block.proto"; import "network.proto"; +import "sidechain_types.proto"; package tari.rpc; @@ -92,6 +93,8 @@ service BaseNode { rpc GetActiveValidatorNodes(GetActiveValidatorNodesRequest) returns (stream ActiveValidatorNode); rpc GetCommittee(GetCommitteeRequest) returns (GetCommitteeResponse); rpc GetShardKey(GetShardKeyRequest) returns (GetShardKeyResponse); + // Get templates + rpc GetTemplateRegistrations(GetTemplateRegistrationsRequest) returns (stream TemplateRegistration); } message GetAssetMetadataRequest { @@ -464,4 +467,8 @@ message GetShardKeyRequest { message GetShardKeyResponse { bytes shard_key = 1; -} \ No newline at end of file +} + +message GetTemplateRegistrationsRequest { + uint64 height = 1; +} diff --git a/applications/tari_base_node/src/grpc/base_node_grpc_server.rs b/applications/tari_base_node/src/grpc/base_node_grpc_server.rs index 258dbdc3d7..e7dd8a5dbb 100644 --- a/applications/tari_base_node/src/grpc/base_node_grpc_server.rs +++ b/applications/tari_base_node/src/grpc/base_node_grpc_server.rs @@ -48,7 +48,10 @@ use tari_core::{ iterators::NonOverlappingIntegerPairIter, mempool::{service::LocalMempoolService, TxStorageResponse}, proof_of_work::PowAlgorithm, - transactions::{aggregated_body::AggregateBody, transaction_components::Transaction}, + transactions::{ + aggregated_body::AggregateBody, + transaction_components::{CodeTemplateRegistration, Transaction}, + }, }; use tari_p2p::{auto_update::SoftwareUpdaterHandle, services::liveness::LivenessHandle}; use tari_utilities::{hex::Hex, message_format::MessageFormat, ByteArray}; @@ -140,6 +143,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer { type GetMempoolTransactionsStream = mpsc::Receiver>; type GetNetworkDifficultyStream = mpsc::Receiver>; type GetPeersStream = mpsc::Receiver>; + type GetTemplateRegistrationsStream = mpsc::Receiver>; type GetTokensInCirculationStream = mpsc::Receiver>; type ListHeadersStream = mpsc::Receiver>; type SearchKernelsStream = mpsc::Receiver>; @@ -1541,6 +1545,76 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer { ); Ok(Response::new(rx)) } + + async fn get_template_registrations( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let report_error_flag = self.report_error_flag(); + debug!(target: LOG_TARGET, "Incoming GRPC request for GetTemplateRegistrations"); + + let mut handler = self.node_service.clone(); + let (mut tx, rx) = mpsc::channel(1000); + + task::spawn(async move { + // TODO: fetch the templates from the db + let template_registrations: Vec = vec![]; + + for template_registration in template_registrations { + let template_registration = match tari_rpc::TemplateRegistration::try_from(template_registration) { + Ok(t) => t, + Err(e) => { + warn!( + target: LOG_TARGET, + "Error sending converting template registration for GRPC: {}", e + ); + match tx + .send(Err(obscure_error_if_true( + report_error_flag, + Status::internal("Error converting template_registration"), + ))) + .await + { + Ok(_) => (), + Err(send_err) => { + warn!(target: LOG_TARGET, "Error sending error to GRPC client: {}", send_err) + }, + } + return; + }, + }; + + match tx.send(Ok(template_registration)).await { + Ok(_) => (), + Err(err) => { + warn!( + target: LOG_TARGET, + "Error sending template registration via GRPC: {}", err + ); + match tx + .send(Err(obscure_error_if_true( + report_error_flag, + Status::unknown("Error sending data"), + ))) + .await + { + Ok(_) => (), + Err(send_err) => { + warn!(target: LOG_TARGET, "Error sending error to GRPC client: {}", send_err) + }, + } + return; + }, + } + } + }); + debug!( + target: LOG_TARGET, + "Sending GetTemplateRegistrations response stream to client" + ); + Ok(Response::new(rx)) + } } enum BlockGroupType { From 5b8582762db9a35453fa0d77133e977a0fe9d080 Mon Sep 17 00:00:00 2001 From: mrnaveira <47919901+mrnaveira@users.noreply.github.com> Date: Thu, 22 Sep 2022 15:58:05 -0600 Subject: [PATCH 2/5] store template registrations in lmdb --- .../tari_app_grpc/proto/base_node.proto | 2 +- .../src/grpc/base_node_grpc_server.rs | 14 ++--- .../comms_interface/comms_request.rs | 4 ++ .../comms_interface/comms_response.rs | 9 +++- .../comms_interface/inbound_handlers.rs | 6 +++ .../comms_interface/local_interface.rs | 16 +++++- base_layer/core/src/chain_storage/async_db.rs | 4 +- .../src/chain_storage/blockchain_backend.rs | 6 ++- .../src/chain_storage/blockchain_database.rs | 10 +++- .../core/src/chain_storage/db_transaction.rs | 8 ++- .../core/src/chain_storage/lmdb_db/lmdb_db.rs | 51 ++++++++++++++++++- .../core/src/test_helpers/blockchain.rs | 9 +++- .../side_chain/template_registration.rs | 38 ++++++++++---- 13 files changed, 152 insertions(+), 25 deletions(-) diff --git a/applications/tari_app_grpc/proto/base_node.proto b/applications/tari_app_grpc/proto/base_node.proto index af5e3cc648..becb93572f 100644 --- a/applications/tari_app_grpc/proto/base_node.proto +++ b/applications/tari_app_grpc/proto/base_node.proto @@ -470,5 +470,5 @@ message GetShardKeyResponse { } message GetTemplateRegistrationsRequest { - uint64 height = 1; + uint64 from_height = 1; } diff --git a/applications/tari_base_node/src/grpc/base_node_grpc_server.rs b/applications/tari_base_node/src/grpc/base_node_grpc_server.rs index e7dd8a5dbb..7342357a77 100644 --- a/applications/tari_base_node/src/grpc/base_node_grpc_server.rs +++ b/applications/tari_base_node/src/grpc/base_node_grpc_server.rs @@ -48,10 +48,7 @@ use tari_core::{ iterators::NonOverlappingIntegerPairIter, mempool::{service::LocalMempoolService, TxStorageResponse}, proof_of_work::PowAlgorithm, - transactions::{ - aggregated_body::AggregateBody, - transaction_components::{CodeTemplateRegistration, Transaction}, - }, + transactions::{aggregated_body::AggregateBody, transaction_components::Transaction}, }; use tari_p2p::{auto_update::SoftwareUpdaterHandle, services::liveness::LivenessHandle}; use tari_utilities::{hex::Hex, message_format::MessageFormat, ByteArray}; @@ -1558,8 +1555,13 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer { let (mut tx, rx) = mpsc::channel(1000); task::spawn(async move { - // TODO: fetch the templates from the db - let template_registrations: Vec = vec![]; + let template_registrations = match handler.get_template_registrations(request.from_height).await { + Err(err) => { + warn!(target: LOG_TARGET, "Error communicating with base node: {}", err,); + return; + }, + Ok(data) => data, + }; for template_registration in template_registrations { let template_registration = match tari_rpc::TemplateRegistration::try_from(template_registration) { diff --git a/base_layer/core/src/base_node/comms_interface/comms_request.rs b/base_layer/core/src/base_node/comms_interface/comms_request.rs index 817438bc05..a46e08df52 100644 --- a/base_layer/core/src/base_node/comms_interface/comms_request.rs +++ b/base_layer/core/src/base_node/comms_interface/comms_request.rs @@ -59,6 +59,7 @@ pub enum NodeCommsRequest { FetchValidatorNodesKeys { height: u64 }, FetchCommittee { height: u64, shard: [u8; 32] }, GetShardKey { height: u64, public_key: PublicKey }, + FetchTemplateRegistrations { from_height: u64 }, } #[derive(Debug, Serialize, Deserialize)] @@ -106,6 +107,9 @@ impl Display for NodeCommsRequest { GetShardKey { height, public_key } => { write!(f, "GetShardKey height ({}), public key ({:?})", height, public_key) }, + FetchTemplateRegistrations { from_height } => { + write!(f, "FetchTemplateRegistrations ({})", from_height) + }, } } } diff --git a/base_layer/core/src/base_node/comms_interface/comms_response.rs b/base_layer/core/src/base_node/comms_interface/comms_response.rs index 6298ba8cc7..21e6a2ec09 100644 --- a/base_layer/core/src/base_node/comms_interface/comms_response.rs +++ b/base_layer/core/src/base_node/comms_interface/comms_response.rs @@ -34,7 +34,12 @@ use crate::{ blocks::{Block, ChainHeader, HistoricalBlock, NewBlockTemplate}, chain_storage::{ActiveValidatorNode, UtxoMinedInfo}, proof_of_work::Difficulty, - transactions::transaction_components::{Transaction, TransactionKernel, TransactionOutput}, + transactions::transaction_components::{ + CodeTemplateRegistration, + Transaction, + TransactionKernel, + TransactionOutput, + }, }; /// API Response enum @@ -74,6 +79,7 @@ pub enum NodeCommsResponse { FetchValidatorNodesKeysResponse(Vec), FetchCommitteeResponse(Vec), GetShardKeyResponse([u8; 32]), + FetchTemplateRegistrationsResponse(Vec), } impl Display for NodeCommsResponse { @@ -115,6 +121,7 @@ impl Display for NodeCommsResponse { FetchValidatorNodesKeysResponse(_) => write!(f, "FetchValidatorNodesKeysResponse"), FetchCommitteeResponse(_) => write!(f, "FetchCommitteeResponse"), GetShardKeyResponse(_) => write!(f, "GetShardKeyResponse"), + FetchTemplateRegistrationsResponse(_) => write!(f, "FetchTemplateRegistrationsResponse"), } } } diff --git a/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs b/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs index 378b77967d..0112e778f3 100644 --- a/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs +++ b/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs @@ -376,6 +376,12 @@ where B: BlockchainBackend + 'static let shard_key = self.blockchain_db.get_shard_key(height, public_key).await?; Ok(NodeCommsResponse::GetShardKeyResponse(shard_key)) }, + NodeCommsRequest::FetchTemplateRegistrations { from_height } => { + let template_registrations = self.blockchain_db.fetch_template_registrations(from_height).await?; + Ok(NodeCommsResponse::FetchTemplateRegistrationsResponse( + template_registrations, + )) + }, } } diff --git a/base_layer/core/src/base_node/comms_interface/local_interface.rs b/base_layer/core/src/base_node/comms_interface/local_interface.rs index be699e725d..5873c678db 100644 --- a/base_layer/core/src/base_node/comms_interface/local_interface.rs +++ b/base_layer/core/src/base_node/comms_interface/local_interface.rs @@ -40,7 +40,7 @@ use crate::{ blocks::{Block, ChainHeader, HistoricalBlock, NewBlockTemplate}, chain_storage::ActiveValidatorNode, proof_of_work::PowAlgorithm, - transactions::transaction_components::{TransactionKernel, TransactionOutput}, + transactions::transaction_components::{CodeTemplateRegistration, TransactionKernel, TransactionOutput}, }; pub type BlockEventSender = broadcast::Sender>; @@ -312,4 +312,18 @@ impl LocalNodeCommsInterface { _ => Err(CommsInterfaceError::UnexpectedApiResponse), } } + + pub async fn get_template_registrations( + &mut self, + from_height: u64, + ) -> Result, CommsInterfaceError> { + match self + .request_sender + .call(NodeCommsRequest::FetchTemplateRegistrations { from_height }) + .await?? + { + NodeCommsResponse::FetchTemplateRegistrationsResponse(template_registrations) => Ok(template_registrations), + _ => Err(CommsInterfaceError::UnexpectedApiResponse), + } + } } diff --git a/base_layer/core/src/chain_storage/async_db.rs b/base_layer/core/src/chain_storage/async_db.rs index 95bccf91b1..747aa55d7c 100644 --- a/base_layer/core/src/chain_storage/async_db.rs +++ b/base_layer/core/src/chain_storage/async_db.rs @@ -62,7 +62,7 @@ use crate::{ }, common::rolling_vec::RollingVec, proof_of_work::{PowAlgorithm, TargetDifficultyWindow}, - transactions::transaction_components::{TransactionKernel, TransactionOutput}, + transactions::transaction_components::{CodeTemplateRegistration, TransactionKernel, TransactionOutput}, }; const LOG_TARGET: &str = "c::bn::async_db"; @@ -271,6 +271,8 @@ impl AsyncBlockchainDb { make_async_fn!(fetch_committee(height: u64, shard: [u8;32]) -> Vec, "fetch_committee"); make_async_fn!(get_shard_key(height:u64, public_key: PublicKey) -> [u8;32], "get_shard_key"); + + make_async_fn!(fetch_template_registrations(from_height: u64) -> Vec, "fetch_template_registrations"); } impl From> for AsyncBlockchainDb { diff --git a/base_layer/core/src/chain_storage/blockchain_backend.rs b/base_layer/core/src/chain_storage/blockchain_backend.rs index c7f0d72626..db6e6ce1ec 100644 --- a/base_layer/core/src/chain_storage/blockchain_backend.rs +++ b/base_layer/core/src/chain_storage/blockchain_backend.rs @@ -31,7 +31,7 @@ use crate::{ Reorg, UtxoMinedInfo, }, - transactions::transaction_components::{TransactionInput, TransactionKernel}, + transactions::transaction_components::{CodeTemplateRegistration, TransactionInput, TransactionKernel}, }; /// Identify behaviour for Blockchain database backends. Implementations must support `Send` and `Sync` so that @@ -196,4 +196,8 @@ pub trait BlockchainBackend: Send + Sync { fn fetch_active_validator_nodes(&self, height: u64) -> Result, ChainStorageError>; fn fetch_committee(&self, height: u64, shard: [u8; 32]) -> Result, ChainStorageError>; fn get_shard_key(&self, height: u64, public_key: PublicKey) -> Result<[u8; 32], ChainStorageError>; + fn fetch_template_registrations( + &self, + from_height: u64, + ) -> Result, ChainStorageError>; } diff --git a/base_layer/core/src/chain_storage/blockchain_database.rs b/base_layer/core/src/chain_storage/blockchain_database.rs index 0b3b882428..c96aff5f72 100644 --- a/base_layer/core/src/chain_storage/blockchain_database.rs +++ b/base_layer/core/src/chain_storage/blockchain_database.rs @@ -80,7 +80,7 @@ use crate::{ common::rolling_vec::RollingVec, consensus::{chain_strength_comparer::ChainStrengthComparer, ConsensusConstants, ConsensusManager}, proof_of_work::{monero_rx::MoneroPowData, PowAlgorithm, TargetDifficultyWindow}, - transactions::transaction_components::{TransactionInput, TransactionKernel}, + transactions::transaction_components::{CodeTemplateRegistration, TransactionInput, TransactionKernel}, validation::{ helpers::calc_median_timestamp, DifficultyCalculator, @@ -1181,6 +1181,14 @@ where B: BlockchainBackend let db = self.db_read_access()?; db.fetch_committee(height, shard) } + + pub fn fetch_template_registrations( + &self, + from_height: u64, + ) -> Result, ChainStorageError> { + let db = self.db_read_access()?; + db.fetch_template_registrations(from_height) + } } fn unexpected_result(request: DbKey, response: DbValue) -> Result { diff --git a/base_layer/core/src/chain_storage/db_transaction.rs b/base_layer/core/src/chain_storage/db_transaction.rs index 105f217f29..9caf4d1fdd 100644 --- a/base_layer/core/src/chain_storage/db_transaction.rs +++ b/base_layer/core/src/chain_storage/db_transaction.rs @@ -34,7 +34,7 @@ use super::ActiveValidatorNode; use crate::{ blocks::{Block, BlockHeader, BlockHeaderAccumulatedData, ChainBlock, ChainHeader, UpdateBlockAccumulatedData}, chain_storage::{error::ChainStorageError, HorizonData, Reorg}, - transactions::transaction_components::{TransactionKernel, TransactionOutput}, + transactions::transaction_components::{CodeTemplateRegistration, TransactionKernel, TransactionOutput}, }; #[derive(Debug)] @@ -365,6 +365,9 @@ pub enum WriteOperation { DeleteValidatorNode { public_key: PublicKey, }, + InsertTemplateRegistration { + template_registration: CodeTemplateRegistration, + }, } impl fmt::Display for WriteOperation { @@ -465,6 +468,9 @@ impl fmt::Display for WriteOperation { write!(f, "Inserting VN {:?}", validator_node) }, DeleteValidatorNode { public_key } => write!(f, "Delete VN key {}", public_key), + InsertTemplateRegistration { template_registration } => { + write!(f, "Inserting Template {:?}", template_registration) + }, } } } diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 8586c28516..8a5ff3e0c7 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -96,7 +96,13 @@ use crate::{ consensus::ConsensusManager, transactions::{ aggregated_body::AggregateBody, - transaction_components::{TransactionError, TransactionInput, TransactionKernel, TransactionOutput}, + transaction_components::{ + CodeTemplateRegistration, + TransactionError, + TransactionInput, + TransactionKernel, + TransactionOutput, + }, }, MutablePrunedOutputMmr, PrunedKernelMmr, @@ -133,6 +139,7 @@ const LMDB_DB_BAD_BLOCK_LIST: &str = "bad_blocks"; const LMDB_DB_REORGS: &str = "reorgs"; const LMDB_DB_VALIDATOR_NODES: &str = "validator_nodes"; const LMDB_DB_VALIDATOR_NODES_MAPPING: &str = "validator_nodes_mapping"; +const LMDB_DB_TEMPLATE_REGISTRATIONS: &str = "template_registrations"; pub fn create_lmdb_database>( path: P, @@ -177,6 +184,7 @@ pub fn create_lmdb_database>( .add_database(LMDB_DB_REORGS, flags | db::INTEGERKEY) .add_database(LMDB_DB_VALIDATOR_NODES, flags | db::DUPSORT) .add_database(LMDB_DB_VALIDATOR_NODES_MAPPING, flags | db::DUPSORT) + .add_database(LMDB_DB_TEMPLATE_REGISTRATIONS, flags | db::DUPSORT) .build() .map_err(|err| ChainStorageError::CriticalError(format!("Could not create LMDB store:{}", err)))?; debug!(target: LOG_TARGET, "LMDB database creation successful"); @@ -239,6 +247,8 @@ pub struct LMDBDatabase { validator_nodes: DatabaseRef, /// Maps VN Shard Key -> VN Public Key validator_nodes_mapping: DatabaseRef, + /// Maps CodeTemplateRegistration hash-> CodeTemplateRegistration + template_registrations: DatabaseRef, _file_lock: Arc, consensus_manager: ConsensusManager, } @@ -281,6 +291,7 @@ impl LMDBDatabase { reorgs: get_database(store, LMDB_DB_REORGS)?, validator_nodes: get_database(store, LMDB_DB_VALIDATOR_NODES)?, validator_nodes_mapping: get_database(store, LMDB_DB_VALIDATOR_NODES_MAPPING)?, + template_registrations: get_database(store, LMDB_DB_TEMPLATE_REGISTRATIONS)?, env, env_config: store.env_config(), _file_lock: Arc::new(file_lock), @@ -491,6 +502,9 @@ impl LMDBDatabase { let shard_key = self.get_vn_mapping(&txn, public_key)?; self.delete_validator_node(&write_txn, public_key, &shard_key)?; }, + InsertTemplateRegistration { template_registration } => { + self.insert_template_registration(&write_txn, template_registration)?; + }, } } write_txn.commit()?; @@ -498,7 +512,7 @@ impl LMDBDatabase { Ok(()) } - fn all_dbs(&self) -> [(&'static str, &DatabaseRef); 26] { + fn all_dbs(&self) -> [(&'static str, &DatabaseRef); 27] { [ ("metadata_db", &self.metadata_db), ("headers_db", &self.headers_db), @@ -532,6 +546,7 @@ impl LMDBDatabase { ("reorgs", &self.reorgs), ("validator_nodes", &self.validator_nodes), ("validator_nodes_mapping", &self.validator_nodes_mapping), + ("template_registrations", &self.template_registrations), ] } @@ -1316,6 +1331,14 @@ impl LMDBDatabase { }; self.insert_validator_node(txn, &validator_node)?; } + if let Some(template_reg) = output + .features + .sidechain_feature + .as_ref() + .and_then(|f| f.template_registration()) + { + self.insert_template_registration(txn, template_reg)?; + } self.insert_output( txn, &block_hash, @@ -1600,6 +1623,21 @@ impl LMDBDatabase { Ok(()) } + fn insert_template_registration( + &self, + txn: &WriteTransaction<'_>, + template_registration: &CodeTemplateRegistration, + ) -> Result<(), ChainStorageError> { + let key = template_registration.hash(); + lmdb_insert( + txn, + &self.template_registrations, + key.as_bytes(), + template_registration, + "template_registrations", + ) + } + fn fetch_output_in_txn( &self, txn: &ConstTransaction<'_>, @@ -2495,6 +2533,15 @@ impl BlockchainBackend for LMDBDatabase { value: public_key.to_hex(), }) } + + fn fetch_template_registrations( + &self, + _from_height: u64, + ) -> Result, ChainStorageError> { + let txn = self.read_transaction()?; + // TODO: filter by block height + lmdb_filter_map_values(&txn, &self.template_registrations, Some) + } } // Fetch the chain metadata diff --git a/base_layer/core/src/test_helpers/blockchain.rs b/base_layer/core/src/test_helpers/blockchain.rs index c37c03cd05..ae6878e36e 100644 --- a/base_layer/core/src/test_helpers/blockchain.rs +++ b/base_layer/core/src/test_helpers/blockchain.rs @@ -74,7 +74,7 @@ use crate::{ proof_of_work::{AchievedTargetDifficulty, Difficulty, PowAlgorithm}, test_helpers::{block_spec::BlockSpecs, create_consensus_rules, BlockSpec}, transactions::{ - transaction_components::{TransactionInput, TransactionKernel, UnblindedOutput}, + transaction_components::{CodeTemplateRegistration, TransactionInput, TransactionKernel, UnblindedOutput}, CryptoFactories, }, validation::{ @@ -428,6 +428,13 @@ impl BlockchainBackend for TempDatabase { fn get_shard_key(&self, height: u64, public_key: PublicKey) -> Result<[u8; 32], ChainStorageError> { self.db.as_ref().unwrap().get_shard_key(height, public_key) } + + fn fetch_template_registrations( + &self, + from_height: u64, + ) -> Result, ChainStorageError> { + self.db.as_ref().unwrap().fetch_template_registrations(from_height) + } } pub fn create_chained_blocks>( diff --git a/base_layer/core/src/transactions/transaction_components/side_chain/template_registration.rs b/base_layer/core/src/transactions/transaction_components/side_chain/template_registration.rs index 83f67156f6..8f3290cd1e 100644 --- a/base_layer/core/src/transactions/transaction_components/side_chain/template_registration.rs +++ b/base_layer/core/src/transactions/transaction_components/side_chain/template_registration.rs @@ -23,15 +23,19 @@ use std::io::{Error, ErrorKind, Read, Write}; use serde::{Deserialize, Serialize}; -use tari_common_types::types::{PublicKey, Signature}; - -use crate::consensus::{ - read_byte, - ConsensusDecoding, - ConsensusEncoding, - ConsensusEncodingSized, - MaxSizeBytes, - MaxSizeString, +use tari_common_types::types::{FixedHash, PublicKey, Signature}; + +use crate::{ + consensus::{ + read_byte, + ConsensusDecoding, + ConsensusEncoding, + ConsensusEncodingSized, + DomainSeparatedConsensusHasher, + MaxSizeBytes, + MaxSizeString, + }, + transactions::TransactionHashDomain, }; #[derive(Debug, Clone, Hash, PartialEq, Eq, Deserialize, Serialize)] @@ -46,6 +50,22 @@ pub struct CodeTemplateRegistration { pub binary_url: MaxSizeString<255>, } +impl CodeTemplateRegistration { + pub fn hash(&self) -> FixedHash { + DomainSeparatedConsensusHasher::::new("template_registration") + .chain(&self.author_public_key) + .chain(&self.author_signature) + .chain(&self.template_name) + .chain(&self.template_version) + .chain(&self.template_type) + .chain(&self.build_info) + .chain(&self.binary_sha) + .chain(&self.binary_url) + .finalize() + .into() + } +} + impl ConsensusEncoding for CodeTemplateRegistration { fn consensus_encode(&self, writer: &mut W) -> Result<(), Error> { self.author_public_key.consensus_encode(writer)?; From d71a01258f6949041e616c37d32a16846d756dff Mon Sep 17 00:00:00 2001 From: mrnaveira <47919901+mrnaveira@users.noreply.github.com> Date: Thu, 22 Sep 2022 16:48:12 -0600 Subject: [PATCH 3/5] store and filter templates by block height --- .../comms_interface/inbound_handlers.rs | 8 ++++- base_layer/core/src/chain_storage/async_db.rs | 6 ++-- .../src/chain_storage/blockchain_backend.rs | 9 ++--- .../src/chain_storage/blockchain_database.rs | 6 ++-- .../core/src/chain_storage/db_transaction.rs | 6 ++-- .../core/src/chain_storage/lmdb_db/lmdb_db.rs | 36 ++++++++++--------- base_layer/core/src/chain_storage/mod.rs | 3 ++ .../src/chain_storage/template_registation.rs | 31 ++++++++++++++++ .../core/src/test_helpers/blockchain.rs | 8 ++--- 9 files changed, 75 insertions(+), 38 deletions(-) create mode 100644 base_layer/core/src/chain_storage/template_registation.rs diff --git a/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs b/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs index 0112e778f3..312fcf2e00 100644 --- a/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs +++ b/base_layer/core/src/base_node/comms_interface/inbound_handlers.rs @@ -377,7 +377,13 @@ where B: BlockchainBackend + 'static Ok(NodeCommsResponse::GetShardKeyResponse(shard_key)) }, NodeCommsRequest::FetchTemplateRegistrations { from_height } => { - let template_registrations = self.blockchain_db.fetch_template_registrations(from_height).await?; + let template_registrations = self + .blockchain_db + .fetch_template_registrations(from_height) + .await? + .into_iter() + .map(|tr| tr.registration_data) + .collect(); Ok(NodeCommsResponse::FetchTemplateRegistrationsResponse( template_registrations, )) diff --git a/base_layer/core/src/chain_storage/async_db.rs b/base_layer/core/src/chain_storage/async_db.rs index 747aa55d7c..bc7fcbfb7d 100644 --- a/base_layer/core/src/chain_storage/async_db.rs +++ b/base_layer/core/src/chain_storage/async_db.rs @@ -30,7 +30,7 @@ use tari_common_types::{ }; use tari_utilities::epoch_time::EpochTime; -use super::ActiveValidatorNode; +use super::{ActiveValidatorNode, TemplateRegistration}; use crate::{ blocks::{ Block, @@ -62,7 +62,7 @@ use crate::{ }, common::rolling_vec::RollingVec, proof_of_work::{PowAlgorithm, TargetDifficultyWindow}, - transactions::transaction_components::{CodeTemplateRegistration, TransactionKernel, TransactionOutput}, + transactions::transaction_components::{TransactionKernel, TransactionOutput}, }; const LOG_TARGET: &str = "c::bn::async_db"; @@ -272,7 +272,7 @@ impl AsyncBlockchainDb { make_async_fn!(get_shard_key(height:u64, public_key: PublicKey) -> [u8;32], "get_shard_key"); - make_async_fn!(fetch_template_registrations(from_height: u64) -> Vec, "fetch_template_registrations"); + make_async_fn!(fetch_template_registrations(from_height: u64) -> Vec, "fetch_template_registrations"); } impl From> for AsyncBlockchainDb { diff --git a/base_layer/core/src/chain_storage/blockchain_backend.rs b/base_layer/core/src/chain_storage/blockchain_backend.rs index db6e6ce1ec..3dc103a441 100644 --- a/base_layer/core/src/chain_storage/blockchain_backend.rs +++ b/base_layer/core/src/chain_storage/blockchain_backend.rs @@ -7,7 +7,7 @@ use tari_common_types::{ types::{Commitment, HashOutput, PublicKey, Signature}, }; -use super::ActiveValidatorNode; +use super::{ActiveValidatorNode, TemplateRegistration}; use crate::{ blocks::{ Block, @@ -31,7 +31,7 @@ use crate::{ Reorg, UtxoMinedInfo, }, - transactions::transaction_components::{CodeTemplateRegistration, TransactionInput, TransactionKernel}, + transactions::transaction_components::{TransactionInput, TransactionKernel}, }; /// Identify behaviour for Blockchain database backends. Implementations must support `Send` and `Sync` so that @@ -196,8 +196,5 @@ pub trait BlockchainBackend: Send + Sync { fn fetch_active_validator_nodes(&self, height: u64) -> Result, ChainStorageError>; fn fetch_committee(&self, height: u64, shard: [u8; 32]) -> Result, ChainStorageError>; fn get_shard_key(&self, height: u64, public_key: PublicKey) -> Result<[u8; 32], ChainStorageError>; - fn fetch_template_registrations( - &self, - from_height: u64, - ) -> Result, ChainStorageError>; + fn fetch_template_registrations(&self, from_height: u64) -> Result, ChainStorageError>; } diff --git a/base_layer/core/src/chain_storage/blockchain_database.rs b/base_layer/core/src/chain_storage/blockchain_database.rs index c96aff5f72..a3aeab03ee 100644 --- a/base_layer/core/src/chain_storage/blockchain_database.rs +++ b/base_layer/core/src/chain_storage/blockchain_database.rs @@ -41,7 +41,7 @@ use tari_common_types::{ use tari_mmr::pruned_hashset::PrunedHashSet; use tari_utilities::{epoch_time::EpochTime, hex::Hex, ByteArray}; -use super::ActiveValidatorNode; +use super::{ActiveValidatorNode, TemplateRegistration}; use crate::{ blocks::{ Block, @@ -80,7 +80,7 @@ use crate::{ common::rolling_vec::RollingVec, consensus::{chain_strength_comparer::ChainStrengthComparer, ConsensusConstants, ConsensusManager}, proof_of_work::{monero_rx::MoneroPowData, PowAlgorithm, TargetDifficultyWindow}, - transactions::transaction_components::{CodeTemplateRegistration, TransactionInput, TransactionKernel}, + transactions::transaction_components::{TransactionInput, TransactionKernel}, validation::{ helpers::calc_median_timestamp, DifficultyCalculator, @@ -1185,7 +1185,7 @@ where B: BlockchainBackend pub fn fetch_template_registrations( &self, from_height: u64, - ) -> Result, ChainStorageError> { + ) -> Result, ChainStorageError> { let db = self.db_read_access()?; db.fetch_template_registrations(from_height) } diff --git a/base_layer/core/src/chain_storage/db_transaction.rs b/base_layer/core/src/chain_storage/db_transaction.rs index 9caf4d1fdd..a3d3d10b8d 100644 --- a/base_layer/core/src/chain_storage/db_transaction.rs +++ b/base_layer/core/src/chain_storage/db_transaction.rs @@ -30,11 +30,11 @@ use croaring::Bitmap; use tari_common_types::types::{BlockHash, Commitment, HashOutput, PublicKey}; use tari_utilities::hex::Hex; -use super::ActiveValidatorNode; +use super::{ActiveValidatorNode, TemplateRegistration}; use crate::{ blocks::{Block, BlockHeader, BlockHeaderAccumulatedData, ChainBlock, ChainHeader, UpdateBlockAccumulatedData}, chain_storage::{error::ChainStorageError, HorizonData, Reorg}, - transactions::transaction_components::{CodeTemplateRegistration, TransactionKernel, TransactionOutput}, + transactions::transaction_components::{TransactionKernel, TransactionOutput}, }; #[derive(Debug)] @@ -366,7 +366,7 @@ pub enum WriteOperation { public_key: PublicKey, }, InsertTemplateRegistration { - template_registration: CodeTemplateRegistration, + template_registration: TemplateRegistration, }, } diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 8a5ff3e0c7..307a152053 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -92,17 +92,12 @@ use crate::{ MmrTree, PrunedOutput, Reorg, + TemplateRegistration, }, consensus::ConsensusManager, transactions::{ aggregated_body::AggregateBody, - transaction_components::{ - CodeTemplateRegistration, - TransactionError, - TransactionInput, - TransactionKernel, - TransactionOutput, - }, + transaction_components::{TransactionError, TransactionInput, TransactionKernel, TransactionOutput}, }, MutablePrunedOutputMmr, PrunedKernelMmr, @@ -247,7 +242,7 @@ pub struct LMDBDatabase { validator_nodes: DatabaseRef, /// Maps VN Shard Key -> VN Public Key validator_nodes_mapping: DatabaseRef, - /// Maps CodeTemplateRegistration hash-> CodeTemplateRegistration + /// Maps CodeTemplateRegistration hash-> TemplateRegistration template_registrations: DatabaseRef, _file_lock: Arc, consensus_manager: ConsensusManager, @@ -1337,7 +1332,12 @@ impl LMDBDatabase { .as_ref() .and_then(|f| f.template_registration()) { - self.insert_template_registration(txn, template_reg)?; + let record = TemplateRegistration { + registration_data: template_reg.clone(), + height: header.height, + }; + + self.insert_template_registration(txn, &record)?; } self.insert_output( txn, @@ -1626,9 +1626,9 @@ impl LMDBDatabase { fn insert_template_registration( &self, txn: &WriteTransaction<'_>, - template_registration: &CodeTemplateRegistration, + template_registration: &TemplateRegistration, ) -> Result<(), ChainStorageError> { - let key = template_registration.hash(); + let key = template_registration.registration_data.hash(); lmdb_insert( txn, &self.template_registrations, @@ -2534,13 +2534,15 @@ impl BlockchainBackend for LMDBDatabase { }) } - fn fetch_template_registrations( - &self, - _from_height: u64, - ) -> Result, ChainStorageError> { + fn fetch_template_registrations(&self, from_height: u64) -> Result, ChainStorageError> { let txn = self.read_transaction()?; - // TODO: filter by block height - lmdb_filter_map_values(&txn, &self.template_registrations, Some) + lmdb_filter_map_values(&txn, &self.template_registrations, |tr: TemplateRegistration| { + if tr.height >= from_height { + Some(tr) + } else { + None + } + }) } } diff --git a/base_layer/core/src/chain_storage/mod.rs b/base_layer/core/src/chain_storage/mod.rs index d374dccf1b..aa65f98a95 100644 --- a/base_layer/core/src/chain_storage/mod.rs +++ b/base_layer/core/src/chain_storage/mod.rs @@ -82,3 +82,6 @@ pub use utxo_mined_info::*; mod active_validator_node; pub use active_validator_node::ActiveValidatorNode; + +mod template_registation; +pub use template_registation::TemplateRegistration; diff --git a/base_layer/core/src/chain_storage/template_registation.rs b/base_layer/core/src/chain_storage/template_registation.rs new file mode 100644 index 0000000000..452fc02ef6 --- /dev/null +++ b/base_layer/core/src/chain_storage/template_registation.rs @@ -0,0 +1,31 @@ +// Copyright 2022, The Tari Project +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +// following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +// disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +// products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use serde::{Deserialize, Serialize}; + +use crate::transactions::transaction_components::CodeTemplateRegistration; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct TemplateRegistration { + pub registration_data: CodeTemplateRegistration, + pub height: u64, +} diff --git a/base_layer/core/src/test_helpers/blockchain.rs b/base_layer/core/src/test_helpers/blockchain.rs index ae6878e36e..b23728a05c 100644 --- a/base_layer/core/src/test_helpers/blockchain.rs +++ b/base_layer/core/src/test_helpers/blockchain.rs @@ -67,6 +67,7 @@ use crate::{ MmrTree, PrunedOutput, Reorg, + TemplateRegistration, UtxoMinedInfo, Validators, }, @@ -74,7 +75,7 @@ use crate::{ proof_of_work::{AchievedTargetDifficulty, Difficulty, PowAlgorithm}, test_helpers::{block_spec::BlockSpecs, create_consensus_rules, BlockSpec}, transactions::{ - transaction_components::{CodeTemplateRegistration, TransactionInput, TransactionKernel, UnblindedOutput}, + transaction_components::{TransactionInput, TransactionKernel, UnblindedOutput}, CryptoFactories, }, validation::{ @@ -429,10 +430,7 @@ impl BlockchainBackend for TempDatabase { self.db.as_ref().unwrap().get_shard_key(height, public_key) } - fn fetch_template_registrations( - &self, - from_height: u64, - ) -> Result, ChainStorageError> { + fn fetch_template_registrations(&self, from_height: u64) -> Result, ChainStorageError> { self.db.as_ref().unwrap().fetch_template_registrations(from_height) } } From e45d377b5d275e318555f25eeab86d5eeca629b9 Mon Sep 17 00:00:00 2001 From: mrnaveira <47919901+mrnaveira@users.noreply.github.com> Date: Tue, 27 Sep 2022 10:26:00 -0600 Subject: [PATCH 4/5] fix clippy warning --- applications/tari_base_node/src/grpc/base_node_grpc_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/tari_base_node/src/grpc/base_node_grpc_server.rs b/applications/tari_base_node/src/grpc/base_node_grpc_server.rs index af02ec0ac5..09bd6783b7 100644 --- a/applications/tari_base_node/src/grpc/base_node_grpc_server.rs +++ b/applications/tari_base_node/src/grpc/base_node_grpc_server.rs @@ -1489,7 +1489,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer { }, Ok(data) => data, }; - dbg!(&active_validator_nodes); + // dbg!(&active_validator_nodes); for (public_key, shard_key) in active_validator_nodes { let active_validator_node = tari_rpc::GetActiveValidatorNodesResponse { public_key: public_key.to_vec(), From 4288493aa821aafc2448a65b05e0cc996bb10e5b Mon Sep 17 00:00:00 2001 From: mrnaveira <47919901+mrnaveira@users.noreply.github.com> Date: Tue, 27 Sep 2022 10:27:06 -0600 Subject: [PATCH 5/5] add comment for future lmdb optimization --- base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 6ee0009266..50238f03ce 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -2578,6 +2578,7 @@ impl BlockchainBackend for LMDBDatabase { } fn fetch_template_registrations(&self, from_height: u64) -> Result, ChainStorageError> { + // TODO: we can optimise this query by making using a compound key let txn = self.read_transaction()?; lmdb_filter_map_values(&txn, &self.template_registrations, |tr: TemplateRegistration| { if tr.height >= from_height {