From 0b6fc8f2763f99d2cc686b2015299e0dd8d001b6 Mon Sep 17 00:00:00 2001 From: Bartosz Zawistowski <39065214+bzawisto@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:58:57 +0200 Subject: [PATCH] [q4-working-branch] Q4 working branch bartosz (#3775) * Fixing test * Increasing POW window * Fixing errorneous test * Some in-progress work * Change genesis account * Doesn't compile but we're getting there * Further progress with desharding * Further work with txns dispatching * Further progress with desharding * Comilation fix --------- Co-authored-by: bzawisto --- src/cmd/genaccounts.cpp | 3 - src/common/Constants.cpp | 8 +- src/common/Constants.h | 1 - src/libData/AccountData/Transaction.cpp | 23 - src/libData/AccountData/Transaction.h | 6 - src/libData/AccountStore/AccountStoreSC.cpp | 21 +- .../DSBlockPostProcessing.cpp | 14 +- .../DSBlockPreProcessing.cpp | 6 +- src/libDirectoryService/DSComposition.cpp | 18 + src/libDirectoryService/DirectoryService.cpp | 36 +- src/libDirectoryService/DirectoryService.h | 10 +- .../FinalBlockPostProcessing.cpp | 8 +- .../FinalBlockPreProcessing.cpp | 8 +- src/libLookup/Lookup.cpp | 451 +++++------------- src/libLookup/Lookup.h | 57 +-- src/libMessage/Messenger.cpp | 232 ++------- src/libMessage/Messenger.h | 17 +- src/libMessage/ZilliqaMessage.proto | 3 +- src/libNode/DSBlockProcessing.cpp | 8 +- src/libNode/FinalBlockProcessing.cpp | 54 ++- src/libNode/Node.cpp | 40 +- src/libNode/Node.h | 4 +- src/libNode/PoWProcessing.cpp | 1 + src/libServer/EthRpcMethods.cpp | 365 ++++++-------- src/libServer/EthRpcMethods.h | 104 ++-- src/libServer/LookupServer.cpp | 167 ++----- src/libServer/LookupServer.h | 21 +- src/libUtils/GetTxnFromFile.h | 1 - src/libValidator/Validator.cpp | 33 -- src/libZilliqa/Zilliqa.cpp | 3 +- tests/Lookup/Test_txn_send.cpp | 4 +- 31 files changed, 582 insertions(+), 1145 deletions(-) diff --git a/src/cmd/genaccounts.cpp b/src/cmd/genaccounts.cpp index 07433d4720..e8609456f8 100644 --- a/src/cmd/genaccounts.cpp +++ b/src/cmd/genaccounts.cpp @@ -71,9 +71,6 @@ int main(int argc, char** argv) { while (true) { PairOfKey keypair = Schnorr::GenKeyPair(); Address addr = CryptoUtils::GetAddressFromPubKey(keypair.second); - if (Transaction::GetShardIndex(addr, numShards) != targetShard) { - continue; - } cout << "\t\t" << endl; cout << "\t\t\t" << keypair.first << "" << endl; diff --git a/src/common/Constants.cpp b/src/common/Constants.cpp index 4470b04775..5050ffb015 100644 --- a/src/common/Constants.cpp +++ b/src/common/Constants.cpp @@ -267,8 +267,6 @@ const unsigned int DS_ANNOUNCEMENT_DELAY_IN_MS{ ReadConstantNumeric("DS_ANNOUNCEMENT_DELAY_IN_MS", "node.epoch_timing.")}; const unsigned int SHARD_ANNOUNCEMENT_DELAY_IN_MS{ReadConstantNumeric( "SHARD_ANNOUNCEMENT_DELAY_IN_MS", "node.epoch_timing.")}; -const unsigned int LOOKUP_DELAY_SEND_TXNPACKET_IN_MS{ReadConstantNumeric( - "LOOKUP_DELAY_SEND_TXNPACKET_IN_MS", "node.epoch_timing.")}; const unsigned int MICROBLOCK_TIMEOUT{ ReadConstantNumeric("MICROBLOCK_TIMEOUT", "node.epoch_timing.")}; const unsigned int NEW_NODE_SYNC_INTERVAL{ @@ -440,8 +438,10 @@ const unsigned int CONNECTION_ALL_TIMEOUT{ ReadConstantNumeric("CONNECTION_ALL_TIMEOUT", "node.jsonrpc.")}; const unsigned int CONNECTION_CALLBACK_TIMEOUT{ ReadConstantNumeric("CONNECTION_CALLBACK_TIMEOUT", "node.jsonrpc.")}; -const size_t REQUEST_PROCESSING_THREADS{ReadConstantNumeric("REQUEST_PROCESSING_THREADS", "node.jsonrpc.", 64)}; -const size_t REQUEST_QUEUE_SIZE{ReadConstantNumeric("REQUEST_QUEUE_SIZE", "node.jsonrpc.", 65536)}; +const size_t REQUEST_PROCESSING_THREADS{ + ReadConstantNumeric("REQUEST_PROCESSING_THREADS", "node.jsonrpc.", 64)}; +const size_t REQUEST_QUEUE_SIZE{ + ReadConstantNumeric("REQUEST_QUEUE_SIZE", "node.jsonrpc.", 65536)}; // Network composition constants const unsigned int COMM_SIZE{ diff --git a/src/common/Constants.h b/src/common/Constants.h index dab5ab5eba..0edcf07fe1 100644 --- a/src/common/Constants.h +++ b/src/common/Constants.h @@ -212,7 +212,6 @@ extern const unsigned int DELAY_FIRSTXNEPOCH_IN_MS; extern const unsigned int FETCHING_MISSING_DATA_TIMEOUT; extern const unsigned int DS_ANNOUNCEMENT_DELAY_IN_MS; extern const unsigned int SHARD_ANNOUNCEMENT_DELAY_IN_MS; -extern const unsigned int LOOKUP_DELAY_SEND_TXNPACKET_IN_MS; extern const unsigned int MICROBLOCK_TIMEOUT; extern const unsigned int NEW_NODE_SYNC_INTERVAL; extern const unsigned int POW_SUBMISSION_TIMEOUT; diff --git a/src/libData/AccountData/Transaction.cpp b/src/libData/AccountData/Transaction.cpp index 9edc3a4205..9e299896b1 100644 --- a/src/libData/AccountData/Transaction.cpp +++ b/src/libData/AccountData/Transaction.cpp @@ -313,29 +313,6 @@ void Transaction::SetSignature(const Signature &signature) { m_signature = signature; } -unsigned int Transaction::GetShardIndex(const Address &fromAddr, - unsigned int numShards) { - uint32_t x = 0; - - if (numShards == 0) { - LOG_GENERAL(WARNING, "numShards is 0 and trying to calculate shard index"); - return 0; - } - - // Take the last four bytes of the address - for (unsigned int i = 0; i < 4; i++) { - x = (x << 8) | fromAddr.asArray().at(ACC_ADDR_SIZE - 4 + i); - } - - return x % numShards; -} - -unsigned int Transaction::GetShardIndex(unsigned int numShards) const { - const auto &fromAddr = GetSenderAddr(); - - return GetShardIndex(fromAddr, numShards); -} - bool Transaction::Verify(const Transaction &tran) { zbytes txnData; tran.SerializeCoreFields(txnData, 0); diff --git a/src/libData/AccountData/Transaction.h b/src/libData/AccountData/Transaction.h index f633ae8363..01866deb8e 100644 --- a/src/libData/AccountData/Transaction.h +++ b/src/libData/AccountData/Transaction.h @@ -176,15 +176,9 @@ class Transaction : public SerializableDataBlock { /// Returns the EC-Schnorr signature over the transaction data. const Signature& GetSignature() const; - unsigned int GetShardIndex(unsigned int numShards) const; - /// Set the signature void SetSignature(const Signature& signature); - /// Identifies the shard number that should process the transaction. - static unsigned int GetShardIndex(const Address& fromAddr, - unsigned int numShards); - enum ContractType { NON_CONTRACT = 0, CONTRACT_CREATION, diff --git a/src/libData/AccountStore/AccountStoreSC.cpp b/src/libData/AccountStore/AccountStoreSC.cpp index 4c99b33154..cac9b9a919 100644 --- a/src/libData/AccountStore/AccountStoreSC.cpp +++ b/src/libData/AccountStore/AccountStoreSC.cpp @@ -16,7 +16,6 @@ */ #include - #include #include @@ -250,8 +249,7 @@ bool AccountStoreSC::UpdateAccounts( .blockTimestamp = extras.block_timestamp, .blockDifficulty = extras.block_difficulty, .contractType = Transaction::GetTransactionType(transaction), - .txnHash = transaction.GetTranID() - }; + .txnHash = transaction.GetTranID()}; AccountStoreCpsInterface acCpsInterface{*this}; libCps::CpsExecutor cpsExecutor{acCpsInterface, receipt}; @@ -264,7 +262,6 @@ bool AccountStoreSC::UpdateAccounts( } error_code = cpsRunResult.txnStatus; - return cpsRunResult.isSuccess; } @@ -1541,18 +1538,6 @@ bool AccountStoreSC::ParseCallContractJsonOutput(const Json::Value &_json, gasRemained -= SCILLA_RUNNER_INVOKE_GAS; } - // check whether the recipient contract is in the same shard with the - // current contract - if (!m_curIsDS && - (Transaction::GetShardIndex(curContractAddr, m_curNumShards) != - Transaction::GetShardIndex(recipient, m_curNumShards))) { - LOG_GENERAL(WARNING, - "another contract doesn't belong to the same shard with " - "current contract"); - receipt.AddError(CHAIN_CALL_DIFF_SHARD); - return false; - } - if (m_curEdges > MAX_CONTRACT_EDGES) { LOG_GENERAL( WARNING, @@ -1691,7 +1676,9 @@ bool AccountStoreSC::TransferBalanceAtomic(const Address &from, const Address &to, const uint128_t &delta) { // LOG_MARKER(); - LOG_GENERAL(WARNING, "AccountStoreSC::TransferBalanceAtomicTransferBalanceAtomic from " << from << ", to: " << to << ", value: " << delta); + LOG_GENERAL(WARNING, + "AccountStoreSC::TransferBalanceAtomicTransferBalanceAtomic from " + << from << ", to: " << to << ", value: " << delta); return m_accountStoreAtomic->TransferBalance(from, to, delta); } diff --git a/src/libDirectoryService/DSBlockPostProcessing.cpp b/src/libDirectoryService/DSBlockPostProcessing.cpp index d8dffc46f8..8a59bdfbc3 100644 --- a/src/libDirectoryService/DSBlockPostProcessing.cpp +++ b/src/libDirectoryService/DSBlockPostProcessing.cpp @@ -253,11 +253,11 @@ void DirectoryService::UpdateMyDSModeAndConsensusId() { // 2. My node was removed by the DS Committee due to lack of sufficient // performance. bool isDropout = true; - for (auto it = m_mediator.m_DSCommittee->begin(); - it != m_mediator.m_DSCommittee->end(); ++it) { + for (auto it = m_mediator.m_DSCommittee->cbegin(); + it != m_mediator.m_DSCommittee->cend(); ++it) { // Look for my public key. if (m_mediator.m_selfKey.second == it->first) { - m_consensusMyID = std::distance(m_mediator.m_DSCommittee->begin(), it); + m_consensusMyID = std::distance(m_mediator.m_DSCommittee->cbegin(), it); isDropout = false; break; } @@ -325,12 +325,14 @@ void DirectoryService::UpdateDSCommitteeComposition() { LOG_MARKER(); std::lock_guard g(m_mediator.m_mutexDSCommittee); - auto old_size = m_mediator.m_DSCommittee->size(); + const bool leader = m_mediator.m_ds->GetConsensusMyID() == + m_mediator.m_ds->GetConsensusMyID(); + LOG_GENERAL(WARNING, "BZ UpdateDSCommitteeComposition enter, I am leader? : " + << (leader ? "true" : "false")); + UpdateDSCommitteeCompositionCore(m_mediator.m_selfKey.second, *m_mediator.m_DSCommittee, m_mediator.m_dsBlockChain.GetLastBlock()); - LOG_EXTRA("m_DSCommittee updated " << old_size << "->" - << m_mediator.m_DSCommittee->size()); } void DirectoryService::StartNextTxEpoch() { diff --git a/src/libDirectoryService/DSBlockPreProcessing.cpp b/src/libDirectoryService/DSBlockPreProcessing.cpp index 1f107ef517..7622a74ccc 100644 --- a/src/libDirectoryService/DSBlockPreProcessing.cpp +++ b/src/libDirectoryService/DSBlockPreProcessing.cpp @@ -1440,7 +1440,7 @@ bool DirectoryService::ProcessShardingStructure( void DirectoryService::SaveDSPerformanceCore( std::map>>& coinbaseRewardees, - std::map& dsMemberPerformance, DequeOfNode& dsComm, + std::map& dsMemberPerformance, const DequeOfNode& dsComm, uint64_t currentEpochNum, unsigned int numOfFinalBlock, int finalblockRewardID) { LOG_MARKER(); @@ -1511,7 +1511,7 @@ unsigned int DirectoryService::DetermineByzantineNodesCore( unsigned int numOfProposedDSMembers, std::vector& removeDSNodePubkeys, uint64_t currentEpochNum, unsigned int numOfFinalBlock, double performanceThreshold, - unsigned int maxByzantineRemoved, DequeOfNode& dsComm, + unsigned int maxByzantineRemoved, const DequeOfNode& dsComm, const std::map& dsMemberPerformance) { LOG_MARKER(); @@ -1538,7 +1538,7 @@ unsigned int DirectoryService::DetermineByzantineNodesCore( INFO, "threshold = " << threshold << " (" << performanceThreshold << ")"); unsigned int numByzantine = 0; unsigned int index = 0; - for (auto it = dsComm.begin(); it != dsComm.end(); ++it) { + for (auto it = dsComm.cbegin(); it != dsComm.cend(); ++it) { // Do not evaluate guard nodes. if (GUARD_MODE && Guard::GetInstance().IsNodeInDSGuardList(it->first)) { continue; diff --git a/src/libDirectoryService/DSComposition.cpp b/src/libDirectoryService/DSComposition.cpp index 2262b9df54..70275e7d94 100644 --- a/src/libDirectoryService/DSComposition.cpp +++ b/src/libDirectoryService/DSComposition.cpp @@ -40,6 +40,9 @@ void UpdateDSCommitteeCompositionCore(const PubKey& selfKeyPub, const auto& NewDSMembers = dsblock.GetHeader().GetDSPoWWinners(); unsigned int NumWinners = NewDSMembers.size(); + LOG_GENERAL(WARNING, "BZ UpdateDSCommitteeCompositionCore enter, winners: " + << NumWinners); + // Get the vector of all non-performant nodes to be removed. const auto& removeDSNodePubkeys = dsblock.GetHeader().GetDSRemovePubKeys(); @@ -72,6 +75,9 @@ void UpdateDSCommitteeCompositionCore(const PubKey& selfKeyPub, dsComm.emplace_back(*it); dsComm.erase(it); + LOG_GENERAL(WARNING, "BZ Pushing node to the end: " + << it->second.GetPrintableIPAddress()); + continue; } @@ -84,11 +90,15 @@ void UpdateDSCommitteeCompositionCore(const PubKey& selfKeyPub, // Peer() is required because my own node's network information is // zeroed out. dsComm.emplace_front(selfKeyPub, Peer()); + LOG_GENERAL(WARNING, "BZ Myself Pushing non-guard to front: " + << DSPowWinner.second.GetPrintableIPAddress()); } else { // Calculate the position to insert the current winner. it = dsComm.begin() + (Guard::GetInstance().GetNumOfDSGuard()); // Place my node's information in front of the DS Committee Community // Nodes. + LOG_GENERAL(WARNING, "BZ Myself Pushing guard to proper position: " + << DSPowWinner.second.GetPrintableIPAddress()); dsComm.emplace(it, selfKeyPub, Peer()); } } else { @@ -96,11 +106,15 @@ void UpdateDSCommitteeCompositionCore(const PubKey& selfKeyPub, // Place the current winner node's information in front of the DS // Committee. dsComm.emplace_front(DSPowWinner); + LOG_GENERAL(WARNING, "BZ Other Pushing non-guard to front: " + << DSPowWinner.second.GetPrintableIPAddress()); } else { // Calculate the position to insert the current winner. it = dsComm.begin() + (Guard::GetInstance().GetNumOfDSGuard()); // Place the winner's information in front of the DS Committee Community // Nodes. + LOG_GENERAL(WARNING, "BZ Other Pushing guard to proper position: " + << DSPowWinner.second.GetPrintableIPAddress()); dsComm.emplace(it, DSPowWinner); } } @@ -132,10 +146,14 @@ void UpdateDSCommitteeCompositionCore(const PubKey& selfKeyPub, } if (LOOKUP_NODE_MODE && !bStoreDSCommittee) { + LOG_GENERAL(WARNING, "BZ Adding ejected node: " + << dsComm.back().second.GetPrintableIPAddress()); minerInfo.m_dsNodesEjected.emplace_back(dsComm.back().first); } // Remove this node from blacklist if it exists + LOG_GENERAL(WARNING, "BZ Removing from dsComm node: " + << dsComm.back().second.GetPrintableIPAddress()); Peer& p = dsComm.back().second; Blacklist::GetInstance().Remove({p.GetIpAddress(),p.GetListenPortHost(),p.GetNodeIndentifier()}); dsComm.pop_back(); diff --git a/src/libDirectoryService/DirectoryService.cpp b/src/libDirectoryService/DirectoryService.cpp index bd49b2c274..75252683f8 100644 --- a/src/libDirectoryService/DirectoryService.cpp +++ b/src/libDirectoryService/DirectoryService.cpp @@ -186,12 +186,6 @@ bool DirectoryService::CheckState(Action action) { return true; } -uint32_t DirectoryService::GetNumShards() const { - lock_guard g(m_mutexShards); - - return m_shards.size(); -} - bool DirectoryService::ProcessSetPrimary( const zbytes& message, unsigned int offset, [[gnu::unused]] const Peer& from, @@ -206,6 +200,7 @@ bool DirectoryService::ProcessSetPrimary( // Note: This function should only be invoked during bootstrap sequence // Message = [Primary node IP] [Primary node port] LOG_MARKER(); + LOG_GENERAL(WARNING, "BZ: ProcessSetPrimary 1"); if (m_mediator.m_currentEpochNum > 1) { // TODO: Get the IP address of who send this message, and deduct its @@ -271,7 +266,7 @@ bool DirectoryService::ProcessSetPrimary( // Load the DS committee, with my own peer set to dummy m_mediator.m_lookup->SetDSCommitteInfo(true); } - + LOG_GENERAL(WARNING, "BZ: ProcessSetPrimary 2"); // Lets start the gossip as earliest as possible if (BROADCAST_GOSSIP_MODE) { VectorOfNode peers; @@ -321,7 +316,7 @@ bool DirectoryService::ProcessSetPrimary( Guard::GetInstance().AddDSGuardToBlacklistExcludeList( *m_mediator.m_DSCommittee); } - + LOG_GENERAL(WARNING, "BZ: ProcessSetPrimary 3"); SetConsensusLeaderID(0); if (m_mediator.m_currentEpochNum > 1) { LOG_GENERAL(WARNING, "ProcessSetPrimary called in epoch " @@ -350,11 +345,12 @@ bool DirectoryService::ProcessSetPrimary( << "][" << std::setw(6) << std::left << m_consensusMyID << "] DSBK"); } - + LOG_GENERAL(WARNING, "BZ: ProcessSetPrimary 4"); if ((m_consensusMyID < POW_PACKET_SENDERS) || (primary == m_mediator.m_selfPeer)) { m_powSubmissionWindowExpired = false; LOG_GENERAL(INFO, "m_consensusMyID: " << m_consensusMyID); + LOG_GENERAL(WARNING, "BZ: ProcessSetPrimary 11"); LOG_EPOCH(INFO, m_mediator.m_currentEpochNum, "Waiting " << POW_WINDOW_IN_SECONDS << " seconds, accepting PoW submissions..."); @@ -366,7 +362,7 @@ bool DirectoryService::ProcessSetPrimary( this->SendPoWPacketSubmissionToOtherDSComm(); }; DetachedFunction(1, func); - + LOG_GENERAL(WARNING, "BZ: ProcessSetPrimary 22"); LOG_EPOCH(INFO, m_mediator.m_currentEpochNum, "Waiting " << POWPACKETSUBMISSION_WINDOW_IN_SECONDS << " seconds, accepting PoW submissions packet from " @@ -1010,10 +1006,12 @@ bool DirectoryService::ProcessNewDSGuardNetworkInfo( if (m_mediator.m_DSCommittee->at(indexOfDSGuard).first == dsGuardPubkey) { foundDSGuardNode = true; - Blacklist::GetInstance().RemoveFromWhitelist({ - m_mediator.m_DSCommittee->at(indexOfDSGuard).second.m_ipAddress, - m_mediator.m_DSCommittee->at(indexOfDSGuard).second.m_listenPortHost, - m_mediator.m_DSCommittee->at(indexOfDSGuard).second.GetNodeIndentifier()} ); + Blacklist::GetInstance().RemoveFromWhitelist( + {m_mediator.m_DSCommittee->at(indexOfDSGuard).second.m_ipAddress, + m_mediator.m_DSCommittee->at(indexOfDSGuard) + .second.m_listenPortHost, + m_mediator.m_DSCommittee->at(indexOfDSGuard) + .second.GetNodeIndentifier()}); LOG_GENERAL(INFO, "Removed " << m_mediator.m_DSCommittee->at(indexOfDSGuard).second @@ -1027,9 +1025,10 @@ bool DirectoryService::ProcessNewDSGuardNetworkInfo( dsGuardNewNetworkInfo; if (GUARD_MODE) { - Blacklist::GetInstance().Whitelist({dsGuardNewNetworkInfo.m_ipAddress, - dsGuardNewNetworkInfo.m_listenPortHost, - dsGuardNewNetworkInfo.GetNodeIndentifier()}); + Blacklist::GetInstance().Whitelist( + {dsGuardNewNetworkInfo.m_ipAddress, + dsGuardNewNetworkInfo.m_listenPortHost, + dsGuardNewNetworkInfo.GetNodeIndentifier()}); LOG_GENERAL(INFO, "Added ds guard " << dsGuardNewNetworkInfo << " to blacklist exclude list"); } @@ -1207,7 +1206,8 @@ bool DirectoryService::Execute(const zbytes& message, unsigned int offset, LOG_EPOCH(WARNING, m_mediator.m_currentEpochNum, "Ignore DS message"); return false; } - + LOG_GENERAL(WARNING, + "BZ Dispatching DS msg type: " << hex << (unsigned int)ins_byte); if (ins_byte < ins_handlers_count) { result = (this->*ins_handlers[ins_byte])(message, offset + 1, from, startByte); diff --git a/src/libDirectoryService/DirectoryService.h b/src/libDirectoryService/DirectoryService.h index 4756de8d23..d618b46fad 100644 --- a/src/libDirectoryService/DirectoryService.h +++ b/src/libDirectoryService/DirectoryService.h @@ -581,7 +581,7 @@ class DirectoryService : public Executable { bool m_dsEpochAfterUpgrade = false; // GetShards - uint32_t GetNumShards() const; + uint32_t GetNumShards() const { return 1; } /// Force multicast when sending block to shard std::atomic m_forceMulticast{}; @@ -713,14 +713,14 @@ class DirectoryService : public Executable { static void SaveDSPerformanceCore( std::map>>& coinbaseRewardees, - std::map& dsMemberPerformance, DequeOfNode& dsComm, - uint64_t currentEpochNum, unsigned int numOfFinalBlock, - int finalblockRewardID); + std::map& dsMemberPerformance, + const DequeOfNode& dsComm, uint64_t currentEpochNum, + unsigned int numOfFinalBlock, int finalblockRewardID); static unsigned int DetermineByzantineNodesCore( unsigned int numOfProposedDSMembers, std::vector& removeDSNodePubkeys, uint64_t currentEpochNum, unsigned int numOfFinalBlock, double performanceThreshold, - unsigned int maxByzantineRemoved, DequeOfNode& dsComm, + unsigned int maxByzantineRemoved, const DequeOfNode& dsComm, const std::map& dsMemberPerformance); private: diff --git a/src/libDirectoryService/FinalBlockPostProcessing.cpp b/src/libDirectoryService/FinalBlockPostProcessing.cpp index 398e2e6c4e..15aa4c7bd1 100644 --- a/src/libDirectoryService/FinalBlockPostProcessing.cpp +++ b/src/libDirectoryService/FinalBlockPostProcessing.cpp @@ -54,8 +54,9 @@ class FinalBlockPostProcessingVariables { void Init() { if (!temp) { - temp = std::make_unique(Z_FL::BLOCKS, "finalblockpostproc.gauge", - "Final block post processing state", "calls", true); + temp = std::make_unique( + Z_FL::BLOCKS, "finalblockpostproc.gauge", + "Final block post processing state", "calls", true); temp->SetCallback([this](auto&& result) { result.Set(mbInFinal, {{"counter", "MbInFinal"}}); @@ -331,6 +332,9 @@ void DirectoryService::ProcessFinalBlockConsensusWhenDone() { ClearDSPoWSolns(); ResetPoWSubmissionCounter(); if (isVacuousEpoch) { + LOG_GENERAL(WARNING, + "BZ FinalBlockConsensusDone, vacuous epoch, setting state to " + "POW_SUBMISSION"); SetState(POW_SUBMISSION); } else { SetState(MICROBLOCK_SUBMISSION); diff --git a/src/libDirectoryService/FinalBlockPreProcessing.cpp b/src/libDirectoryService/FinalBlockPreProcessing.cpp index 6c3255c5b0..a7e1f8d701 100644 --- a/src/libDirectoryService/FinalBlockPreProcessing.cpp +++ b/src/libDirectoryService/FinalBlockPreProcessing.cpp @@ -234,8 +234,8 @@ bool DirectoryService::RunConsensusOnFinalBlockWhenDSPrimary() { // No other shards exists, then allow additional time for txns distribution. if (m_mediator.m_ds->m_mode != DirectoryService::Mode::IDLE && m_mediator.m_node->m_myshardId == 0 && !m_mediator.GetIsVacuousEpoch()) { - std::this_thread::sleep_for(chrono::milliseconds( - EXTRA_TX_DISTRIBUTE_TIME_IN_MS + LOOKUP_DELAY_SEND_TXNPACKET_IN_MS)); + std::this_thread::sleep_for( + chrono::milliseconds(EXTRA_TX_DISTRIBUTE_TIME_IN_MS)); } m_mediator.m_node->m_txn_distribute_window_open = false; @@ -1203,8 +1203,8 @@ bool DirectoryService::RunConsensusOnFinalBlockWhenDSBackup() { // No other shards exists, then allow additional time for txns distribution. if (m_mediator.m_ds->m_mode != DirectoryService::Mode::IDLE && m_mediator.m_node->m_myshardId == 0 && !m_mediator.GetIsVacuousEpoch()) { - std::this_thread::sleep_for(chrono::milliseconds( - EXTRA_TX_DISTRIBUTE_TIME_IN_MS + LOOKUP_DELAY_SEND_TXNPACKET_IN_MS)); + std::this_thread::sleep_for( + chrono::milliseconds(EXTRA_TX_DISTRIBUTE_TIME_IN_MS)); } m_mediator.m_node->m_txn_distribute_window_open = false; diff --git a/src/libLookup/Lookup.cpp b/src/libLookup/Lookup.cpp index 070022f9d2..3cd942af2b 100644 --- a/src/libLookup/Lookup.cpp +++ b/src/libLookup/Lookup.cpp @@ -276,7 +276,6 @@ void Lookup::SetLookupNodes(const VectorOfNode& lookupNodes) { void Lookup::SetLookupNodes() { std::lock_guard lock(m_mutexLookupNodes); - m_startedTxnBatchThread = false; m_lookupNodes.clear(); m_lookupNodesOffline.clear(); // Populate tree structure pt @@ -474,9 +473,7 @@ VectorOfNode Lookup::GetSeedNodes() const { } } -bool Lookup::GenTxnToSend(size_t num_txn, vector& shardTxn, - vector& DSTxn) { - vector txns; +bool Lookup::GenTxnToSend(size_t num_txn, vector& txnContainer) { unsigned int NUM_TXN_TO_DS = num_txn / GENESIS_WALLETS.size(); for (auto& addrStr : GENESIS_WALLETS) { @@ -486,8 +483,6 @@ bool Lookup::GenTxnToSend(size_t num_txn, vector& shardTxn, } Address addr{tempAddrBytes}; - txns.clear(); - uint64_t nonce; { @@ -505,35 +500,26 @@ bool Lookup::GenTxnToSend(size_t num_txn, vector& shardTxn, } if (!GetTxnFromFile::GetFromFile(addr, static_cast(nonce) + 1, - num_txn, txns)) { + num_txn, txnContainer)) { LOG_GENERAL(WARNING, "Failed to get txns from file"); continue; } - copy(txns.begin(), txns.end(), back_inserter(shardTxn)); - LOG_GENERAL(INFO, "[Batching] Last Nonce sent " << nonce + num_txn << " of Addr " << addr.hex()); - txns.clear(); - if (!GetTxnFromFile::GetFromFile(addr, static_cast(nonce) + num_txn + 1, - NUM_TXN_TO_DS, txns)) { + NUM_TXN_TO_DS, txnContainer)) { LOG_GENERAL(WARNING, "Failed to get txns for DS"); continue; } - - copy(txns.begin(), txns.end(), back_inserter(DSTxn)); } - return !(shardTxn.empty() || DSTxn.empty()); + return !(txnContainer.empty()); } bool Lookup::GenTxnToSend(size_t num_txn, - map>>& mp, - uint32_t numShards, + std::vector& txnContainer, const bool updateRemoteStorageDBForGenTxns) { - vector txns; - if (GENESIS_WALLETS.size() == 0) { LOG_GENERAL(WARNING, "No genesis accounts found"); return false; @@ -574,44 +560,31 @@ bool Lookup::GenTxnToSend(size_t num_txn, nonce = AccountStore::GetInstance().GetAccount(addr)->GetNonce(); } - txns.clear(); if (!GetTxnFromFile::GetFromFile(addr, static_cast(nonce) + 1, - num_txn, txns)) { + num_txn, txnContainer)) { LOG_GENERAL(WARNING, "Failed to get txns from file"); continue; } - hasTransactions = hasTransactions || (txns.size() > 0); + hasTransactions = hasTransactions || (txnContainer.size() > 0); if (j == 1) { // shard txn dispatching - auto txnShard = Transaction::GetShardIndex(addr, numShards); if (REMOTESTORAGE_DB_ENABLE && updateRemoteStorageDBForGenTxns) { - for (const auto& txn : txns) { - mp[txnShard].emplace_back(make_pair(txn, 0)); + for (const auto& txn : txnContainer) { RemoteStorageDB::GetInstance().InsertTxn( txn, TxnStatus::DISPATCHED, m_mediator.m_currentEpochNum); } - } else { - for (const auto& txn : txns) { - mp[txnShard].emplace_back(make_pair(txn, 0)); - } } - LOG_GENERAL(INFO, "[Batching] Last Nonce sent to shard-" - << txnShard << " " << nonce + num_txn - << " of Addr " << addr.hex()); + LOG_GENERAL(INFO, "[Batching] Last Nonce sent nonce: " + << nonce + num_txn << " of Addr " << addr.hex()); m_gentxnAddrLatestNonceSent[addr] = nonce + num_txn; } else { // ds txn dispatching if (REMOTESTORAGE_DB_ENABLE && updateRemoteStorageDBForGenTxns) { - for (const auto& txn : txns) { - mp[numShards].emplace_back(make_pair(txn, 0)); + for (const auto& txn : txnContainer) { RemoteStorageDB::GetInstance().InsertTxn( txn, TxnStatus::DISPATCHED, m_mediator.m_currentEpochNum); } - } else { - for (const auto& txn : txns) { - mp[numShards].emplace_back(make_pair(txn, 0)); - } } LOG_GENERAL(INFO, "[Batching] Last Nonce sent to DS " @@ -679,7 +652,8 @@ void Lookup::SendMessageToLookupNodes(const zbytes& message) const { for (const auto& node : m_lookupNodes) { auto resolved_ip = TryGettingResolvedIP(node.second); - Blacklist::GetInstance().Whitelist({resolved_ip,node.second.GetListenPortHost(),""}); + Blacklist::GetInstance().Whitelist( + {resolved_ip, node.second.GetListenPortHost(), ""}); Peer tmp(resolved_ip, node.second.GetListenPortHost()); LOG_GENERAL(INFO, "Sending to lookup " << tmp); @@ -706,7 +680,8 @@ void Lookup::SendMessageToLookupNodesSerial(const zbytes& message) const { auto resolved_ip = TryGettingResolvedIP(node.second); - Blacklist::GetInstance().Whitelist({resolved_ip,node.second.GetListenPortHost(),""}); + Blacklist::GetInstance().Whitelist( + {resolved_ip, node.second.GetListenPortHost(), ""}); Peer tmp(resolved_ip, node.second.GetListenPortHost()); LOG_GENERAL(INFO, "Sending to lookup " << tmp); @@ -746,7 +721,8 @@ void Lookup::SendMessageToRandomLookupNode(const zbytes& message) const { int index = RandomGenerator::GetRandomInt(tmp.size()); auto resolved_ip = TryGettingResolvedIP(tmp[index].second); - Blacklist::GetInstance().Whitelist({resolved_ip,tmp[index].second.GetListenPortHost(),""}); + Blacklist::GetInstance().Whitelist( + {resolved_ip, tmp[index].second.GetListenPortHost(), ""}); Peer tmpPeer(resolved_ip, tmp[index].second.GetListenPortHost()); LOG_GENERAL(INFO, "Sending to Random lookup: " << tmpPeer); zil::p2p::GetInstance().SendMessage(tmpPeer, message); @@ -760,7 +736,8 @@ void Lookup::SendMessageToSeedNodes(const zbytes& message) const { for (const auto& node : m_seedNodes) { auto resolved_ip = TryGettingResolvedIP(node.second); - Blacklist::GetInstance().Whitelist({resolved_ip,node.second.GetListenPortHost(),""}); + Blacklist::GetInstance().Whitelist( + {resolved_ip, node.second.GetListenPortHost(), ""}); Peer tmpPeer(resolved_ip, node.second.GetListenPortHost()); LOG_EPOCH(INFO, m_mediator.m_currentEpochNum, "Sending msg to seed node " << tmpPeer); @@ -1226,8 +1203,9 @@ bool Lookup::ProcessGetDSInfoFromSeed(const zbytes& message, uint32_t portNo = 0; bool initialDS; - if (!ARCHIVAL_LOOKUP && - !Blacklist::GetInstance().IsWhitelistedSeed({from.m_ipAddress,from.GetListenPortHost(),from.GetNodeIndentifier()} )) { + if (!ARCHIVAL_LOOKUP && !Blacklist::GetInstance().IsWhitelistedSeed( + {from.m_ipAddress, from.GetListenPortHost(), + from.GetNodeIndentifier()})) { LOG_GENERAL( WARNING, "Requesting IP : " @@ -1291,13 +1269,14 @@ void Lookup::SendMessageToRandomL2lDataProvider(const zbytes& message) const { int index = RandomGenerator::GetRandomInt(m_l2lDataProviders.size()); auto resolved_ip = TryGettingResolvedIP(m_l2lDataProviders[index].second); - Blacklist::GetInstance().Whitelist({resolved_ip,m_l2lDataProviders[index].second.GetListenPortHost(),""}); + Blacklist::GetInstance().Whitelist( + {resolved_ip, m_l2lDataProviders[index].second.GetListenPortHost(), ""}); Peer tmpPeer(resolved_ip, m_l2lDataProviders[index].second.GetListenPortHost()); LOG_GENERAL(INFO, "Sending message to l2l: " << tmpPeer); unsigned char startByte = zil::p2p::START_BYTE_NORMAL; // TODO Disabled in updated protocol - //if (ENABLE_SEED_TO_SEED_COMMUNICATION) { + // if (ENABLE_SEED_TO_SEED_COMMUNICATION) { // startByte = zil::p2p::START_BYTE_SEED_TO_SEED_REQUEST; //} zil::p2p::GetInstance().SendMessage(tmpPeer, message, startByte); @@ -1314,10 +1293,13 @@ void Lookup::SendMessageToRandomSeedNode(const zbytes& message) const { for (const auto& node : m_seedNodes) { auto seedNodeIpToSend = TryGettingResolvedIP(node.second); - if (!Blacklist::GetInstance().Exist({seedNodeIpToSend,node.second.GetListenPortHost(),node.second.GetNodeIndentifier()}) && + if (!Blacklist::GetInstance().Exist({seedNodeIpToSend, + node.second.GetListenPortHost(), + node.second.GetNodeIndentifier()}) && (m_mediator.m_selfPeer.GetIpAddress() != seedNodeIpToSend)) { notBlackListedSeedNodes.push_back( - Peer(seedNodeIpToSend, node.second.GetListenPortHost(),node.second.GetNodeIndentifier())); + Peer(seedNodeIpToSend, node.second.GetListenPortHost(), + node.second.GetNodeIndentifier())); } } } @@ -1806,8 +1788,9 @@ bool Lookup::ProcessGetDSBlockFromSeed(const zbytes& message, uint32_t portNo = 0; bool includeMinerInfo = false; - if (!ARCHIVAL_LOOKUP && - !Blacklist::GetInstance().IsWhitelistedSeed({from.m_ipAddress,from.m_listenPortHost,from.GetNodeIndentifier()} )) { + if (!ARCHIVAL_LOOKUP && !Blacklist::GetInstance().IsWhitelistedSeed( + {from.m_ipAddress, from.m_listenPortHost, + from.GetNodeIndentifier()})) { LOG_GENERAL( WARNING, "Requesting IP : " @@ -1979,8 +1962,9 @@ bool Lookup::ProcessGetTxBlockFromSeed(const zbytes& message, uint64_t highBlockNum = 0; uint32_t portNo = 0; - if (!ARCHIVAL_LOOKUP && - !Blacklist::GetInstance().IsWhitelistedSeed({from.m_ipAddress,from.m_listenPortHost,from.GetNodeIndentifier()} )) { + if (!ARCHIVAL_LOOKUP && !Blacklist::GetInstance().IsWhitelistedSeed( + {from.m_ipAddress, from.m_listenPortHost, + from.GetNodeIndentifier()})) { LOG_GENERAL( WARNING, "Requesting IP : " @@ -2099,8 +2083,9 @@ bool Lookup::ProcessGetStateDeltaFromSeed(const zbytes& message, uint64_t blockNum = 0; uint32_t portNo = 0; - if (!ARCHIVAL_LOOKUP && - !Blacklist::GetInstance().IsWhitelistedSeed({from.m_ipAddress,from.m_listenPortHost,from.GetNodeIndentifier()})) { + if (!ARCHIVAL_LOOKUP && !Blacklist::GetInstance().IsWhitelistedSeed( + {from.m_ipAddress, from.m_listenPortHost, + from.GetNodeIndentifier()})) { LOG_GENERAL( WARNING, "Requesting IP : " @@ -2163,8 +2148,9 @@ bool Lookup::ProcessGetStateDeltasFromSeed(const zbytes& message, uint64_t highBlockNum = 0; uint32_t portNo = 0; - if (!ARCHIVAL_LOOKUP && - !Blacklist::GetInstance().IsWhitelistedSeed({from.m_ipAddress,from.m_listenPortHost,from.GetNodeIndentifier()})) { + if (!ARCHIVAL_LOOKUP && !Blacklist::GetInstance().IsWhitelistedSeed( + {from.m_ipAddress, from.m_listenPortHost, + from.GetNodeIndentifier()})) { LOG_GENERAL( WARNING, "Requesting IP : " @@ -2350,7 +2336,9 @@ bool Lookup::ProcessGetMicroBlockFromLookup(const zbytes& message, // verify if sender is from whitelisted list uint128_t ipAddr = from.m_ipAddress; - if (!Blacklist::GetInstance().IsWhitelistedSeed({from.m_ipAddress,from.m_listenPortHost,from.GetNodeIndentifier()})) { + if (!Blacklist::GetInstance().IsWhitelistedSeed( + {from.m_ipAddress, from.m_listenPortHost, + from.GetNodeIndentifier()})) { LOG_GENERAL( WARNING, "Requesting IP : " @@ -3545,10 +3533,6 @@ void Lookup::FindMissingMBsForLastNTxBlks(const uint32_t& num) { } } -deque>& Lookup::GetTxnFromShardMap(uint32_t index) { - return m_txnShardMap[index]; -} - bool Lookup::ProcessSetStateDeltaFromSeed( const zbytes& message, unsigned int offset, const Peer& from, [[gnu::unused]] const unsigned char& startByte) { @@ -3715,7 +3699,8 @@ bool Lookup::ProcessGetTxnsFromLookup([[gnu::unused]] const zbytes& message, // verify if sender is from whitelisted list uint128_t ipAddr = from.m_ipAddress; - if (!Blacklist::GetInstance().IsWhitelistedSeed({ipAddr,from.GetListenPortHost(),from.GetNodeIndentifier()})) { + if (!Blacklist::GetInstance().IsWhitelistedSeed( + {ipAddr, from.GetListenPortHost(), from.GetNodeIndentifier()})) { LOG_GENERAL( WARNING, "Requesting IP : " @@ -3746,7 +3731,8 @@ bool Lookup::ProcessGetTxnsFromLookup([[gnu::unused]] const zbytes& message, << max(DS_MICROBLOCK_GAS_LIMIT, SHARD_MICROBLOCK_GAS_LIMIT) << " missing txns. Looks suspicious so will " "ignore the message and blacklist sender"); - Blacklist::GetInstance().Add({from.GetIpAddress(),from.GetListenPortHost(),from.GetNodeIndentifier()}); + Blacklist::GetInstance().Add({from.GetIpAddress(), from.GetListenPortHost(), + from.GetNodeIndentifier()}); return false; } @@ -3833,7 +3819,8 @@ bool Lookup::ProcessGetTxnsFromL2l(const zbytes& message, unsigned int offset, << max(DS_MICROBLOCK_GAS_LIMIT, SHARD_MICROBLOCK_GAS_LIMIT) << " missing txns. Looks suspicious so will " "ignore the message and blacklist sender"); - Blacklist::GetInstance().Add({from.GetIpAddress(),from.m_listenPortHost,""}); + Blacklist::GetInstance().Add( + {from.GetIpAddress(), from.m_listenPortHost, ""}); return false; } @@ -4093,6 +4080,7 @@ bool Lookup::InitMining() { LOG_EPOCH(INFO, m_mediator.m_currentEpochNum, "Starting PoW for new ds block number " << curDsBlockNum + 1); + LOG_GENERAL(WARNING, "BZ Init mining, will start PoW soon"); m_mediator.m_node->StartPoW( curDsBlockNum + 1, m_mediator.m_dsBlockChain.GetLastBlock().GetHeader().GetDSDifficulty(), @@ -4790,7 +4778,6 @@ bool Lookup::CleanVariables() { m_seedNodes.clear(); m_currDSExpired = false; - m_startedTxnBatchThread = false; m_isFirstLoop = true; { std::lock_guard lock(m_mediator.m_ds->m_mutexShards); @@ -5233,7 +5220,8 @@ bool Lookup::Execute(const zbytes& message, unsigned int offset, return false; } } - + LOG_GENERAL(WARNING, "BZ Dispatching Lookup msg type: " + << hex << (unsigned int)ins_byte); if (ins_byte < ins_handlers_count) { result = (this->*ins_handlers[ins_byte])(message, offset + 1, from, startByte); @@ -5258,39 +5246,34 @@ void Lookup::RemoveSeedNodesFromBlackList() { for (auto& node : m_seedNodes) { auto seedNodeIp = TryGettingResolvedIP(node.second); - Blacklist::GetInstance().Remove({seedNodeIp,node.second.GetListenPortHost(),node.second.GetNodeIndentifier()}); + Blacklist::GetInstance().Remove({seedNodeIp, + node.second.GetListenPortHost(), + node.second.GetNodeIndentifier()}); } } -bool Lookup::AddToTxnShardMap(const Transaction& tx, uint32_t shardId, - TxnShardMap& txnShardMap, - mutex& txnShardMapMutex) { - lock_guard g(txnShardMapMutex); - uint32_t size = 0; - - for (const auto& x : txnShardMap) { - size += x.second.size(); - } +bool Lookup::AddTxnToMemPool(const Transaction& tx, TxnMemPool& txnMemPool, + mutex& txnMemPoolMutex) { + lock_guard g(txnMemPoolMutex); - if (size >= TXN_STORAGE_LIMIT) { + if (std::size(txnMemPool) >= TXN_STORAGE_LIMIT) { LOG_GENERAL(INFO, "Number of txns exceeded limit"); return false; } // case where txn already exist - if (find_if(txnShardMap[shardId].begin(), txnShardMap[shardId].end(), - [tx](const pair& txn_and_count) { - return tx.GetTranID() == txn_and_count.first.GetTranID(); - }) != txnShardMap[shardId].end()) { + if (find_if(m_txnMemPool.cbegin(), m_txnMemPool.cend(), + [tx](const Transaction& txn) { + return tx.GetTranID() == txn.GetTranID(); + }) != m_txnMemPool.end()) { LOG_GENERAL(WARNING, "Same hash present " << tx.GetTranID()); return false; } - txnShardMap[shardId].emplace_back(make_pair(tx, 0)); - LOG_GENERAL(INFO, "Added Txn " << tx.GetTranID().hex() << " to shard " - << shardId << " of fromAddr " - << tx.GetSenderAddr() << ", nonce: " - << tx.GetNonce()); + txnMemPool.push_back(tx); + LOG_GENERAL(INFO, "Added Txn " << tx.GetTranID().hex() << " fromAddr " + << tx.GetSenderAddr() + << ", nonce: " << tx.GetNonce()); if (REMOTESTORAGE_DB_ENABLE && !ARCHIVAL_LOOKUP) { RemoteStorageDB::GetInstance().InsertTxn(tx, TxnStatus::DISPATCHED, m_mediator.m_currentEpochNum); @@ -5299,32 +5282,31 @@ bool Lookup::AddToTxnShardMap(const Transaction& tx, uint32_t shardId, return true; } -bool Lookup::AddToTxnShardMap(const Transaction& tx, uint32_t shardId) { +bool Lookup::AddTxnToMemPool(const Transaction& tx) { if (!LOOKUP_NODE_MODE) { LOG_GENERAL(WARNING, - "Lookup::AddToTxnShardMap not expected to be called from " + "Lookup::AddTxnToMemPool not expected to be called from " "other than the LookUp node."); return true; } - return AddToTxnShardMap(tx, shardId, m_txnShardMap, m_txnShardMapMutex); + return AddTxnToMemPool(tx, m_txnMemPool, m_txnMemPoolMutex); } -bool Lookup::DeleteTxnShardMap(uint32_t shardId) { +bool Lookup::ClearTxnMemPool() { if (!LOOKUP_NODE_MODE) { LOG_GENERAL(WARNING, - "Lookup::DeleteTxnShardMap not expected to be called from " + "Lookup::ClearTxnMemPool not expected to be called from " "other than the LookUp node."); return true; } - - m_txnShardMap[shardId].clear(); + lock_guard g(m_txnMemPoolMutex); + m_txnMemPool.clear(); return true; } -void Lookup::SenderTxnBatchThread(const uint32_t oldNumShards, - bool newDSEpoch) { +void Lookup::SenderTxnBatchThread(std::vector transactions) { if (!LOOKUP_NODE_MODE) { LOG_GENERAL(WARNING, "Lookup::SenderTxnBatchThread not expected to be called from " @@ -5332,82 +5314,11 @@ void Lookup::SenderTxnBatchThread(const uint32_t oldNumShards, return; } - if (m_startedTxnBatchThread) { - LOG_GENERAL(WARNING, - "The last TxnBatchThread hasn't finished, discard this time"); - return; - } - - auto main_func = [this, oldNumShards, newDSEpoch]() mutable -> void { - m_startedTxnBatchThread = true; - uint32_t numShards = 0; - while (true) { - if (!m_mediator.GetIsVacuousEpoch()) { - numShards = m_mediator.m_ds->GetNumShards(); - if (newDSEpoch) { - m_gentxnAddrLatestNonceSent.clear(); - SendTxnPacketToNodes(oldNumShards, numShards); - } else { - SendTxnPacketToDS(oldNumShards, numShards); - } - } - break; - } - m_startedTxnBatchThread = false; - }; - DetachedFunction(1, main_func); + SendTxnsToDSShard(std::move(transactions)); } -void Lookup::RectifyTxnShardMap(const uint32_t oldNumShards, - const uint32_t newNumShards) { - auto t_start = std::chrono::high_resolution_clock::now(); - - map>> tempTxnShardMap; - - lock_guard g(m_txnShardMapMutex); - - LOG_GENERAL(INFO, "Shard dropped or gained, shuffling txn shard map"); - LOG_GENERAL(INFO, "New Shard Size: " << newNumShards - << " Old Shard Size: " << oldNumShards); - for (const auto& shard : m_txnShardMap) { - if (shard.first == oldNumShards) { - // ds txns - continue; - } - for (const auto& tx_and_count : shard.second) { - unsigned int fromShard = tx_and_count.first.GetShardIndex(newNumShards); - - if (Transaction::GetTransactionType(tx_and_count.first) == - Transaction::CONTRACT_CALL) { - // if shard do not match directly send to ds - unsigned int toShard = Transaction::GetShardIndex( - tx_and_count.first.GetToAddr(), newNumShards); - if (toShard != fromShard) { - // later would be placed in the new ds shard - m_txnShardMap[oldNumShards].emplace_back(tx_and_count); - continue; - } - } - - tempTxnShardMap[fromShard].emplace_back(tx_and_count); - } - } - tempTxnShardMap[newNumShards] = std::move(m_txnShardMap[oldNumShards]); - - m_txnShardMap.clear(); - - m_txnShardMap = std::move(tempTxnShardMap); - - auto t_end = std::chrono::high_resolution_clock::now(); - - double elaspedTimeMs = - std::chrono::duration(t_end - t_start).count(); - - LOG_GENERAL(INFO, "Elapsed time for exchange " << elaspedTimeMs); -} - -void Lookup::SendTxnPacketToShard(const uint32_t shardId, bool toDS, - bool afterSoftConfirmation) { +void Lookup::SendTxnPacketToShard(std::vector transactions) { + LOG_GENERAL(WARNING, "BZ SendTxnPacketToShard"); if (!LOOKUP_NODE_MODE) { LOG_GENERAL(WARNING, "Lookup::SendTxnPacketToShard not expected to be called from" @@ -5417,77 +5328,37 @@ void Lookup::SendTxnPacketToShard(const uint32_t shardId, bool toDS, zbytes msg = {MessageType::NODE, NodeInstructionType::FORWARDTXNPACKET}; bool result = false; - uint64_t epoch = afterSoftConfirmation ? m_mediator.m_currentEpochNum + 1 - : m_mediator.m_currentEpochNum; + uint64_t epoch = m_mediator.m_currentEpochNum; { - unique_lock g(m_txnShardMapMutex, defer_lock); - unique_lock g2(m_txnShardMapGeneratedMutex, defer_lock); + unique_lock g(m_txnMemPoolMutex, defer_lock); + unique_lock g2(m_txnMemPoolGeneratedMutex, defer_lock); lock(g, g2); - auto transactionNumber = m_txnShardMapGenerated[shardId].size(); + auto transactionNumber = transactions.size(); LOG_GENERAL(INFO, "Txn number generated: " << transactionNumber); - if (GetTxnFromShardMap(shardId).empty() && - m_txnShardMapGenerated[shardId].empty()) { - LOG_GENERAL(INFO, "No txns to send to shard " << shardId); + if (transactions.empty()) { + LOG_GENERAL(INFO, "No txns to send to ds shard"); return; } + LOG_GENERAL(WARNING, "BZ Creating message with size of txns: " + << transactions.size()); + result = Messenger::SetNodeForwardTxnBlock( msg, MessageOffset::BODY, epoch, m_mediator.m_dsBlockChain.GetLastBlock().GetHeader().GetBlockNum(), - shardId, m_mediator.m_selfKey, GetTxnFromShardMap(shardId), - m_txnShardMapGenerated[shardId]); + m_mediator.m_selfKey, transactions); } if (!result) { LOG_EPOCH(WARNING, epoch, "Messenger::SetNodeForwardTxnBlock failed."); - LOG_GENERAL(WARNING, "Cannot create packet for " << shardId << " shard"); return; } vector toSend; - if (!toDS) { - { - lock_guard g(m_mediator.m_ds->m_mutexShards); - uint16_t lastBlockHash = DataConversion::charArrTo16Bits( - m_mediator.m_txBlockChain.GetLastBlock().GetBlockHash().asBytes()); - - if (m_mediator.m_ds->m_shards.at(shardId).empty()) { - return; - } - - // Lookup sends to NUM_NODES_TO_SEND_LOOKUP including Leader - - PairOfNode shardLeader; - uint32_t leader_id = m_mediator.m_node->CalculateShardLeaderFromShard( - lastBlockHash, m_mediator.m_ds->m_shards.at(shardId).size(), - m_mediator.m_ds->m_shards.at(shardId), shardLeader); - LOG_EPOCH(INFO, m_mediator.m_currentEpochNum, - "Shard leader id " << leader_id); - - auto it = m_mediator.m_ds->m_shards.at(shardId).begin(); - - // Send to the leader - toSend.push_back(shardLeader.second); - - // Send to remaining ones - for (; it != m_mediator.m_ds->m_shards.at(shardId).end(); it++) { - if (toSend.size() < NUM_NODES_TO_SEND_LOOKUP && - std::get(*it) != shardLeader.first) { - toSend.push_back(std::get(*it)); - } - - if (toSend.size() >= NUM_NODES_TO_SEND_LOOKUP) { - break; - } - } - } - zil::p2p::GetInstance().SendBroadcastMessage(toSend, msg); - - DeleteTxnShardMap(shardId); - } else { + { // To send DS { lock_guard g(m_mediator.m_mutexDSCommittee); @@ -5515,28 +5386,24 @@ void Lookup::SendTxnPacketToShard(const uint32_t shardId, bool toDS, } } } - + LOG_GENERAL(WARNING, "BZ Broadcasting messages to shard from lookup..."); zil::p2p::GetInstance().SendBroadcastMessage(toSend, msg); LOG_GENERAL(INFO, "[DSMB]" << " Sent DS the txns"); - - DeleteTxnShardMap(shardId); } } -void Lookup::SendTxnPacketToDS(const uint32_t oldNumShards, - const uint32_t newNumShards) { +void Lookup::SendTxnsToDSShard(std::vector transactionsToSend) { // This will generate txns for all shards include ds shard. - SendTxnPacketPrepare(oldNumShards, newNumShards); + SendTxnPacketPrepare(transactionsToSend); - // But will only send txn pkt to ds-shard - SendTxnPacketToShard(newNumShards, true); + SendTxnPacketToShard(std::move(transactionsToSend)); } -void Lookup::SendTxnPacketPrepare(const uint32_t oldNumShards, - const uint32_t newNumShards, - const bool updateRemoteStorageDBForGenTxns) { +void Lookup::SendTxnPacketPrepare( + std::vector& transactionsToSend) { + LOG_GENERAL(WARNING, "BZ SendTxnPacketPrepare"); if (!LOOKUP_NODE_MODE) { LOG_GENERAL(WARNING, "Lookup::SendTxnPacketPrepare not expected to be called from " @@ -5549,56 +5416,17 @@ void Lookup::SendTxnPacketPrepare(const uint32_t oldNumShards, return; } - const uint32_t numShards = newNumShards; - if (USE_REMOTE_TXN_CREATOR) { - lock_guard g(m_txnShardMapGeneratedMutex); - m_txnShardMapGenerated.clear(); + lock_guard g(m_txnMemPoolGeneratedMutex); + m_txnMemPoolGenerated.clear(); - if (!GenTxnToSend(NUM_TXN_TO_SEND_PER_ACCOUNT, m_txnShardMapGenerated, - numShards, updateRemoteStorageDBForGenTxns)) { + if (!GenTxnToSend(NUM_TXN_TO_SEND_PER_ACCOUNT, m_txnMemPoolGenerated, + true)) { LOG_GENERAL(WARNING, "GenTxnToSend failed"); } - } - - if (oldNumShards != newNumShards) { - auto rectifyFunc = [this, oldNumShards, newNumShards]() mutable -> void { - RectifyTxnShardMap(oldNumShards, newNumShards); - }; - DetachedFunction(1, rectifyFunc); - } - - this_thread::sleep_for( - chrono::milliseconds(LOOKUP_DELAY_SEND_TXNPACKET_IN_MS)); -} - -void Lookup::SendTxnPacketToNodes(const uint32_t oldNumShards, - const uint32_t newNumShards) { - if (!LOOKUP_NODE_MODE) { - LOG_GENERAL(WARNING, - "Lookup::SendTxnPacketToNodes not expected to be called from " - "other than the LookUp node."); - return; - } - - SendTxnPacketPrepare(oldNumShards, newNumShards); - - for (unsigned int i = 0; i < newNumShards + 1; i++) { - SendTxnPacketToShard(i, i == newNumShards); - } - - // Fix: To prepare for new set of transaction for second epoch - // since txns from first epoch will be hard confirmed only on receving FB. - // And we need to avoid sending same txns in second epoch after receivng MB - // for first epoch - SendTxnPacketPrepare(oldNumShards, newNumShards, false); - // Now we have generated txns for all shards and ds-shard. - // But we only need them for normal shards to be send later after recv soft - // confirmation. - { - lock_guard g(m_txnShardMapGeneratedMutex); - // remove for ds-shard - m_txnShardMapGenerated[newNumShards].clear(); + transactionsToSend.insert(std::end(transactionsToSend), + std::cbegin(m_txnMemPoolGenerated), + std::cend(m_txnMemPoolGenerated)); } } @@ -5673,6 +5501,7 @@ bool Lookup::VerifySenderNode(const Shard& shard, bool Lookup::ProcessForwardTxn(const zbytes& message, unsigned int offset, const Peer& from, [[gnu::unused]] const unsigned char& startByte) { + LOG_GENERAL(WARNING, "BZ Received ProcessForwardTxn"); if (!LOOKUP_NODE_MODE) { LOG_GENERAL(WARNING, "Lookup::ProcessForwardTxn not expected to be called from " @@ -5684,58 +5513,26 @@ bool Lookup::ProcessForwardTxn(const zbytes& message, unsigned int offset, return false; } - vector txnsShard; - vector txnsDS; - - if (!Messenger::GetForwardTxnBlockFromSeed(message, offset, txnsShard, - txnsDS)) { - LOG_GENERAL(WARNING, "Failed to Messenger::GetForwardTxnBlockFromSeed"); - return false; - } - - LOG_GENERAL(INFO, "Recvd from " << from); - - if (!ARCHIVAL_LOOKUP) { - uint32_t shard_size = m_mediator.m_ds->GetNumShards(); - - if (!(m_sendSCCallsToDS || m_sendAllToDS)) { - for (const auto& txn : txnsShard) { - unsigned int shard = txn.GetShardIndex(shard_size); - AddToTxnShardMap(txn, shard); - } - } else if (m_sendAllToDS) { - for (const auto& txn : txnsShard) { - AddToTxnShardMap(txn, shard_size); - } - } else { - LOG_GENERAL(INFO, "Sending all contract calls to DS committee"); - for (const auto& txn : txnsShard) { - const Transaction::ContractType txnType = - Transaction::GetTransactionType(txn); - if (txnType == Transaction::ContractType::CONTRACT_CALL) { - AddToTxnShardMap(txn, shard_size); - } else { - unsigned int shard = txn.GetShardIndex(shard_size); - AddToTxnShardMap(txn, shard); - } - } - } - - LOG_GENERAL(INFO, "Size of DS txns " << txnsDS.size()); - - for (const auto& txn : txnsDS) { - AddToTxnShardMap(txn, shard_size); - } - if (REMOTESTORAGE_DB_ENABLE) { - RemoteStorageDB::GetInstance().ExecuteWriteDetached(); - } + // TODO: this should be reworked once we drop support for Lookup nodes + // I'm either upper seed (archival lookup) for external node or a Lookup for + // private seed nodes + if (ARCHIVAL_LOOKUP) { + // I'm seed/external-seed - forward message to next layer of 'lookups' + LOG_GENERAL(WARNING, + "BZ: Received txn message at lookup, sending to next lookup in " + "the chain"); + SendMessageToRandomSeedNode(message); } else { - for (const auto& txn : txnsShard) { - AddToTxnShardMap(txn, SEND_TYPE::ARCHIVAL_SEND_SHARD); - } - for (const auto& txn : txnsDS) { - AddToTxnShardMap(txn, SEND_TYPE::ARCHIVAL_SEND_DS); + // I'm a lookup (non-seed & non-external) - forward messages to ds shard + std::vector transactions; + if (!Messenger::GetForwardTxnBlockFromSeed(message, offset, transactions)) { + LOG_GENERAL(WARNING, + "Unable to deserialize message from by Lookup from Seed"); + return false; } + LOG_GENERAL(WARNING, "BZ Sending from lookup to dsshard txn with size: " + << transactions.size()); + SenderTxnBatchThread(std::move(transactions)); } return true; diff --git a/src/libLookup/Lookup.h b/src/libLookup/Lookup.h index 856b630da0..0c8cc17787 100644 --- a/src/libLookup/Lookup.h +++ b/src/libLookup/Lookup.h @@ -41,14 +41,7 @@ class Synchronizer; class LookupServer; class StakingServer; -// The "first" element in the pair is a map of shard to its transactions -// The "second" element in the pair counts the total number of transactions in -// the whole map -using TxnShardMap = - std::map>>; - -// Enum used to tell send type to seed node -enum SEND_TYPE { ARCHIVAL_SEND_SHARD = 0, ARCHIVAL_SEND_DS }; +using TxnMemPool = std::vector; /// Processes requests pertaining to network, transaction, or block information class Lookup : public Executable { @@ -90,8 +83,6 @@ class Lookup : public Executable { VectorOfPeer m_nodesInNetwork; std::unordered_set l_nodesInNetwork; - std::atomic m_startedTxnBatchThread{}; - std::atomic m_startedFetchMissingMBsThread{}; // Store the StateRootHash of latest txBlock before States are repopulated. @@ -117,8 +108,8 @@ class Lookup : public Executable { std::mutex m_mutexCheckDirBlocks; std::mutex m_mutexMicroBlocksBuffer; - TxnShardMap m_txnShardMap; - TxnShardMap m_txnShardMapGenerated; + TxnMemPool m_txnMemPool; + TxnMemPool m_txnMemPoolGenerated; std::map m_gentxnAddrLatestNonceSent; // Get StateDeltas from seed @@ -184,11 +175,10 @@ class Lookup : public Executable { // Getter for m_seedNodes VectorOfNode GetSeedNodes() const; - std::mutex m_txnShardMapMutex; - std::mutex m_txnShardMapGeneratedMutex; + auto GetTransactionsFromMemPool() const { return m_txnMemPool; } - std::deque>& GetTxnFromShardMap( - uint32_t index); // Use m_txnShardMapMutex with this function + std::mutex m_txnMemPoolMutex; + std::mutex m_txnMemPoolGeneratedMutex; std::mutex m_mutexShardStruct; std::condition_variable cv_shardStruct; @@ -201,12 +191,9 @@ class Lookup : public Executable { bool IsLookupNode(const Peer& peerInfo) const; // Gen n valid txns - bool GenTxnToSend( - size_t num_txn, - std::map>>& mp, - uint32_t numShards, const bool updateRemoteStorageDBForGenTxns); - bool GenTxnToSend(size_t num_txn, std::vector& shardTxn, - std::vector& DSTxn); + bool GenTxnToSend(size_t num_txn, std::vector& txnContainer, + const bool updateRemoteStorageDBForGenTxns); + bool GenTxnToSend(size_t num_txn, std::vector& txns); // Try resolving ip from the given peer's DNS uint128_t TryGettingResolvedIP(const Peer& peer) const; @@ -227,8 +214,6 @@ class Lookup : public Executable { void SendMessageToRandomL2lDataProvider(const zbytes& message) const; - void RectifyTxnShardMap(const uint32_t, const uint32_t); - // TODO: move the Get and ProcessSet functions to Synchronizer bool GetDSInfoFromSeedNodes(); bool GetDSInfoLoop(); @@ -295,29 +280,23 @@ class Lookup : public Executable { // protocol void RejoinAsNewLookup(bool fromLookup = true); - bool AddToTxnShardMap(const Transaction& tx, uint32_t shardId); - bool AddToTxnShardMap(const Transaction& tx, uint32_t shardId, - TxnShardMap& txnShardMap, std::mutex& txnShardMapMutex); + bool AddTxnToMemPool(const Transaction& tx); + bool AddTxnToMemPool(const Transaction& tx, TxnMemPool& txnMemPool, + std::mutex& txnMemPoolMutex); void CheckBufferTxBlocks(); - bool DeleteTxnShardMap(uint32_t shardId); + bool ClearTxnMemPool(); void SetServerTrue(); bool GetIsServer(); - void SenderTxnBatchThread(const uint32_t, bool newDSEpoch = false); - - void SendTxnPacketPrepare(const uint32_t oldNumShards, - const uint32_t newNumShards, - const bool updateRemoteStorageDBForGenTxns = true); - void SendTxnPacketToNodes(const uint32_t oldNumShards, - const uint32_t newNumShards); - void SendTxnPacketToDS(const uint32_t oldNumShards, - const uint32_t newNumShards); - void SendTxnPacketToShard(const uint32_t shardId, bool toDS, - bool afterSoftConfirmation = false); + void SenderTxnBatchThread(std::vector transactions); + + void SendTxnPacketPrepare(std::vector& transactionsToSend); + void SendTxnsToDSShard(std::vector transactions); + void SendTxnPacketToShard(std::vector transactions); bool ProcessEntireShardingStructure(); bool ProcessGetDSInfoFromSeed(const zbytes& message, unsigned int offset, diff --git a/src/libMessage/Messenger.cpp b/src/libMessage/Messenger.cpp index 7a25d0682e..d93c3537d8 100644 --- a/src/libMessage/Messenger.cpp +++ b/src/libMessage/Messenger.cpp @@ -1061,7 +1061,6 @@ bool SetConsensusAnnouncementCore( ZilliqaMessage::ConsensusAnnouncement& announcement, const uint32_t consensusID, uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const PairOfKey& leaderKey) { - // Set the consensus parameters announcement.mutable_consensusinfo()->set_consensusid(consensusID); @@ -1170,7 +1169,6 @@ bool GetConsensusAnnouncementCore( const ZilliqaMessage::ConsensusAnnouncement& announcement, const uint32_t consensusID, const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const PubKey& leaderKey) { - // Check the consensus parameters if (announcement.consensusinfo().consensusid() != consensusID) { @@ -1733,7 +1731,8 @@ bool Messenger::GetAccountStoreDelta(const zbytes& src, (unsigned int)address.size), address.asArray().begin()); - LOG_GENERAL(WARNING, "Messenger::GetAccountStoreDelta revertible address: " << address.hex()); + LOG_GENERAL(WARNING, "Messenger::GetAccountStoreDelta revertible address: " + << address.hex()); const Account* oriAccount = accountStore.GetAccount(address); bool fullCopy = false; if (oriAccount == nullptr) { @@ -1750,7 +1749,10 @@ bool Messenger::GetAccountStoreDelta(const zbytes& src, t_account = *oriAccount; account = *oriAccount; - LOG_GENERAL(WARNING, "Messenger::GetAccountStoreDelta ProtobufToAccountDelta revertible for addr: " << address.hex()); + LOG_GENERAL(WARNING, + "Messenger::GetAccountStoreDelta ProtobufToAccountDelta " + "revertible for addr: " + << address.hex()); if (!ProtobufToAccountDelta(entry.account(), account, address, fullCopy, temp, revertible)) { LOG_GENERAL(WARNING, @@ -1759,9 +1761,11 @@ bool Messenger::GetAccountStoreDelta(const zbytes& src, return false; } - LOG_GENERAL(WARNING, "Messenger::GetAccountStoreDelta ProtobufToAccountDelta revertible: " << address.hex() - << ", balance: " << account.GetBalance() - << ", nonce: " << account.GetNonce()); + LOG_GENERAL( + WARNING, + "Messenger::GetAccountStoreDelta ProtobufToAccountDelta revertible: " + << address.hex() << ", balance: " << account.GetBalance() + << ", nonce: " << account.GetNonce()); accountStore.AddAccountDuringDeserialization(address, account, t_account, fullCopy, revertible); @@ -1794,7 +1798,8 @@ bool Messenger::GetAccountStoreDelta(const zbytes& src, (unsigned int)address.size), address.asArray().begin()); - LOG_GENERAL(WARNING, "Messenger::GetAccountStoreDelta address: " << address.hex()); + LOG_GENERAL(WARNING, + "Messenger::GetAccountStoreDelta address: " << address.hex()); const Account* oriAccount = accountStoreTemp.GetAccount(address); bool fullCopy = false; @@ -1813,7 +1818,10 @@ bool Messenger::GetAccountStoreDelta(const zbytes& src, } account = *oriAccount; - LOG_GENERAL(WARNING, "Messenger::GetAccountStoreDelta ProtobufToAccountDelta for addr: " << address.hex()); + LOG_GENERAL( + WARNING, + "Messenger::GetAccountStoreDelta ProtobufToAccountDelta for addr: " + << address.hex()); if (!ProtobufToAccountDelta(entry.account(), account, address, fullCopy, temp)) { LOG_GENERAL(WARNING, @@ -1821,7 +1829,10 @@ bool Messenger::GetAccountStoreDelta(const zbytes& src, << address.hex()); return false; } - LOG_GENERAL(WARNING, "DESERIALIZE DELTA ACC: " << address.hex() << ", balance: " << account.GetBalance() << ", nonce: " << account.GetNonce()); + LOG_GENERAL(WARNING, "DESERIALIZE DELTA ACC: " + << address.hex() + << ", balance: " << account.GetBalance() + << ", nonce: " << account.GetNonce()); accountStoreTemp.AddAccountDuringDeserialization(address, account); } @@ -2355,7 +2366,6 @@ bool Messenger::GetDiagnosticDataCoinbase(const zbytes& src, bool Messenger::SetPMHello(zbytes& dst, const unsigned int offset, const PairOfKey& key, const uint32_t listenPort) { - PMHello result; SerializableToProtobufByteArray(key.second, @@ -2387,7 +2397,6 @@ bool Messenger::SetPMHello(zbytes& dst, const unsigned int offset, bool Messenger::GetPMHello(const zbytes& src, const unsigned int offset, PubKey& pubKey, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -2430,7 +2439,6 @@ bool Messenger::SetDSPoWSubmission( const string& resultingHash, const string& mixHash, const uint32_t& lookupId, const uint128_t& gasPrice, const GovProposalIdVotePair& govProposal, const string& version) { - DSPoWSubmission result; result.mutable_data()->set_blocknumber(blockNumber); @@ -2492,7 +2500,6 @@ bool Messenger::GetDSPoWSubmission( uint64_t& nonce, string& resultingHash, string& mixHash, Signature& signature, uint32_t& lookupId, uint128_t& gasPrice, uint32_t& govProposalId, uint32_t& govVoteValue, string& version) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -2543,7 +2550,6 @@ bool Messenger::GetDSPoWSubmission( bool Messenger::SetDSPoWPacketSubmission( zbytes& dst, const unsigned int offset, const vector& dsPowSolutions, const PairOfKey& keys) { - DSPoWPacketSubmission result; for (const auto& sol : dsPowSolutions) { @@ -2574,7 +2580,6 @@ bool Messenger::GetDSPowPacketSubmission(const zbytes& src, const unsigned int offset, vector& dsPowSolutions, PubKey& pubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -2612,7 +2617,6 @@ bool Messenger::SetDSMicroBlockSubmission( zbytes& dst, const unsigned int offset, const unsigned char microBlockType, const uint64_t epochNumber, const vector& microBlocks, const vector& stateDeltas, const PairOfKey& keys) { - DSMicroBlockSubmission result; result.mutable_data()->set_microblocktype(microBlockType); @@ -2655,7 +2659,6 @@ bool Messenger::GetDSMicroBlockSubmission( const zbytes& src, const unsigned int offset, unsigned char& microBlockType, uint64_t& epochNumber, vector& microBlocks, vector& stateDeltas, PubKey& pubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -2706,7 +2709,6 @@ bool Messenger::SetDSDSBlockAnnouncement( const uint16_t leaderID, const PairOfKey& leaderKey, const DSBlock& dsBlock, const DequeOfShard& shards, const MapOfPubKeyPoW& allPoWs, const MapOfPubKeyPoW& dsWinnerPoWs, zbytes& messageToCosign) { - ConsensusAnnouncement announcement; // Set the DSBlock announcement parameters @@ -2769,7 +2771,6 @@ bool Messenger::GetDSDSBlockAnnouncement( const uint16_t leaderID, const PubKey& leaderKey, DSBlock& dsBlock, DequeOfShard& shards, MapOfPubKeyPoW& allPoWs, MapOfPubKeyPoW& dsWinnerPoWs, zbytes& messageToCosign) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -2865,7 +2866,6 @@ bool Messenger::SetDSFinalBlockAnnouncement( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const PairOfKey& leaderKey, const TxBlock& txBlock, const shared_ptr& microBlock, zbytes& messageToCosign) { - ConsensusAnnouncement announcement; // Set the FinalBlock announcement parameters @@ -2910,7 +2910,6 @@ bool Messenger::GetDSFinalBlockAnnouncement( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const PubKey& leaderKey, TxBlock& txBlock, shared_ptr& microBlock, zbytes& messageToCosign) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -2969,7 +2968,6 @@ bool Messenger::SetDSVCBlockAnnouncement( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const PairOfKey& leaderKey, const VCBlock& vcBlock, zbytes& messageToCosign) { - ConsensusAnnouncement announcement; // Set the VCBlock announcement parameters @@ -3009,7 +3007,6 @@ bool Messenger::GetDSVCBlockAnnouncement( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const PubKey& leaderKey, VCBlock& vcBlock, zbytes& messageToCosign) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3058,7 +3055,6 @@ bool Messenger::SetDSMissingMicroBlocksErrorMsg( zbytes& dst, const unsigned int offset, const vector& missingMicroBlockHashes, const uint64_t epochNum, const uint32_t listenPort) { - DSMissingMicroBlocksErrorMsg result; for (const auto& hash : missingMicroBlockHashes) { @@ -3080,7 +3076,6 @@ bool Messenger::GetDSMissingMicroBlocksErrorMsg( const zbytes& src, const unsigned int offset, vector& missingMicroBlockHashes, uint64_t& epochNum, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3117,7 +3112,6 @@ bool Messenger::SetNodeVCDSBlocksMessage( zbytes& dst, const unsigned int offset, const uint32_t shardId, const DSBlock& dsBlock, const std::vector& vcBlocks, const uint32_t& shardingStructureVersion, const DequeOfShard& shards) { - NodeDSBlock result; result.set_shardid(shardId); @@ -3143,7 +3137,6 @@ bool Messenger::GetNodeVCDSBlocksMessage(const zbytes& src, std::vector& vcBlocks, uint32_t& shardingStructureVersion, DequeOfShard& shards) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3182,7 +3175,6 @@ bool Messenger::SetNodeVCFinalBlock(zbytes& dst, const unsigned int offset, const TxBlock& txBlock, const zbytes& stateDelta, const std::vector& vcBlocks) { - NodeVCFinalBlock result; result.set_dsblocknumber(dsBlockNumber); @@ -3208,7 +3200,6 @@ bool Messenger::GetNodeVCFinalBlock(const zbytes& src, uint32_t& consensusID, TxBlock& txBlock, zbytes& stateDelta, std::vector& vcBlocks) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3248,7 +3239,6 @@ bool Messenger::SetNodeFinalBlock(zbytes& dst, const unsigned int offset, const uint32_t consensusID, const TxBlock& txBlock, const zbytes& stateDelta) { - NodeFinalBlock result; result.set_dsblocknumber(dsBlockNumber); @@ -3268,7 +3258,6 @@ bool Messenger::GetNodeFinalBlock(const zbytes& src, const unsigned int offset, uint64_t& dsBlockNumber, uint32_t& consensusID, TxBlock& txBlock, zbytes& stateDelta) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3298,7 +3287,6 @@ bool Messenger::GetNodeFinalBlock(const zbytes& src, const unsigned int offset, bool Messenger::SetNodeMBnForwardTransaction( zbytes& dst, const unsigned int offset, const MicroBlock& microBlock, const vector& txns) { - NodeMBnForwardTransaction result; io::MicroBlockToProtobuf(microBlock, *result.mutable_microblock()); @@ -3326,7 +3314,6 @@ bool Messenger::SetNodePendingTxn( zbytes& dst, const unsigned offset, const uint64_t& epochnum, const unordered_map& hashCodeMap, const uint32_t shardId, const PairOfKey& key) { - NodePendingTxn result; SerializableToProtobufByteArray(key.second, @@ -3378,7 +3365,6 @@ bool Messenger::GetNodePendingTxn( const zbytes& src, const unsigned offset, uint64_t& epochnum, unordered_map& hashCodeMap, uint32_t& shardId, PubKey& pubKey, zbytes& txnListHash) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3430,7 +3416,6 @@ bool Messenger::GetNodePendingTxn( bool Messenger::GetNodeMBnForwardTransaction(const zbytes& src, const unsigned int offset, MBnForwardedTxnEntry& entry) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3463,7 +3448,6 @@ bool Messenger::GetNodeMBnForwardTransaction(const zbytes& src, bool Messenger::SetNodeVCBlock(zbytes& dst, const unsigned int offset, const VCBlock& vcBlock) { - NodeVCBlock result; io::VCBlockToProtobuf(vcBlock, *result.mutable_vcblock()); @@ -3478,7 +3462,6 @@ bool Messenger::SetNodeVCBlock(zbytes& dst, const unsigned int offset, bool Messenger::GetNodeVCBlock(const zbytes& src, const unsigned int offset, VCBlock& vcBlock) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3496,70 +3479,35 @@ bool Messenger::GetNodeVCBlock(const zbytes& src, const unsigned int offset, return io::ProtobufToVCBlock(result.vcblock(), vcBlock); } -bool Messenger::SetNodeForwardTxnBlock( - zbytes& dst, const unsigned int offset, const uint64_t& epochNumber, - const uint64_t& dsBlockNum, const uint32_t& shardId, - const PairOfKey& lookupKey, - deque>& txnsCurrent, - deque>& txnsGenerated) { - +bool Messenger::SetNodeForwardTxnBlock(zbytes& dst, const unsigned int offset, + const uint64_t& epochNumber, + const uint64_t& dsBlockNum, + const PairOfKey& lookupKey, + std::vector& transactions) { NodeForwardTxnBlock result; result.set_epochnumber(epochNumber); result.set_dsblocknum(dsBlockNum); - result.set_shardid(shardId); SerializableToProtobufByteArray(lookupKey.second, *result.mutable_pubkey()); - unsigned int txnsCurrentCount = 0, txnsGeneratedCount = 0, msg_size = 0; + unsigned int txnsCurrentCount = 0, msg_size = 0; - for (auto txn = txnsCurrent.begin(); txn != txnsCurrent.end();) { + for (auto txn = transactions.begin(); txn != transactions.end();) { if (msg_size >= PACKET_BYTESIZE_LIMIT) { break; } auto protoTxn = std::make_unique(); - TransactionToProtobuf(txn->first, *protoTxn); + TransactionToProtobuf(*txn, *protoTxn); unsigned txn_size = protoTxn->ByteSizeLong(); if ((msg_size + txn_size) > PACKET_BYTESIZE_LIMIT && txn_size >= SMALL_TXN_SIZE) { - if (++(txn->second) >= TXN_DISPATCH_ATTEMPT_LIMIT) { - LOG_GENERAL(WARNING, - "Failed to dispatch txn " << txn->first.GetTranID()); - txn = txnsCurrent.erase(txn); - } else { - txn++; - } continue; } *result.add_transactions() = *protoTxn; txnsCurrentCount++; msg_size += protoTxn->ByteSizeLong(); - txn = txnsCurrent.erase(txn); - } - - for (auto txn = txnsGenerated.begin(); txn != txnsGenerated.end();) { - if (msg_size >= PACKET_BYTESIZE_LIMIT) { - break; - } - - auto protoTxn = std::make_unique(); - TransactionToProtobuf(txn->first, *protoTxn); - unsigned txn_size = protoTxn->ByteSizeLong(); - if ((msg_size + txn_size) > PACKET_BYTESIZE_LIMIT && - txn_size >= SMALL_TXN_SIZE) { - if (++(txn->second) >= TXN_DISPATCH_ATTEMPT_LIMIT) { - LOG_GENERAL(WARNING, - "Failed to dispatch txn " << txn->first.GetTranID()); - txn = txnsGenerated.erase(txn); - } else { - txn++; - } - continue; - } - *result.add_transactions() = *protoTxn; - txnsGeneratedCount++; - msg_size += txn_size; - txn = txnsGenerated.erase(txn); + txn = transactions.erase(txn); } Signature signature; @@ -3582,9 +3530,8 @@ bool Messenger::SetNodeForwardTxnBlock( return false; } - LOG_GENERAL(INFO, "Epoch: " << epochNumber << " shardId: " << shardId - << " Current txns: " << txnsCurrentCount - << " Generated txns: " << txnsGeneratedCount); + LOG_GENERAL( + INFO, "Epoch: " << epochNumber << " Current txns: " << txnsCurrentCount); return SerializeToArray(result, dst, offset); } @@ -3596,7 +3543,6 @@ bool Messenger::SetNodeForwardTxnBlock(zbytes& dst, const unsigned int offset, const PubKey& lookupKey, std::vector& txns, const Signature& signature) { - NodeForwardTxnBlock result; result.set_epochnumber(epochNumber); @@ -3640,7 +3586,6 @@ bool Messenger::GetNodeForwardTxnBlock( const zbytes& src, const unsigned int offset, uint64_t& epochNumber, uint64_t& dsBlockNum, uint32_t& shardId, PubKey& lookupPubKey, std::vector& txns, Signature& signature) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3694,7 +3639,6 @@ bool Messenger::SetNodeMicroBlockAnnouncement( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const PairOfKey& leaderKey, const MicroBlock& microBlock, zbytes& messageToCosign) { - ConsensusAnnouncement announcement; // Set the MicroBlock announcement parameters @@ -3734,7 +3678,6 @@ bool Messenger::GetNodeMicroBlockAnnouncement( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const PubKey& leaderKey, MicroBlock& microBlock, zbytes& messageToCosign) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3817,7 +3760,6 @@ bool Messenger::SetNodeMissingTxnsErrorMsg( zbytes& dst, const unsigned int offset, const vector& missingTxnHashes, const uint64_t epochNum, const uint32_t listenPort) { - NodeMissingTxnsErrorMsg result; for (const auto& hash : missingTxnHashes) { @@ -3841,7 +3783,6 @@ bool Messenger::GetNodeMissingTxnsErrorMsg(const zbytes& src, vector& missingTxnHashes, uint64_t& epochNum, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3872,7 +3813,6 @@ bool Messenger::GetNodeMissingTxnsErrorMsg(const zbytes& src, bool Messenger::SetNodeGetVersion(zbytes& dst, const unsigned int offset, const uint32_t listenPort) { - NodeGetVersion result; result.set_listenport(listenPort); if (!result.IsInitialized()) { @@ -3885,7 +3825,6 @@ bool Messenger::SetNodeGetVersion(zbytes& dst, const unsigned int offset, bool Messenger::GetNodeGetVersion(const zbytes& src, const unsigned int offset, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3907,7 +3846,6 @@ bool Messenger::GetNodeGetVersion(const zbytes& src, const unsigned int offset, bool Messenger::SetNodeSetVersion(zbytes& dst, const unsigned int offset, const std::string& version) { - NodeSetVersion result; result.set_version(version); if (!result.IsInitialized()) { @@ -3920,7 +3858,6 @@ bool Messenger::SetNodeSetVersion(zbytes& dst, const unsigned int offset, bool Messenger::GetNodeSetVersion(const zbytes& src, const unsigned int offset, std::string& version) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3946,7 +3883,6 @@ bool Messenger::GetNodeSetVersion(const zbytes& src, const unsigned int offset, bool Messenger::SetLookupGetSeedPeers(zbytes& dst, const unsigned int offset, const uint32_t listenPort) { - LookupGetSeedPeers result; result.set_listenport(listenPort); @@ -3962,7 +3898,6 @@ bool Messenger::SetLookupGetSeedPeers(zbytes& dst, const unsigned int offset, bool Messenger::GetLookupGetSeedPeers(const zbytes& src, const unsigned int offset, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -3985,7 +3920,6 @@ bool Messenger::GetLookupGetSeedPeers(const zbytes& src, bool Messenger::SetLookupSetSeedPeers(zbytes& dst, const unsigned int offset, const PairOfKey& lookupKey, const vector& candidateSeeds) { - LookupSetSeedPeers result; unordered_set indicesAlreadyAdded; @@ -4034,7 +3968,6 @@ bool Messenger::GetLookupSetSeedPeers(const zbytes& src, const unsigned int offset, PubKey& lookupPubKey, vector& candidateSeeds) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4080,7 +4013,6 @@ bool Messenger::SetLookupGetDSInfoFromSeed(zbytes& dst, const unsigned int offset, const uint32_t listenPort, const bool initialDS) { - LookupGetDSInfoFromSeed result; result.set_listenport(listenPort); @@ -4098,7 +4030,6 @@ bool Messenger::GetLookupGetDSInfoFromSeed(const zbytes& src, const unsigned int offset, uint32_t& listenPort, bool& initialDS) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4125,7 +4056,6 @@ bool Messenger::SetLookupSetDSInfoFromSeed(zbytes& dst, const uint32_t& dsCommitteeVersion, const DequeOfNode& dsNodes, const bool initialDS) { - LookupSetDSInfoFromSeed result; DSCommitteeToProtobuf(dsCommitteeVersion, dsNodes, @@ -4160,7 +4090,6 @@ bool Messenger::SetLookupSetDSInfoFromSeed(zbytes& dst, bool Messenger::GetLookupSetDSInfoFromSeed( const zbytes& src, const unsigned int offset, PubKey& senderPubKey, uint32_t& dsCommitteeVersion, DequeOfNode& dsNodes, bool& initialDS) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4207,7 +4136,6 @@ bool Messenger::SetLookupGetDSBlockFromSeed(zbytes& dst, const uint64_t highBlockNum, const uint32_t listenPort, const bool includeMinerInfo) { - LookupGetDSBlockFromSeed result; result.set_lowblocknum(lowBlockNum); @@ -4226,7 +4154,6 @@ bool Messenger::SetLookupGetDSBlockFromSeed(zbytes& dst, bool Messenger::GetLookupGetDSBlockFromSeed( const zbytes& src, const unsigned int offset, uint64_t& lowBlockNum, uint64_t& highBlockNum, uint32_t& listenPort, bool& includeMinerInfo) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4255,7 +4182,6 @@ bool Messenger::SetLookupSetDSBlockFromSeed(zbytes& dst, const uint64_t highBlockNum, const PairOfKey& lookupKey, const vector& dsBlocks) { - LookupSetDSBlockFromSeed result; result.mutable_data()->set_lowblocknum(lowBlockNum); @@ -4293,7 +4219,6 @@ bool Messenger::SetLookupSetDSBlockFromSeed(zbytes& dst, bool Messenger::GetLookupSetDSBlockFromSeed( const zbytes& src, const unsigned int offset, uint64_t& lowBlockNum, uint64_t& highBlockNum, PubKey& lookupPubKey, vector& dsBlocks) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4339,7 +4264,6 @@ bool Messenger::SetLookupSetMinerInfoFromSeed( zbytes& dst, const unsigned int offset, const PairOfKey& lookupKey, const map>& minerInfoPerDS) { - LookupSetMinerInfoFromSeed result; for (const auto& dsBlockAndMinerInfo : minerInfoPerDS) { @@ -4413,7 +4337,6 @@ bool Messenger::SetLookupSetMinerInfoFromSeed( bool Messenger::GetLookupSetMinerInfoFromSeed( const zbytes& src, const unsigned int offset, PubKey& lookupPubKey, map>& minerInfoPerDS) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4484,7 +4407,6 @@ bool Messenger::SetLookupGetTxBlockFromSeed(zbytes& dst, const uint64_t lowBlockNum, const uint64_t highBlockNum, const uint32_t listenPort) { - LookupGetTxBlockFromSeed result; result.set_lowblocknum(lowBlockNum); @@ -4504,7 +4426,6 @@ bool Messenger::SetLookupGetVCFinalBlockFromL2l(zbytes& dst, const uint64_t& blockNum, const Peer& sender, const PairOfKey& seedKey) { - LookupGetVCFinalBlockFromL2l result; result.mutable_data()->set_blocknum(blockNum); @@ -4542,7 +4463,6 @@ bool Messenger::GetLookupGetVCFinalBlockFromL2l(const zbytes& src, const unsigned int offset, uint64_t& blockNum, Peer& from, PubKey& senderPubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4579,7 +4499,6 @@ bool Messenger::SetLookupGetDSBlockFromL2l(zbytes& dst, const uint64_t& blockNum, const Peer& sender, const PairOfKey& seedKey) { - LookupGetDSBlockFromL2l result; result.mutable_data()->set_blocknum(blockNum); @@ -4616,7 +4535,6 @@ bool Messenger::GetLookupGetDSBlockFromL2l(const zbytes& src, const unsigned int offset, uint64_t& blockNum, Peer& from, PubKey& senderPubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4650,7 +4568,6 @@ bool Messenger::GetLookupGetDSBlockFromL2l(const zbytes& src, bool Messenger::SetLookupGetMBnForwardTxnFromL2l( zbytes& dst, const unsigned int offset, const uint64_t& blockNum, const uint32_t& shardId, const Peer& sender, const PairOfKey& seedKey) { - LookupGetMBnForwardTxnFromL2l result; result.mutable_data()->set_blocknum(blockNum); @@ -4690,7 +4607,6 @@ bool Messenger::GetLookupGetMBnForwardTxnFromL2l(const zbytes& src, uint64_t& blockNum, uint32_t& shardId, Peer& from, PubKey& senderPubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4725,7 +4641,6 @@ bool Messenger::GetLookupGetMBnForwardTxnFromL2l(const zbytes& src, bool Messenger::SetLookupGetPendingTxnFromL2l( zbytes& dst, const unsigned int offset, const uint64_t& blockNum, const uint32_t& shardId, const Peer& sender, const PairOfKey& seedKey) { - LookupGetPendingTxnFromL2l result; result.mutable_data()->set_blocknum(blockNum); @@ -4765,7 +4680,6 @@ bool Messenger::GetLookupGetPendingTxnFromL2l(const zbytes& src, uint64_t& blockNum, uint32_t& shardId, Peer& from, PubKey& senderPubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4802,7 +4716,6 @@ bool Messenger::GetLookupGetTxBlockFromSeed(const zbytes& src, uint64_t& lowBlockNum, uint64_t& highBlockNum, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4830,7 +4743,6 @@ bool Messenger::SetLookupSetTxBlockFromSeed(zbytes& dst, const uint64_t highBlockNum, const PairOfKey& lookupKey, const vector& txBlocks) { - LookupSetTxBlockFromSeed result; result.mutable_data()->set_lowblocknum(lowBlockNum); @@ -4869,7 +4781,6 @@ bool Messenger::SetLookupSetTxBlockFromSeed(zbytes& dst, bool Messenger::GetLookupSetTxBlockFromSeed( const zbytes& src, const unsigned int offset, uint64_t& lowBlockNum, uint64_t& highBlockNum, PubKey& lookupPubKey, vector& txBlocks) { - LookupSetTxBlockFromSeed result; google::protobuf::io::ArrayInputStream arrayIn(src.data() + offset, @@ -4915,7 +4826,6 @@ bool Messenger::SetLookupGetStateDeltaFromSeed(zbytes& dst, const unsigned int offset, const uint64_t blockNum, const uint32_t listenPort) { - LookupGetStateDeltaFromSeed result; result.set_blocknum(blockNum); @@ -4934,7 +4844,6 @@ bool Messenger::SetLookupGetStateDeltasFromSeed(zbytes& dst, uint64_t& lowBlockNum, uint64_t& highBlockNum, const uint32_t listenPort) { - LookupGetStateDeltasFromSeed result; result.set_lowblocknum(lowBlockNum); @@ -4953,7 +4862,6 @@ bool Messenger::GetLookupGetStateDeltaFromSeed(const zbytes& src, const unsigned int offset, uint64_t& blockNum, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -4979,7 +4887,6 @@ bool Messenger::GetLookupGetStateDeltasFromSeed(const zbytes& src, uint64_t& lowBlockNum, uint64_t& highBlockNum, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5006,7 +4913,6 @@ bool Messenger::SetLookupSetStateDeltaFromSeed(zbytes& dst, const uint64_t blockNum, const PairOfKey& lookupKey, const zbytes& stateDelta) { - LookupSetStateDeltaFromSeed result; result.mutable_data()->set_blocknum(blockNum); @@ -5043,7 +4949,6 @@ bool Messenger::SetLookupSetStateDeltasFromSeed( zbytes& dst, const unsigned int offset, const uint64_t lowBlockNum, const uint64_t highBlockNum, const PairOfKey& lookupKey, const vector& stateDeltas) { - LookupSetStateDeltasFromSeed result; result.mutable_data()->set_lowblocknum(lowBlockNum); @@ -5084,7 +4989,6 @@ bool Messenger::GetLookupSetStateDeltaFromSeed(const zbytes& src, uint64_t& blockNum, PubKey& lookupPubKey, zbytes& stateDelta) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5123,7 +5027,6 @@ bool Messenger::GetLookupSetStateDeltaFromSeed(const zbytes& src, bool Messenger::GetLookupSetStateDeltasFromSeed( const zbytes& src, const unsigned int offset, uint64_t& lowBlockNum, uint64_t& highBlockNum, PubKey& lookupPubKey, vector& stateDeltas) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5168,7 +5071,6 @@ bool Messenger::SetLookupSetLookupOffline(zbytes& dst, const uint8_t msgType, const uint32_t listenPort, const PairOfKey& lookupKey) { - LookupSetLookupOffline result; result.mutable_data()->set_msgtype(msgType); @@ -5203,7 +5105,6 @@ bool Messenger::GetLookupSetLookupOffline(const zbytes& src, uint8_t& msgType, uint32_t& listenPort, PubKey& lookupPubkey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5240,7 +5141,6 @@ bool Messenger::SetLookupSetLookupOnline(zbytes& dst, const unsigned int offset, const uint8_t msgType, const uint32_t listenPort, const PairOfKey& lookupKey) { - LookupSetLookupOnline result; result.mutable_data()->set_msgtype(msgType); @@ -5273,7 +5173,6 @@ bool Messenger::GetLookupSetLookupOnline(const zbytes& src, const unsigned int offset, uint8_t& msgType, uint32_t& listenPort, PubKey& pubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5309,7 +5208,6 @@ bool Messenger::GetLookupSetLookupOnline(const zbytes& src, bool Messenger::SetLookupGetOfflineLookups(zbytes& dst, const unsigned int offset, const uint32_t listenPort) { - LookupGetOfflineLookups result; result.set_listenport(listenPort); @@ -5348,7 +5246,6 @@ bool Messenger::SetLookupSetOfflineLookups(zbytes& dst, const unsigned int offset, const PairOfKey& lookupKey, const vector& nodes) { - LookupSetOfflineLookups result; for (const auto& node : nodes) { @@ -5384,7 +5281,6 @@ bool Messenger::GetLookupSetOfflineLookups(const zbytes& src, const unsigned int offset, PubKey& lookupPubKey, vector& nodes) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5427,17 +5323,11 @@ bool Messenger::GetLookupSetOfflineLookups(const zbytes& src, bool Messenger::SetForwardTxnBlockFromSeed( zbytes& dst, const unsigned int offset, - const deque>& shardTransactions, - const deque>& dsTransactions) { + const std::vector& dsTransactions) { LookupForwardTxnsFromSeed result; - if (!shardTransactions.empty()) { - TransactionArrayToProtobuf(shardTransactions, - *result.mutable_shardtransactions()); - } if (!dsTransactions.empty()) { - TransactionArrayToProtobuf(dsTransactions, - *result.mutable_dstransactions()); + TransactionArrayToProtobuf(dsTransactions, *result.mutable_transactions()); } if (!result.IsInitialized()) { @@ -5447,10 +5337,9 @@ bool Messenger::SetForwardTxnBlockFromSeed( return SerializeToArray(result, dst, offset); } -bool Messenger::GetForwardTxnBlockFromSeed( - const zbytes& src, const unsigned int offset, - vector& shardTransactions, - vector& dsTransactions) { +bool Messenger::GetForwardTxnBlockFromSeed(const zbytes& src, + const unsigned int offset, + vector& txnsContainer) { if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5465,20 +5354,18 @@ bool Messenger::GetForwardTxnBlockFromSeed( return false; } - if (!ProtobufToTransactionArray(result.shardtransactions(), - shardTransactions)) { + if (!ProtobufToTransactionArray(result.transactions(), txnsContainer)) { LOG_GENERAL(WARNING, "ProtobufToTransactionArray failed"); return false; } - return ProtobufToTransactionArray(result.dstransactions(), dsTransactions); + return true; } // UNUSED bool Messenger::SetLookupGetShardsFromSeed(zbytes& dst, const unsigned int offset, const uint32_t listenPort) { - LookupGetShardsFromSeed result; result.set_listenport(listenPort); @@ -5495,7 +5382,6 @@ bool Messenger::SetLookupGetShardsFromSeed(zbytes& dst, bool Messenger::GetLookupGetShardsFromSeed(const zbytes& src, const unsigned int offset, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5519,7 +5405,6 @@ bool Messenger::GetLookupGetShardsFromSeed(const zbytes& src, bool Messenger::SetLookupSetShardsFromSeed( zbytes& dst, const unsigned int offset, const PairOfKey& lookupKey, const uint32_t& shardingStructureVersion, const DequeOfShard& shards) { - LookupSetShardsFromSeed result; ShardingStructureToProtobuf(shardingStructureVersion, shards, @@ -5553,7 +5438,6 @@ bool Messenger::GetLookupSetShardsFromSeed(const zbytes& src, PubKey& lookupPubKey, uint32_t& shardingStructureVersion, DequeOfShard& shards) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5595,7 +5479,6 @@ bool Messenger::GetLookupSetShardsFromSeed(const zbytes& src, bool Messenger::SetLookupGetMicroBlockFromLookup( zbytes& dst, const unsigned int offset, const vector& microBlockHashes, const uint32_t portNo) { - LookupGetMicroBlockFromLookup result; result.set_portno(portNo); @@ -5615,7 +5498,6 @@ bool Messenger::SetLookupGetMicroBlockFromL2l( zbytes& dst, const unsigned int offset, const vector& microBlockHashes, uint32_t portNo, const PairOfKey& seedKey) { - LookupGetMicroBlockFromL2l result; result.mutable_data()->set_portno(portNo); @@ -5654,7 +5536,6 @@ bool Messenger::GetLookupGetMicroBlockFromL2l( const zbytes& src, const unsigned int offset, vector& microBlockHashes, uint32_t& portNo, PubKey& senderPubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5696,7 +5577,6 @@ bool Messenger::GetLookupGetMicroBlockFromL2l( bool Messenger::GetLookupGetMicroBlockFromLookup( const zbytes& src, const unsigned int offset, vector& microBlockHashes, uint32_t& portNo) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5763,7 +5643,6 @@ bool Messenger::GetLookupSetMicroBlockFromLookup(const zbytes& src, const unsigned int offset, PubKey& lookupPubKey, vector& mbs) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5812,7 +5691,6 @@ bool Messenger::SetLookupGetTxnsFromLookup(zbytes& dst, const BlockHash& mbHash, const vector& txnhashes, const uint32_t portNo) { - LookupGetTxnsFromLookup result; result.set_portno(portNo); @@ -5836,7 +5714,6 @@ bool Messenger::GetLookupGetTxnsFromLookup(const zbytes& src, BlockHash& mbHash, vector& txnhashes, uint32_t& portNo) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5869,7 +5746,6 @@ bool Messenger::SetLookupGetTxnsFromL2l(zbytes& dst, const unsigned int offset, const vector& txnhashes, const uint32_t portNo, const PairOfKey& seedKey) { - LookupGetTxnsFromL2l result; result.mutable_data()->set_portno(portNo); @@ -5908,7 +5784,6 @@ bool Messenger::SetLookupGetTxnsFromL2l(zbytes& dst, const unsigned int offset, bool Messenger::GetLookupGetTxnsFromL2l( const zbytes& src, const unsigned int offset, BlockHash& mbHash, vector& txnhashes, uint32_t& portNo, PubKey& senderPubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -5950,7 +5825,6 @@ bool Messenger::GetLookupGetTxnsFromL2l( bool Messenger::SetLookupSetTxnsFromLookup( zbytes& dst, const unsigned int offset, const PairOfKey& lookupKey, const BlockHash& mbHash, const vector& txns) { - LookupSetTxnsFromLookup result; result.set_mbhash(mbHash.data(), mbHash.size); @@ -5988,7 +5862,6 @@ bool Messenger::SetLookupSetTxnsFromLookup( bool Messenger::GetLookupSetTxnsFromLookup( const zbytes& src, const unsigned int offset, PubKey& lookupPubKey, BlockHash& mbHash, vector& txns) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -6209,7 +6082,6 @@ bool Messenger::SetConsensusCommit(zbytes& dst, const unsigned int offset, const uint16_t backupID, const vector& commitInfo, const PairOfKey& backupKey) { - ConsensusCommit result; result.mutable_consensusinfo()->set_consensusid(consensusID); @@ -6258,7 +6130,6 @@ bool Messenger::GetConsensusCommit(const zbytes& src, const unsigned int offset, const zbytes& blockHash, uint16_t& backupID, vector& commitInfo, const DequeOfNode& committeeKeys) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -6351,7 +6222,6 @@ bool Messenger::SetConsensusChallenge( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const vector& subsetInfo, const PairOfKey& leaderKey) { - ConsensusChallenge result; result.mutable_consensusinfo()->set_consensusid(consensusID); @@ -6402,7 +6272,6 @@ bool Messenger::GetConsensusChallenge( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, vector& subsetInfo, const PubKey& leaderKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -6494,7 +6363,6 @@ bool Messenger::SetConsensusResponse( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t backupID, const vector& subsetInfo, const PairOfKey& backupKey) { - ConsensusResponse result; result.mutable_consensusinfo()->set_consensusid(consensusID); @@ -6539,7 +6407,6 @@ bool Messenger::GetConsensusResponse( const zbytes& src, const unsigned int offset, const uint32_t consensusID, const uint64_t blockNumber, const zbytes& blockHash, uint16_t& backupID, vector& subsetInfo, const DequeOfNode& committeeKeys) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -6632,7 +6499,6 @@ bool Messenger::SetConsensusCollectiveSig( const uint16_t leaderID, const Signature& collectiveSig, const vector& bitmap, const PairOfKey& leaderKey, const zbytes& newAnnouncementMessage) { - ConsensusCollectiveSig result; result.mutable_consensusinfo()->set_consensusid(consensusID); @@ -6692,7 +6558,6 @@ bool Messenger::GetConsensusCollectiveSig( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, vector& bitmap, Signature& collectiveSig, const PubKey& leaderKey, zbytes& newAnnouncement) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -6796,7 +6661,6 @@ bool Messenger::SetConsensusCommitFailure( const uint64_t blockNumber, const zbytes& blockHash, const uint16_t backupID, const zbytes& errorMsg, const PairOfKey& backupKey) { - ConsensusCommitFailure result; result.mutable_consensusinfo()->set_consensusid(consensusID); @@ -6837,7 +6701,6 @@ bool Messenger::GetConsensusCommitFailure( const zbytes& src, const unsigned int offset, const uint32_t consensusID, const uint64_t blockNumber, const zbytes& blockHash, uint16_t& backupID, zbytes& errorMsg, const DequeOfNode& committeeKeys) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -6924,7 +6787,6 @@ bool Messenger::SetConsensusConsensusFailure( zbytes& dst, const unsigned int offset, const uint32_t consensusID, const uint64_t blockNumber, const zbytes& blockHash, const uint16_t leaderID, const PairOfKey& leaderKey) { - ConsensusConsensusFailure result; result.mutable_consensusinfo()->set_consensusid(consensusID); @@ -6964,7 +6826,6 @@ bool Messenger::GetConsensusConsensusFailure( const zbytes& src, const unsigned int offset, const uint32_t consensusID, const uint64_t blockNumber, const zbytes& blockHash, uint16_t& leaderID, const PubKey& leaderKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -7049,7 +6910,6 @@ bool Messenger::SetLookupGetDSTxBlockFromSeed( zbytes& dst, const unsigned int offset, const uint64_t dsLowBlockNum, const uint64_t dsHighBlockNum, const uint64_t txLowBlockNum, const uint64_t txHighBlockNum, const uint32_t listenPort) { - LookupGetDSTxBlockFromSeed result; result.set_dslowblocknum(dsLowBlockNum); @@ -7070,7 +6930,6 @@ bool Messenger::GetLookupGetDSTxBlockFromSeed( const zbytes& src, const unsigned int offset, uint64_t& dsLowBlockNum, uint64_t& dsHighBlockNum, uint64_t& txLowBlockNum, uint64_t& txHighBlockNum, uint32_t& listenPort) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -7099,7 +6958,6 @@ bool Messenger::SetVCNodeSetDSTxBlockFromSeed(zbytes& dst, const PairOfKey& lookupKey, const vector& DSBlocks, const vector& txBlocks) { - VCNodeSetDSTxBlockFromSeed result; for (const auto& dsblock : DSBlocks) { @@ -7142,7 +7000,6 @@ bool Messenger::GetVCNodeSetDSTxBlockFromSeed(const zbytes& src, vector& dsBlocks, vector& txBlocks, PubKey& lookupPubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -7237,7 +7094,6 @@ bool Messenger::GetNodeNewShardNodeNetworkInfo(const zbytes& src, Peer& shardNodeNewNetworkInfo, uint64_t& timestamp, PubKey& shardNodePubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -7320,7 +7176,6 @@ bool Messenger::SetDSLookupNewDSGuardNetworkInfo( bool Messenger::GetDSLookupNewDSGuardNetworkInfo( const zbytes& src, const unsigned int offset, uint64_t& dsEpochNumber, Peer& dsGuardNewNetworkInfo, uint64_t& timestamp, PubKey& dsGuardPubkey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -7360,7 +7215,6 @@ bool Messenger::GetDSLookupNewDSGuardNetworkInfo( bool Messenger::SetLookupGetNewDSGuardNetworkInfoFromLookup( zbytes& dst, const unsigned int offset, const uint32_t portNo, const uint64_t dsEpochNumber, const PairOfKey& lookupKey) { - NodeGetGuardNodeNetworkInfoUpdate result; result.mutable_data()->set_portno(portNo); result.mutable_data()->set_dsepochnumber(dsEpochNumber); @@ -7388,7 +7242,6 @@ bool Messenger::SetLookupGetNewDSGuardNetworkInfoFromLookup( bool Messenger::GetLookupGetNewDSGuardNetworkInfoFromLookup( const zbytes& src, const unsigned int offset, uint32_t& portNo, uint64_t& dsEpochNumber) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -7475,7 +7328,6 @@ bool Messenger::SetNodeGetNewDSGuardNetworkInfo( const zbytes& src, const unsigned int offset, vector& vecOfDSGuardUpdateStruct, PubKey& lookupPubKey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -7589,7 +7441,6 @@ bool Messenger::SetLookupGetCosigsRewardsFromSeed(zbytes& dst, const uint64_t txBlkNum, const uint32_t listenPort, const PairOfKey& keys) { - LookupGetCosigsRewardsFromSeed result; result.mutable_data()->set_epochnumber(txBlkNum); @@ -7832,7 +7683,6 @@ bool Messenger::SetLookupSetCosigsRewardsFromSeed( bool Messenger::GetLookupSetCosigsRewardsFromSeed( const zbytes& src, const unsigned int offset, vector& cosigrewards, PubKey& senderPubkey) { - if (offset >= src.size()) { LOG_GENERAL(WARNING, "Invalid data and offset, data size " << src.size() << ", offset " << offset); @@ -7892,7 +7742,6 @@ bool Messenger::GetLookupSetCosigsRewardsFromSeed( bool Messenger::SetMinerInfoDSComm(zbytes& dst, const unsigned int offset, const MinerInfoDSComm& minerInfo) { - ProtoMinerInfoDSComm result; for (const auto& dsnode : minerInfo.m_dsNodes) { @@ -7915,7 +7764,6 @@ bool Messenger::SetMinerInfoDSComm(zbytes& dst, const unsigned int offset, bool Messenger::GetMinerInfoDSComm(const zbytes& src, const unsigned int offset, MinerInfoDSComm& minerInfo) { - minerInfo.m_dsNodes.clear(); minerInfo.m_dsNodesEjected.clear(); @@ -7955,7 +7803,6 @@ bool Messenger::GetMinerInfoDSComm(const zbytes& src, const unsigned int offset, bool Messenger::SetMinerInfoShards(zbytes& dst, const unsigned int offset, const MinerInfoShards& minerInfo) { - ProtoMinerInfoShards result; for (const auto& shard : minerInfo.m_shards) { @@ -7978,7 +7825,6 @@ bool Messenger::SetMinerInfoShards(zbytes& dst, const unsigned int offset, bool Messenger::GetMinerInfoShards(const zbytes& src, const unsigned int offset, MinerInfoShards& minerInfo) { - minerInfo.m_shards.clear(); if (src.size() == 0) { diff --git a/src/libMessage/Messenger.h b/src/libMessage/Messenger.h index 41df49352c..4d4ee7e089 100644 --- a/src/libMessage/Messenger.h +++ b/src/libMessage/Messenger.h @@ -336,12 +336,11 @@ class Messenger { const std::unordered_map& hashCodeMap, const uint32_t shardId, const PairOfKey& key); - static bool SetNodeForwardTxnBlock( - zbytes& dst, const unsigned int offset, const uint64_t& epochNumber, - const uint64_t& dsBlockNum, const uint32_t& shardId, - const PairOfKey& lookupKey, - std::deque>& txnsCurrent, - std::deque>& txnsGenerated); + static bool SetNodeForwardTxnBlock(zbytes& dst, const unsigned int offset, + const uint64_t& epochNumber, + const uint64_t& dsBlockNum, + const PairOfKey& lookupKey, + std::vector& transactions); static bool SetNodeForwardTxnBlock(zbytes& dst, const unsigned int offset, const uint64_t& epochNumber, const uint64_t& dsBlockNum, @@ -595,13 +594,11 @@ class Messenger { static bool SetForwardTxnBlockFromSeed( zbytes& dst, const unsigned int offset, - const std::deque>& shardTransactions, - const std::deque>& dsTransactions); + const std::vector& transactions); static bool GetForwardTxnBlockFromSeed( const zbytes& src, const unsigned int offset, - std::vector& shardTransactions, - std::vector& dsTransactions); + std::vector& transactions); static bool SetLookupGetMicroBlockFromLookup( zbytes& dst, const unsigned int offset, diff --git a/src/libMessage/ZilliqaMessage.proto b/src/libMessage/ZilliqaMessage.proto index 4317f8d2f0..2535d904eb 100644 --- a/src/libMessage/ZilliqaMessage.proto +++ b/src/libMessage/ZilliqaMessage.proto @@ -865,8 +865,7 @@ message LookupGetMicroBlockFromL2l message LookupForwardTxnsFromSeed { - ProtoTransactionArray shardtransactions = 1; - ProtoTransactionArray dstransactions = 2; + ProtoTransactionArray transactions = 1; } message LookupGetTxnsFromLookup diff --git a/src/libNode/DSBlockProcessing.cpp b/src/libNode/DSBlockProcessing.cpp index 3562096853..4b1d45a7f3 100644 --- a/src/libNode/DSBlockProcessing.cpp +++ b/src/libNode/DSBlockProcessing.cpp @@ -357,6 +357,10 @@ bool Node::ProcessVCDSBlocksMessage( [[gnu::unused]] const unsigned char& startByte) { LOG_MARKER(); + const bool leader = m_mediator.m_ds->GetConsensusMyID() == + m_mediator.m_ds->GetConsensusMyID(); + LOG_GENERAL(WARNING, "BZ ProcessVCDSBlocksMessage enter, I am leader? : " + << (leader ? "true" : "false")); unsigned int oldNumShards = m_mediator.m_ds->GetNumShards(); lock_guard g(m_mutexDSBlock); @@ -717,8 +721,8 @@ bool Node::ProcessVCDSBlocksMessage( m_mediator.m_node->CleanWhitelistReqs(); - if (m_mediator.m_lookup->GetIsServer() && !ARCHIVAL_LOOKUP) { - m_mediator.m_lookup->SenderTxnBatchThread(oldNumShards, true); + if (m_mediator.m_lookup->GetIsServer() && ARCHIVAL_LOOKUP) { + SendTxnMemPoolToNextLayer(); } } diff --git a/src/libNode/FinalBlockProcessing.cpp b/src/libNode/FinalBlockProcessing.cpp index 6c6f087d84..7ae1b42259 100644 --- a/src/libNode/FinalBlockProcessing.cpp +++ b/src/libNode/FinalBlockProcessing.cpp @@ -118,7 +118,8 @@ class FinalBLockProcessingVariables { result.Set(lastVcBlockHeight, {{"counter", "LastVcBlockHeight"}}); result.Set(forwardedTx, {{"counter", "ForwardedTx"}}); result.Set(timedOutMicroblock, {{"counter", "TimedOutMicroblock"}}); - result.Set(missedMicroblockConsensus, {{"counter", "MissedMicroblockConsensus"}}); + result.Set(missedMicroblockConsensus, + {{"counter", "MissedMicroblockConsensus"}}); result.Set(isShardLeader, {{"counter", "IsShardLeader"}}); result.Set(shard, {{"counter", "Shard"}}); }); @@ -395,7 +396,8 @@ void Node::InitiatePoW() { "Node::InitiatePoW not expected to be called from LookUp node."); return; } - + LOG_GENERAL(WARNING, "BZ InitiatePoW enter"); + LOG_GENERAL(WARNING, "BZ Setting state to POW_SUBMISSION"); SetState(POW_SUBMISSION); if (m_mediator.m_disablePoW) { @@ -412,6 +414,7 @@ void Node::InitiatePoW() { m_mediator.m_dsBlockChain.GetLastBlock().GetHeader().GetBlockNum() + 1; auto dsBlockRand = m_mediator.m_dsBlockRand; auto txBlockRand = m_mediator.m_txBlockRand; + LOG_GENERAL(WARNING, "BZ Will start PoW in detached function enter"); StartPoW( epochNumber, m_mediator.m_dsBlockChain.GetLastBlock().GetHeader().GetDSDifficulty(), @@ -1042,11 +1045,12 @@ bool Node::ProcessFinalBlockCore(uint64_t& dsBlockNumber, (m_state == MICROBLOCK_CONSENSUS || m_state == MICROBLOCK_CONSENSUS_PREP)) { std::unique_lock cv_lk(m_MutexCVFBWaitMB); - // TODO: cv fix if (cv_FBWaitMB.wait_for( cv_lk, std::chrono::seconds(CONSENSUS_MSG_ORDER_BLOCK_WINDOW)) == std::cv_status::timeout) { - LOG_GENERAL(WARNING, "Timeout, I didn't finish microblock consensus. Timeout: " << CONSENSUS_MSG_ORDER_BLOCK_WINDOW << " seconds"); + LOG_GENERAL(WARNING, + "Timeout, I didn't finish microblock consensus. Timeout: " + << CONSENSUS_MSG_ORDER_BLOCK_WINDOW << " seconds"); zil::local::variables.AddTimedOutMicroblock(1); } } @@ -1112,7 +1116,9 @@ bool Node::ProcessFinalBlockCore(uint64_t& dsBlockNumber, m_pendingTxns.erase(txnHash); } } - LOG_GENERAL(WARNING, "Node::ProcessFinalBlockCore ProcessStateDeltaFromFinalBlock ENTER"); + LOG_GENERAL( + WARNING, + "Node::ProcessFinalBlockCore ProcessStateDeltaFromFinalBlock ENTER"); if (!ProcessStateDeltaFromFinalBlock( stateDelta, txBlock.GetHeader().GetStateDeltaHash())) { return false; @@ -1287,6 +1293,7 @@ bool Node::ProcessFinalBlockCore(uint64_t& dsBlockNumber, } ClearUnconfirmedTxn(); if (isVacuousEpoch) { + LOG_GENERAL(WARNING, "BZ It's vacuous epoch, will start PoW soon"); InitiatePoW(); } else { auto main_func = [this]() mutable -> void { BeginNextConsensusRound(); }; @@ -1301,14 +1308,14 @@ bool Node::ProcessFinalBlockCore(uint64_t& dsBlockNumber, } // Now only forwarded txn are left, so only call in lookup - uint32_t numShards = m_mediator.m_ds->GetNumShards(); - CommitMBnForwardedTransactionBuffer(); - if (!ARCHIVAL_LOOKUP && m_mediator.m_lookup->GetIsServer() && + // Seed/external nodes send mempool transactions upon arrival of final block + if (ARCHIVAL_LOOKUP && m_mediator.m_lookup->GetIsServer() && !isVacuousEpoch && !m_mediator.GetIsVacuousEpoch() && ((m_mediator.m_currentEpochNum + NUM_VACUOUS_EPOCHS) % NUM_FINAL_BLOCK_PER_POW) != 0) { - m_mediator.m_lookup->SenderTxnBatchThread(numShards); + SendTxnMemPoolToNextLayer(); + m_mediator.m_lookup->ClearTxnMemPool(); } m_mediator.m_lookup->CheckAndFetchUnavailableMBs( @@ -1317,6 +1324,25 @@ bool Node::ProcessFinalBlockCore(uint64_t& dsBlockNumber, return true; } +void Node::SendTxnMemPoolToNextLayer() { + LOG_GENERAL(WARNING, + "BZ Sending from seed to lookup txn with size: " + << m_mediator.m_lookup->GetTransactionsFromMemPool().size()); + zbytes msg = {MessageType::LOOKUP, LookupInstructionType::FORWARDTXN}; + + { + lock_guard g(m_mediator.m_lookup->m_txnMemPoolMutex); + if (!Messenger::SetForwardTxnBlockFromSeed( + msg, MessageOffset::BODY, + m_mediator.m_lookup->GetTransactionsFromMemPool())) { + LOG_GENERAL(WARNING, "Unable to serialize txn mempool into protobuf msg"); + } + } + // Sending mempool txns to random node next in the ladder (either lookup or + // upper seed for external nodes) + m_mediator.m_lookup->SendMessageToRandomSeedNode(msg); +} + bool Node::ProcessStateDeltaFromFinalBlock( const zbytes& stateDeltaBytes, const StateHash& finalBlockStateDeltaHash) { LOG_MARKER(); @@ -1407,8 +1433,10 @@ void Node::CommitForwardedTransactions(const MBnForwardedTxnEntry& entry) { LOG_GENERAL(WARNING, "BlockStorage::PutTxBody failed " << txhash); return; } - LOG_GENERAL(WARNING, "Node::CommitForwardedTransactions Commititng: " << txhash.hex() << ", NONCE: " << tran.GetNonce()); - + LOG_GENERAL(WARNING, "Node::CommitForwardedTransactions Commititng: " + << txhash.hex() + << ", NONCE: " << tran.GetNonce()); + if (LOOKUP_NODE_MODE) { LookupServer::AddToRecentTransactions(txhash); const auto& receipt = twr.GetTransactionReceipt(); @@ -1656,13 +1684,13 @@ bool Node::ProcessMBnForwardTransaction( // soft confirmation SoftConfirmForwardedTransactions(entry); // invoke txn distribution - if (!ARCHIVAL_LOOKUP && !m_mediator.GetIsVacuousEpoch() && + /*if (!ARCHIVAL_LOOKUP && !m_mediator.GetIsVacuousEpoch() && ((m_mediator.m_currentEpochNum + NUM_VACUOUS_EPOCHS + 1) % NUM_FINAL_BLOCK_PER_POW != 0)) { m_mediator.m_lookup->SendTxnPacketToShard( entry.m_microBlock.GetHeader().GetShardId(), false, true); - } + }*/ return true; } diff --git a/src/libNode/Node.cpp b/src/libNode/Node.cpp index d6dc129916..388aa0c28c 100644 --- a/src/libNode/Node.cpp +++ b/src/libNode/Node.cpp @@ -809,6 +809,7 @@ void Node::Prepare(bool runInitializeGenesisBlocks) { m_mediator.m_txBlockChain.GetLastBlock().GetHeader().GetBlockNum() + 1; m_mediator.UpdateDSBlockRand(runInitializeGenesisBlocks); m_mediator.UpdateTxBlockRand(runInitializeGenesisBlocks); + LOG_GENERAL(WARNING, "BZ: Prepare -> POW_SUBMISSION"); SetState(POW_SUBMISSION); POW::GetInstance().EthashConfigureClient( m_mediator.m_dsBlockChain.GetLastBlock().GetHeader().GetBlockNum() + 1, @@ -1482,10 +1483,9 @@ uint32_t Node::CalculateShardLeaderFromDequeOfNode( uint16_t lastBlockHash, uint32_t sizeOfShard, const DequeOfNode &shardMembers) { LOG_MARKER(); - LOG_GENERAL(INFO,"STEVE lastBlockHash: " << lastBlockHash - << ", sizeOfShard: " << sizeOfShard - << ", shardMembers.size(): " - << shardMembers.size()); + LOG_GENERAL(INFO, "STEVE lastBlockHash: " + << lastBlockHash << ", sizeOfShard: " << sizeOfShard + << ", shardMembers.size(): " << shardMembers.size()); if (GUARD_MODE) { uint32_t consensusLeaderIndex = lastBlockHash % sizeOfShard; @@ -1729,6 +1729,8 @@ bool Node::ProcessTxnPacketFromLookup( [[gnu::unused]] const unsigned char &startByte) { LOG_MARKER(); + LOG_GENERAL(WARNING, "BZ ProcessTxnPacketFromLookup"); + if (LOOKUP_NODE_MODE) { LOG_GENERAL(WARNING, "Node::ProcessTxnPacketFromLookup not expected to " @@ -1871,6 +1873,8 @@ bool Node::ProcessTxnPacketFromLookupCore(const zbytes &message, const vector &txns) { LOG_MARKER(); + LOG_GENERAL(WARNING, "BZ ProcessTxnPacketFromLookupCore"); + if (LOOKUP_NODE_MODE) { LOG_GENERAL(WARNING, "Node::ProcessTxnPacketFromLookupCore not expected to " @@ -1920,8 +1924,8 @@ bool Node::ProcessTxnPacketFromLookupCore(const zbytes &message, } if (shardId != m_myshardId) { - LOG_GENERAL(WARNING, "Wrong Shard (" << shardId << "), m_myshardId (" - << m_myshardId << ")"); + LOG_GENERAL(WARNING, "BZ Wrong Shard (" << shardId << "), m_myshardId (" + << m_myshardId << ")"); return false; } @@ -2373,6 +2377,7 @@ void Node::SetMyshardId(uint32_t shardId) { "Node::SetMyshardId not expected to be called from LookUp node."); return; } + LOG_GENERAL(WARNING, "BZ MyShardId is: " << shardId); m_myshardId = shardId; } @@ -2531,7 +2536,8 @@ bool Node::ProcessRemoveNodeFromBlacklist( if (!WhitelistReqsValidator(from.GetIpAddress())) { // Blacklist - strict one - since too many whitelist request in current ds // epoch. - Blacklist::GetInstance().Add({from.GetIpAddress(),from.GetListenPortHost(),from.GetNodeIndentifier()}); + Blacklist::GetInstance().Add({from.GetIpAddress(), from.GetListenPortHost(), + from.GetNodeIndentifier()}); return false; } @@ -2567,7 +2573,8 @@ bool Node::ProcessRemoveNodeFromBlacklist( return false; } - Blacklist::GetInstance().Remove({ipAddress,from.GetListenPortHost(),from.GetNodeIndentifier()}); + Blacklist::GetInstance().Remove( + {ipAddress, from.GetListenPortHost(), from.GetNodeIndentifier()}); return true; } @@ -2821,7 +2828,7 @@ bool Node::ProcessGetVersion(const zbytes &message, unsigned int offset, return false; } zil::p2p::GetInstance().SendMessage(Peer(from.m_ipAddress, portNo), - response); + response); m_versionChecked = true; } @@ -2963,9 +2970,9 @@ bool Node::ProcessDSGuardNetworkInfoUpdate( }); if (it != m_mediator.m_DSCommittee->end()) { - Blacklist::GetInstance().RemoveFromWhitelist({it->second.m_ipAddress, - it->second.m_listenPortHost, - it->second.GetNodeIndentifier()} ); + Blacklist::GetInstance().RemoveFromWhitelist( + {it->second.m_ipAddress, it->second.m_listenPortHost, + it->second.GetNodeIndentifier()}); LOG_GENERAL(INFO, "Removed " << it->second.m_ipAddress << " from blacklist exclude list"); } @@ -2986,9 +2993,10 @@ bool Node::ProcessDSGuardNetworkInfoUpdate( << " new network info is " << dsguardupdate.m_dsGuardNewNetworkInfo) if (GUARD_MODE) { - Blacklist::GetInstance().Whitelist({dsguardupdate.m_dsGuardNewNetworkInfo.m_ipAddress, - dsguardupdate.m_dsGuardNewNetworkInfo.m_listenPortHost, - dsguardupdate.m_dsGuardNewNetworkInfo.GetNodeIndentifier()}); + Blacklist::GetInstance().Whitelist( + {dsguardupdate.m_dsGuardNewNetworkInfo.m_ipAddress, + dsguardupdate.m_dsGuardNewNetworkInfo.m_listenPortHost, + dsguardupdate.m_dsGuardNewNetworkInfo.GetNodeIndentifier()}); LOG_GENERAL(INFO, "Added ds guard " << dsguardupdate.m_dsGuardNewNetworkInfo.m_ipAddress @@ -3184,6 +3192,8 @@ bool Node::Execute(const zbytes &message, unsigned int offset, const Peer &from, return false; } + LOG_GENERAL(WARNING, "BZ Dispatching Node msg type: " + << hex << (unsigned int)ins_byte); if (ins_byte < ins_handlers_count) { result = (this->*ins_handlers[ins_byte])(message, offset + 1, from, startByte); diff --git a/src/libNode/Node.h b/src/libNode/Node.h index d66ad5f48a..a13407733d 100644 --- a/src/libNode/Node.h +++ b/src/libNode/Node.h @@ -419,6 +419,8 @@ class Node : public Executable { void UpdateGovProposalRemainingVoteInfo(); bool CheckIfGovProposalActive(); + void SendTxnMemPoolToNextLayer(); + public: enum NodeState : unsigned char { POW_SUBMISSION = 0x00, @@ -590,7 +592,7 @@ class Node : public Executable { bool DownloadPersistenceFromS3(); /// Recover the previous state by retrieving persistence data - bool StartRetrieveHistory(const SyncType syncType, bool &allowRecoverAllSync, + bool StartRetrieveHistory(const SyncType syncType, bool& allowRecoverAllSync, bool rejoiningAfterRecover = false); bool CheckIntegrity(const bool fromValidateDBBinary = false); diff --git a/src/libNode/PoWProcessing.cpp b/src/libNode/PoWProcessing.cpp index 72cdad6ff4..99939e5e09 100644 --- a/src/libNode/PoWProcessing.cpp +++ b/src/libNode/PoWProcessing.cpp @@ -457,6 +457,7 @@ bool Node::ProcessStartPoW(const zbytes& message, unsigned int offset, Guard::GetInstance().AddDSGuardToBlacklistExcludeList( *m_mediator.m_DSCommittee); // Start mining + LOG_GENERAL(WARNING, "BZ Start Processing POW"); StartPoW(block_num, dsDifficulty, difficulty, rand1, rand2); return true; diff --git a/src/libServer/EthRpcMethods.cpp b/src/libServer/EthRpcMethods.cpp index 88e9b66239..2df2b62348 100644 --- a/src/libServer/EthRpcMethods.cpp +++ b/src/libServer/EthRpcMethods.cpp @@ -377,14 +377,14 @@ void EthRpcMethods::Init(LookupServer *lookupServer) { m_lookupServer->bindAndAddExternalMethod( jsonrpc::Procedure("ots_enable", jsonrpc::PARAMS_BY_POSITION, - jsonrpc::JSON_STRING, "param01", - jsonrpc::JSON_BOOLEAN, NULL), + jsonrpc::JSON_STRING, "param01", jsonrpc::JSON_BOOLEAN, + NULL), &EthRpcMethods::OtterscanEnableI); m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("ots_getInternalOperations", jsonrpc::PARAMS_BY_POSITION, - jsonrpc::JSON_STRING, "param01", jsonrpc::JSON_STRING, - NULL), + jsonrpc::Procedure("ots_getInternalOperations", + jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_STRING, + "param01", jsonrpc::JSON_STRING, NULL), &EthRpcMethods::OtterscanGetInternalOperationsI); m_lookupServer->bindAndAddExternalMethod( @@ -394,20 +394,24 @@ void EthRpcMethods::Init(LookupServer *lookupServer) { &EthRpcMethods::OtterscanTraceTransactionI); m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("ots_searchTransactionsBefore", - jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_STRING, - "param01", jsonrpc::JSON_STRING, "param02", jsonrpc::JSON_INTEGER, "param03", jsonrpc::JSON_INTEGER, NULL), + jsonrpc::Procedure( + "ots_searchTransactionsBefore", jsonrpc::PARAMS_BY_POSITION, + jsonrpc::JSON_STRING, "param01", jsonrpc::JSON_STRING, "param02", + jsonrpc::JSON_INTEGER, "param03", jsonrpc::JSON_INTEGER, NULL), &EthRpcMethods::OtterscanSearchTransactionsBeforeI); - + m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("ots_searchTransactionsAfter", - jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_STRING, - "param01", jsonrpc::JSON_STRING, "param02", jsonrpc::JSON_INTEGER, "param03", jsonrpc::JSON_INTEGER, NULL), + jsonrpc::Procedure( + "ots_searchTransactionsAfter", jsonrpc::PARAMS_BY_POSITION, + jsonrpc::JSON_STRING, "param01", jsonrpc::JSON_STRING, "param02", + jsonrpc::JSON_INTEGER, "param03", jsonrpc::JSON_INTEGER, NULL), &EthRpcMethods::OtterscanSearchTransactionsAfterI); m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("ots_getTransactionBySenderAndNonce", jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_STRING, - "param01", jsonrpc::JSON_STRING, "param02", jsonrpc::JSON_INTEGER, NULL), + jsonrpc::Procedure("ots_getTransactionBySenderAndNonce", + jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_STRING, + "param01", jsonrpc::JSON_STRING, "param02", + jsonrpc::JSON_INTEGER, NULL), &EthRpcMethods::OtterscanGetTransactionBySenderAndNonceI); m_lookupServer->bindAndAddExternalMethod( @@ -429,60 +433,46 @@ void EthRpcMethods::Init(LookupServer *lookupServer) { &EthRpcMethods::GetDSLeaderTxnPoolI); m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("erigon_getHeaderByNumber", jsonrpc::PARAMS_BY_POSITION, - jsonrpc::JSON_OBJECT, - "param01", jsonrpc::JSON_INTEGER, - NULL), - &EthRpcMethods::GetHeaderByNumberI - ); + jsonrpc::Procedure("erigon_getHeaderByNumber", + jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_OBJECT, + "param01", jsonrpc::JSON_INTEGER, NULL), + &EthRpcMethods::GetHeaderByNumberI); m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("ots_getApiLevel", jsonrpc::PARAMS_BY_POSITION, - jsonrpc::JSON_INTEGER, - NULL), - &EthRpcMethods::GetOtterscanApiLevelI - ); + jsonrpc::Procedure("ots_getApiLevel", jsonrpc::PARAMS_BY_POSITION, + jsonrpc::JSON_INTEGER, NULL), + &EthRpcMethods::GetOtterscanApiLevelI); m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("ots_hasCode", jsonrpc::PARAMS_BY_POSITION, - jsonrpc::JSON_BOOLEAN, - "param01", jsonrpc::JSON_STRING, - "param02", jsonrpc::JSON_STRING, - NULL), - &EthRpcMethods::HasCodeI - ); + jsonrpc::Procedure("ots_hasCode", jsonrpc::PARAMS_BY_POSITION, + jsonrpc::JSON_BOOLEAN, "param01", jsonrpc::JSON_STRING, + "param02", jsonrpc::JSON_STRING, NULL), + &EthRpcMethods::HasCodeI); m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("ots_getBlockDetails", jsonrpc::PARAMS_BY_POSITION, - jsonrpc::JSON_OBJECT, - "param01", jsonrpc::JSON_INTEGER, - NULL), - &EthRpcMethods::GetBlockDetailsI - ); + jsonrpc::Procedure("ots_getBlockDetails", jsonrpc::PARAMS_BY_POSITION, + jsonrpc::JSON_OBJECT, "param01", jsonrpc::JSON_INTEGER, + NULL), + &EthRpcMethods::GetBlockDetailsI); m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("ots_getBlockTransactions", jsonrpc::PARAMS_BY_POSITION, - jsonrpc::JSON_OBJECT, - "param01", jsonrpc::JSON_INTEGER, - "param02", jsonrpc::JSON_INTEGER, - "param03", jsonrpc::JSON_INTEGER, - NULL), - &EthRpcMethods::GetBlockTransactionsI - ); + jsonrpc::Procedure( + "ots_getBlockTransactions", jsonrpc::PARAMS_BY_POSITION, + jsonrpc::JSON_OBJECT, "param01", jsonrpc::JSON_INTEGER, "param02", + jsonrpc::JSON_INTEGER, "param03", jsonrpc::JSON_INTEGER, NULL), + &EthRpcMethods::GetBlockTransactionsI); m_lookupServer->bindAndAddExternalMethod( - jsonrpc::Procedure("ots_getContractCreator", jsonrpc::PARAMS_BY_POSITION, - jsonrpc::JSON_OBJECT, - "param01", jsonrpc::JSON_STRING, - NULL), - &EthRpcMethods::GetContractCreatorI - ); -} - -std::string EthRpcMethods::CreateTransactionEth( - Eth::EthFields const &fields, zbytes const &pubKey, - const unsigned int num_shards, const uint128_t &gasPrice, - const CreateTransactionTargetFunc &targetFunc) { + jsonrpc::Procedure("ots_getContractCreator", jsonrpc::PARAMS_BY_POSITION, + jsonrpc::JSON_OBJECT, "param01", jsonrpc::JSON_STRING, + NULL), + &EthRpcMethods::GetContractCreatorI); +} + +std::string EthRpcMethods::CreateTransactionEth(Eth::EthFields const &fields, + zbytes const &pubKey, + const unsigned int num_shards, + const uint128_t &gasPrice) { TRACE(zil::trace::FilterClass::TXN); INC_CALLS(GetInvocationsCounter()); @@ -524,20 +514,12 @@ std::string EthRpcMethods::CreateTransactionEth( try { const Address fromAddr = tx.GetSenderAddr(); - bool toAccountExist; - bool toAccountIsContract; - { unique_lock lock( AccountStore::GetInstance().GetPrimaryMutex()); const Account *sender = AccountStore::GetInstance().GetAccount(fromAddr, true); - const Account *toAccount = - AccountStore::GetInstance().GetAccount(tx.GetToAddr(), true); - - toAccountExist = (toAccount != nullptr); - toAccountIsContract = toAccountExist && toAccount->isContract(); uint64_t minGasLimit = 0; if (Transaction::GetTransactionType(tx) == @@ -554,33 +536,25 @@ std::string EthRpcMethods::CreateTransactionEth( TRACE_EVENT("Validated", "status", "OK"); } - LOG_GENERAL(WARNING, "EthRpcMethods::CreateTransactionEth ADDR: " << tx.GetSenderAddr() - << ", NONCE: " << tx.GetNonce() - << ", HASH: " << tx.GetTranID().hex() << ", TO: " << tx.GetCoreInfo().toAddr.hex()); + if (tx.GetGasLimitZil() > DS_MICROBLOCK_GAS_LIMIT) { + throw JsonRpcException( + ServerBase::RPC_INVALID_PARAMETER, + (boost::format( + "txn gas limit exceeding ds maximum limit! Tx: %i DS: %i") % + tx.GetGasLimitZil() % DS_MICROBLOCK_GAS_LIMIT) + .str()); + } + + LOG_GENERAL(WARNING, "EthRpcMethods::CreateTransactionEth ADDR: " + << tx.GetSenderAddr() + << ", NONCE: " << tx.GetNonce() + << ", HASH: " << tx.GetTranID().hex() + << ", TO: " << tx.GetCoreInfo().toAddr.hex()); - const unsigned int shard = Transaction::GetShardIndex(fromAddr, num_shards); - unsigned int mapIndex = shard; - bool priority = false; switch (Transaction::GetTransactionType(tx)) { case Transaction::ContractType::NON_CONTRACT: - if (ARCHIVAL_LOOKUP) { - mapIndex = SEND_TYPE::ARCHIVAL_SEND_SHARD; - } - if (toAccountExist && toAccountIsContract) { - // A simple transfer to an account that is a contract - // is processed like a CONTRACT_CALL. - auto check = - CheckContractTxnShards(priority, shard, tx, num_shards, - toAccountExist, toAccountIsContract); - mapIndex = check.second; - } - break; case Transaction::ContractType::CONTRACT_CREATION: case Transaction::ContractType::CONTRACT_CALL: { - auto check = - CheckContractTxnShards(priority, shard, tx, num_shards, - toAccountExist, toAccountIsContract); - mapIndex = check.second; } break; case Transaction::ContractType::ERROR: throw JsonRpcException(ServerBase::RPC_INVALID_ADDRESS_OR_KEY, @@ -590,19 +564,12 @@ std::string EthRpcMethods::CreateTransactionEth( throw JsonRpcException(ServerBase::RPC_MISC_ERROR, "Txn type unexpected"); } - if (m_sharedMediator.m_lookup->m_sendAllToDS) { - if (ARCHIVAL_LOOKUP) { - mapIndex = SEND_TYPE::ARCHIVAL_SEND_DS; - } else { - mapIndex = num_shards; - } - } - TRACE_EVENT("Dispatching", "MapIndex", "some function"); - if (!targetFunc(tx, mapIndex)) { + if (!m_sharedMediator.m_lookup->AddTxnToMemPool(tx)) { throw JsonRpcException(ServerBase::RPC_DATABASE_ERROR, "Txn could not be added as database exceeded " "limit or the txn was already present"); } + } catch (const JsonRpcException &je) { LOG_GENERAL(INFO, "[Error]" << je.what() << " Input: N/A"); throw je; @@ -613,69 +580,6 @@ std::string EthRpcMethods::CreateTransactionEth( return ret; } -std::pair EthRpcMethods::CheckContractTxnShards( - bool priority, unsigned int shard, const Transaction &tx, - unsigned int num_shards, bool toAccountExist, bool toAccountIsContract) { - INC_CALLS(GetInvocationsCounter()); - - unsigned int mapIndex = shard; - std::string resultStr; - - if (!ENABLE_SC) { - throw JsonRpcException(ServerBase::RPC_MISC_ERROR, - "Smart contract is disabled"); - } - - if (!toAccountExist) { - throw JsonRpcException(ServerBase::RPC_INVALID_ADDRESS_OR_KEY, - "Target account does not exist"); - } else if (Transaction::GetTransactionType(tx) == - Transaction::CONTRACT_CALL && - !toAccountIsContract) { - throw JsonRpcException(ServerBase::RPC_INVALID_ADDRESS_OR_KEY, - "Non - contract address called"); - } - - Address affectedAddress = - (Transaction::GetTransactionType(tx) == Transaction::CONTRACT_CREATION) - ? Account::GetAddressForContract(tx.GetSenderAddr(), tx.GetNonce(), - tx.GetVersionIdentifier()) - : tx.GetToAddr(); - - unsigned int to_shard = - Transaction::GetShardIndex(affectedAddress, num_shards); - // Use m_sendSCCallsToDS as initial setting - bool sendToDs = priority || m_sharedMediator.m_lookup->m_sendSCCallsToDS; - if ((to_shard == shard) && !sendToDs) { - if (tx.GetGasLimitZil() > SHARD_MICROBLOCK_GAS_LIMIT) { - throw JsonRpcException(ServerBase::RPC_INVALID_PARAMETER, - "txn gas limit exceeding shard maximum limit"); - } - if (ARCHIVAL_LOOKUP) { - mapIndex = SEND_TYPE::ARCHIVAL_SEND_SHARD; - } - resultStr = - "Contract Creation/Call Txn, Shards Match of the sender " - "and receiver"; - } else { - if (tx.GetGasLimitZil() > DS_MICROBLOCK_GAS_LIMIT) { - throw JsonRpcException( - ServerBase::RPC_INVALID_PARAMETER, - (boost::format( - "txn gas limit exceeding ds maximum limit! Tx: %i DS: %i") % - tx.GetGasLimitZil() % DS_MICROBLOCK_GAS_LIMIT) - .str()); - } - if (ARCHIVAL_LOOKUP) { - mapIndex = SEND_TYPE::ARCHIVAL_SEND_DS; - } else { - mapIndex = num_shards; - } - resultStr = "Contract Creation/Call Txn, Sent To Ds"; - } - return make_pair(resultStr, mapIndex); -} - Json::Value EthRpcMethods::GetBalanceAndNonce(const string &address) { if (!LOOKUP_NODE_MODE) { throw JsonRpcException(ServerBase::RPC_INVALID_REQUEST, @@ -705,13 +609,15 @@ Json::Value EthRpcMethods::GetBalanceAndNonce(const string &address) { "DEBUG: Addr: " << address << " balance: " << balance.str() << " nonce: " << nonce << " " << account); } else if (account == nullptr) { - throw JsonRpcException(ServerBase::RPC_INVALID_ADDRESS_OR_KEY, - "Account with addr: " + address + " is not created"); + throw JsonRpcException( + ServerBase::RPC_INVALID_ADDRESS_OR_KEY, + "Account with addr: " + address + " is not created"); } return ret; } catch (const JsonRpcException &je) { - LOG_GENERAL(INFO, "[Error] getting balance for acc: " << address << ", msg: " << je.GetMessage()); + LOG_GENERAL(INFO, "[Error] getting balance for acc: " + << address << ", msg: " << je.GetMessage()); throw je; } catch (exception &e) { LOG_GENERAL(INFO, "[Error]" << e.what() << " Input: " << address); @@ -758,9 +664,9 @@ Json::Value extractTracer(const std::string &tracer, const std::string &trace) { parsed = item; } else if (tracer.compare("otter_internal_tracer") == 0) { auto const item = trace_json["otter_internal_tracer"]; - if(item.isNull()){ + if (item.isNull()) { parsed = Json::Value(Json::ValueType::arrayValue); - }else{ + } else { parsed = item; } } else if (tracer.compare("otter_call_tracer") == 0) { @@ -769,15 +675,17 @@ Json::Value extractTracer(const std::string &tracer, const std::string &trace) { } else if (tracer.compare("otter_transaction_error") == 0) { auto const item = trace_json["otter_transaction_error"]; // If there was no error return 0x - if(item.isNull()){ + if (item.isNull()) { parsed = Json::Value("0x"); - }else{ + } else { parsed = item; } } else { throw JsonRpcException( ServerBase::RPC_MISC_ERROR, - std::string("Only callTracer, internal_tracer, otter_call_tracer, otter_transaction_error, and raw are supported. Received: ") + + std::string( + "Only callTracer, internal_tracer, otter_call_tracer, " + "otter_transaction_error, and raw are supported. Received: ") + tracer); } } catch (exception &e) { @@ -844,7 +752,8 @@ bool EthRpcMethods::UnpackRevert(const std::string &data_in, return true; } -std::string EthRpcMethods::GetEthEstimateGas(const Json::Value &json, const std::string *block_or_tag) { +std::string EthRpcMethods::GetEthEstimateGas(const Json::Value &json, + const std::string *block_or_tag) { Address fromAddr; auto span = zil::trace::Tracing::CreateSpan(zil::trace::FilterClass::TXN, @@ -1392,7 +1301,8 @@ Json::Value EthRpcMethods::GetEthCode(std::string const &address, LOG_GENERAL(INFO, "[Error]" << e.what() << " Input: " << address); } std::string result{"0x"}; - boost::algorithm::hex_lower(code.begin(), code.end(), std::back_inserter(result)); + boost::algorithm::hex_lower(code.begin(), code.end(), + std::back_inserter(result)); return result; } @@ -2100,42 +2010,49 @@ Json::Value EthRpcMethods::DebugTraceTransaction(const std::string &txHash, } } - -Json::Value EthRpcMethods::OtterscanSearchTransactions(const std::string& address, unsigned long blockNumber, unsigned long pageSize, bool before) { +Json::Value EthRpcMethods::OtterscanSearchTransactions( + const std::string &address, unsigned long blockNumber, + unsigned long pageSize, bool before) { if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES) { - throw JsonRpcException(ServerBase::RPC_MISC_ERROR, - "The node is not configured to store otter internal operations"); + throw JsonRpcException( + ServerBase::RPC_MISC_ERROR, + "The node is not configured to store otter internal operations"); } if (!TX_TRACES) { - throw JsonRpcException(ServerBase::RPC_MISC_ERROR, - "The node is not configured to store otter internal operations"); + throw JsonRpcException( + ServerBase::RPC_MISC_ERROR, + "The node is not configured to store otter internal operations"); } - //Records whether blockNumber was 0 on input + // Records whether blockNumber was 0 on input bool blockNumberWasZero = !blockNumber; - // if blocnumber is 0 and it's a before search, then we need to get the latest block number + // if blocnumber is 0 and it's a before search, then we need to get the latest + // block number if (blockNumber == 0 && before) { - blockNumber = m_sharedMediator.m_txBlockChain.GetLastBlock().GetHeader().GetBlockNum(); + blockNumber = m_sharedMediator.m_txBlockChain.GetLastBlock() + .GetHeader() + .GetBlockNum(); } try { bool wasMore = false; - const auto res = - BlockStorage::GetBlockStorage().GetOtterTxAddressMapping(address, blockNumber, pageSize, before, wasMore); + const auto res = BlockStorage::GetBlockStorage().GetOtterTxAddressMapping( + address, blockNumber, pageSize, before, wasMore); Json::Value response = Json::objectValue; Json::Value txs = Json::arrayValue; Json::Value receipts = Json::arrayValue; - for(const auto& hash : res) { + for (const auto &hash : res) { // Get Tx result auto const txByHash = GetEthTransactionByHash(hash); auto txReceipt = GetEthTransactionReceipt(hash); // For some reason otterscan expects a timestamp in the receipts... - auto const block = GetEthBlockByNumber(txReceipt["blockNumber"].asString(), false); + auto const block = + GetEthBlockByNumber(txReceipt["blockNumber"].asString(), false); txReceipt["timestamp"] = block["timestamp"]; txs.append(txByHash); @@ -2147,8 +2064,10 @@ Json::Value EthRpcMethods::OtterscanSearchTransactions(const std::string& addres // Otterscan docs: // These are the conditions for which these variables are set to true - response["firstPage"] = (before && blockNumberWasZero) || (!before && !wasMore); - response["lastPage"] = (!before && blockNumberWasZero) || (before && !wasMore); + response["firstPage"] = + (before && blockNumberWasZero) || (!before && !wasMore); + response["lastPage"] = + (!before && blockNumberWasZero) || (before && !wasMore); return response; } catch (exception &e) { @@ -2157,20 +2076,23 @@ Json::Value EthRpcMethods::OtterscanSearchTransactions(const std::string& addres } } -Json::Value EthRpcMethods::OtterscanGetTransactionBySenderAndNonce(const std::string& address, uint64_t nonce) { +Json::Value EthRpcMethods::OtterscanGetTransactionBySenderAndNonce( + const std::string &address, uint64_t nonce) { if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES) { - throw JsonRpcException(ServerBase::RPC_MISC_ERROR, - "The node is not configured to store otter internal operations"); + throw JsonRpcException( + ServerBase::RPC_MISC_ERROR, + "The node is not configured to store otter internal operations"); } if (!TX_TRACES) { - throw JsonRpcException(ServerBase::RPC_MISC_ERROR, - "The node is not configured to store otter internal operations"); + throw JsonRpcException( + ServerBase::RPC_MISC_ERROR, + "The node is not configured to store otter internal operations"); } try { - const auto res = - BlockStorage::GetBlockStorage().GetOtterAddressNonceLookup(address, nonce); + const auto res = BlockStorage::GetBlockStorage().GetOtterAddressNonceLookup( + address, nonce); // Perhaps this should just return empty array if (res.empty()) { @@ -2185,15 +2107,18 @@ Json::Value EthRpcMethods::OtterscanGetTransactionBySenderAndNonce(const std::st } } -Json::Value EthRpcMethods::OtterscanGetInternalOperations(const std::string &txHash, const std::string &tracer) { +Json::Value EthRpcMethods::OtterscanGetInternalOperations( + const std::string &txHash, const std::string &tracer) { if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES) { - throw JsonRpcException(ServerBase::RPC_MISC_ERROR, - "The node is not configured to store otter internal operations"); + throw JsonRpcException( + ServerBase::RPC_MISC_ERROR, + "The node is not configured to store otter internal operations"); } if (!TX_TRACES) { - throw JsonRpcException(ServerBase::RPC_MISC_ERROR, - "The node is not configured to store otter internal operations"); + throw JsonRpcException( + ServerBase::RPC_MISC_ERROR, + "The node is not configured to store otter internal operations"); } std::string trace; @@ -2217,13 +2142,15 @@ Json::Value EthRpcMethods::OtterscanGetInternalOperations(const std::string &txH } Json::Value EthRpcMethods::GetHeaderByNumber(const uint64_t blockNumber) { - // Erigon headers are a subset of a full block - So just return the full block. + // Erigon headers are a subset of a full block - So just return the full + // block. return EthRpcMethods::GetEthBlockByNumber(std::to_string(blockNumber), false); } -bool EthRpcMethods::HasCode(const std::string& address, const std::string& /*block*/) { - // TODO: Respect block parameter - We can probably do this by finding the contract creation transaction and comparing - // the block numbers. +bool EthRpcMethods::HasCode(const std::string &address, + const std::string & /*block*/) { + // TODO: Respect block parameter - We can probably do this by finding the + // contract creation transaction and comparing the block numbers. Address addr{address, Address::FromHex}; unique_lock lock( AccountStore::GetInstance().GetPrimaryMutex()); @@ -2239,12 +2166,17 @@ Json::Value EthRpcMethods::GetBlockDetails(const uint64_t blockNumber) { Json::Value response; auto txBlock = m_sharedMediator.m_txBlockChain.GetBlock(blockNumber); - bool isVacuous = CommonUtils::IsVacuousEpoch(txBlock.GetHeader().GetBlockNum()); - uint128_t rewards = (isVacuous ? txBlock.GetHeader().GetRewards() * EVM_ZIL_SCALING_FACTOR : 0); - uint128_t fees = (isVacuous ? 0 : txBlock.GetHeader().GetRewards() * EVM_ZIL_SCALING_FACTOR); + bool isVacuous = + CommonUtils::IsVacuousEpoch(txBlock.GetHeader().GetBlockNum()); + uint128_t rewards = + (isVacuous ? txBlock.GetHeader().GetRewards() * EVM_ZIL_SCALING_FACTOR + : 0); + uint128_t fees = + (isVacuous ? 0 + : txBlock.GetHeader().GetRewards() * EVM_ZIL_SCALING_FACTOR); auto jsonBlock = GetEthBlockCommon(txBlock, false); - if(jsonBlock["gasLimit"].asString() == "0x0") jsonBlock["gasLimit"] = "0x1"; + if (jsonBlock["gasLimit"].asString() == "0x0") jsonBlock["gasLimit"] = "0x1"; jsonBlock.removeMember("transactions"); jsonBlock["transactionCount"] = txBlock.GetHeader().GetNumTxs(); @@ -2259,7 +2191,9 @@ Json::Value EthRpcMethods::GetBlockDetails(const uint64_t blockNumber) { return response; } -Json::Value EthRpcMethods::GetBlockTransactions(const uint64_t blockNumber, const uint32_t pageNumber, const uint32_t pageSize) { +Json::Value EthRpcMethods::GetBlockTransactions(const uint64_t blockNumber, + const uint32_t pageNumber, + const uint32_t pageSize) { Json::Value response; auto txBlock = m_sharedMediator.m_txBlockChain.GetBlock(blockNumber); @@ -2274,9 +2208,11 @@ Json::Value EthRpcMethods::GetBlockTransactions(const uint64_t blockNumber, cons std::vector receipts; for (Json::Value::ArrayIndex i = start; i < end; i++) { auto transaction = transactions[i]; - // TODO: Truncate input to 4 bytes (plus 0x) - Work out why the 0x is optional + // TODO: Truncate input to 4 bytes (plus 0x) - Work out why the 0x is + // optional - auto receipt = EthRpcMethods::GetEthTransactionReceipt(transaction["hash"].asString()); + auto receipt = + EthRpcMethods::GetEthTransactionReceipt(transaction["hash"].asString()); receipt["logs"] = Json::nullValue; receipt["logsBloom"] = Json::nullValue; receipts.push_back(receipt); @@ -2291,10 +2227,11 @@ Json::Value EthRpcMethods::GetBlockTransactions(const uint64_t blockNumber, cons return response; } -Json::Value EthRpcMethods::GetContractCreator(const std::string& address) { +Json::Value EthRpcMethods::GetContractCreator(const std::string &address) { Address addr{address, Address::FromHex}; - dev::h256 creationTxn = BlockStorage::GetBlockStorage().GetContractCreator(addr); + dev::h256 creationTxn = + BlockStorage::GetBlockStorage().GetContractCreator(addr); if (creationTxn == dev::h256()) { return Json::nullValue; @@ -2302,18 +2239,20 @@ Json::Value EthRpcMethods::GetContractCreator(const std::string& address) { TxBodySharedPtr txnBodyPtr; bool isPresent = - BlockStorage::GetBlockStorage().GetTxBody(creationTxn, txnBodyPtr); + BlockStorage::GetBlockStorage().GetTxBody(creationTxn, txnBodyPtr); if (!isPresent) { LOG_GENERAL(WARNING, "Contract creator transaction doesn't exist"); return Json::nullValue; } - const TransactionWithReceipt& txnBody = *txnBodyPtr; + const TransactionWithReceipt &txnBody = *txnBodyPtr; Json::Value response = Json::objectValue; response["hash"] = "0x" + creationTxn.hex(); // FIXME: This is wrong for deployer contracts. - // "For deployer contracts, i.e., the contract is created as a result of a method call, this corresponds to the address of the contract who created it." + // "For deployer contracts, i.e., the contract is created as a result of a + // method call, this corresponds to the address of the contract who created + // it." response["creator"] = "0x" + txnBody.GetTransaction().GetSenderAddr().hex(); return response; diff --git a/src/libServer/EthRpcMethods.h b/src/libServer/EthRpcMethods.h index d2c170c451..aa3f10e8f9 100644 --- a/src/libServer/EthRpcMethods.h +++ b/src/libServer/EthRpcMethods.h @@ -38,15 +38,6 @@ class EthRpcMethods { Z_I64METRIC m_apiCallCount{Z_FL::EVM_RPC, "ethrpc_invocation_count", "Calls to ethereum API", "Calls"}; - std::pair CheckContractTxnShards( - bool priority, unsigned int shard, const Transaction& tx, - unsigned int num_shards, bool toAccountExist, bool toAccountIsContract); - - CreateTransactionTargetFunc m_createTransactionTarget = - [this](const Transaction& tx, uint32_t shardId) -> bool { - return m_sharedMediator.m_lookup->AddToTxnShardMap(tx, shardId); - }; - void Init(LookupServer* lookupServer); virtual void GetEthCallEthI(const Json::Value& request, @@ -122,12 +113,12 @@ class EthRpcMethods { LOG_MARKER_CONTITIONAL(LOG_SC); EnsureEvmAndLookupEnabled(); std::string block_or_tag; - if (request.size()> 1) { + if (request.size() > 1) { block_or_tag = request[1u].asString(); } - response = this->GetEthEstimateGas(request[0u], - (request.size() > 1 ? &block_or_tag : nullptr)); + response = this->GetEthEstimateGas( + request[0u], (request.size() > 1 ? &block_or_tag : nullptr)); } inline virtual void GetEthTransactionCountI(const Json::Value& request, @@ -177,8 +168,7 @@ class EthRpcMethods { EVM_ZIL_SCALING_FACTOR) / GasConv::GetScalingFactor(); - response = CreateTransactionEth(fields, pubKey, shards, gasPrice, - m_createTransactionTarget); + response = CreateTransactionEth(fields, pubKey, shards, gasPrice); } inline virtual void GetEthBalanceI(const Json::Value& request, @@ -596,7 +586,7 @@ class EthRpcMethods { * @param response : none */ inline virtual void OtterscanEnableI(const Json::Value& request, - Json::Value& response) { + Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); ARCHIVAL_LOOKUP_WITH_TX_TRACES = request[0u].asBool(); } @@ -606,10 +596,11 @@ class EthRpcMethods { * @param request : transaction hash * @param response : transaction internal operations */ - inline virtual void OtterscanGetInternalOperationsI(const Json::Value& request, - Json::Value& response) { + inline virtual void OtterscanGetInternalOperationsI( + const Json::Value& request, Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); - response = this->OtterscanGetInternalOperations(request[0u].asString(), "otter_internal_tracer"); + response = this->OtterscanGetInternalOperations(request[0u].asString(), + "otter_internal_tracer"); } /** @@ -618,9 +609,10 @@ class EthRpcMethods { * @param response : transaction internal operations */ inline virtual void OtterscanGetTransactionErrorI(const Json::Value& request, - Json::Value& response) { + Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); - response = this->OtterscanGetInternalOperations(request[0u].asString(), "otter_transaction_error"); + response = this->OtterscanGetInternalOperations(request[0u].asString(), + "otter_transaction_error"); } /** @@ -629,9 +621,10 @@ class EthRpcMethods { * @param response : transaction trace (abridged) */ inline virtual void OtterscanTraceTransactionI(const Json::Value& request, - Json::Value& response) { + Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); - response = this->OtterscanGetInternalOperations(request[0u].asString(), "otter_call_tracer"); + response = this->OtterscanGetInternalOperations(request[0u].asString(), + "otter_call_tracer"); } /** @@ -639,10 +632,12 @@ class EthRpcMethods { * @param request : transaction hash * @param response : transaction trace (abridged) */ - inline virtual void OtterscanSearchTransactionsBeforeI(const Json::Value& request, - Json::Value& response) { + inline virtual void OtterscanSearchTransactionsBeforeI( + const Json::Value& request, Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); - response = this->OtterscanSearchTransactions(request[0u].asString(), request[1u].asInt64(), request[2u].asInt64(), true); + response = this->OtterscanSearchTransactions(request[0u].asString(), + request[1u].asInt64(), + request[2u].asInt64(), true); } /** @@ -650,10 +645,12 @@ class EthRpcMethods { * @param request : transaction hash * @param response : transaction trace (abridged) */ - inline virtual void OtterscanSearchTransactionsAfterI(const Json::Value& request, - Json::Value& response) { + inline virtual void OtterscanSearchTransactionsAfterI( + const Json::Value& request, Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); - response = this->OtterscanSearchTransactions(request[0u].asString(), request[1u].asInt64(), request[2u].asInt64(), false); + response = this->OtterscanSearchTransactions(request[0u].asString(), + request[1u].asInt64(), + request[2u].asInt64(), false); } /** @@ -661,10 +658,11 @@ class EthRpcMethods { * @param request : transaction hash * @param response : transaction trace (abridged) */ - inline virtual void OtterscanGetTransactionBySenderAndNonceI(const Json::Value& request, - Json::Value& response) { + inline virtual void OtterscanGetTransactionBySenderAndNonceI( + const Json::Value& request, Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); - response = this->OtterscanGetTransactionBySenderAndNonce(request[0u].asString(), request[1u].asInt64()); + response = this->OtterscanGetTransactionBySenderAndNonce( + request[0u].asString(), request[1u].asInt64()); } /** @@ -686,29 +684,34 @@ class EthRpcMethods { } inline virtual void GetOtterscanApiLevelI(const Json::Value& request, - Json::Value& response) { + Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); response = 8; } - inline virtual void HasCodeI(const Json::Value& request, Json::Value& response) { + inline virtual void HasCodeI(const Json::Value& request, + Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); response = this->HasCode(request[0u].asString(), request[1u].asString()); } inline virtual void GetBlockDetailsI(const Json::Value& request, - Json::Value& response) { + Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); response = this->GetBlockDetails(request[0u].asUInt64()); } inline virtual void GetBlockTransactionsI(const Json::Value& request, - Json::Value& response) { + Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); - response = this->GetBlockTransactions(request[0u].asUInt64(), boost::numeric_cast(request[1u].asUInt64()), boost::numeric_cast(request[2u].asUInt64())); + response = this->GetBlockTransactions( + request[0u].asUInt64(), + boost::numeric_cast(request[1u].asUInt64()), + boost::numeric_cast(request[2u].asUInt64())); } - inline virtual void GetContractCreatorI(const Json::Value& request, Json::Value& response) { + inline virtual void GetContractCreatorI(const Json::Value& request, + Json::Value& response) { LOG_MARKER_CONTITIONAL(LOG_SC); response = this->GetContractCreator(request[0u].asString()); } @@ -720,7 +723,8 @@ class EthRpcMethods { std::string DebugTraceCallEth(const Json::Value& _json, const std::string& block_or_tag, const Json::Value& tracer); - std::string GetEthEstimateGas(const Json::Value& _json, const std::string *block_or_tag); + std::string GetEthEstimateGas(const Json::Value& _json, + const std::string* block_or_tag); std::string GetEthCallImpl(const Json::Value& _json, const ApiKeys& apiKeys, std::string const& tracer = ""); Json::Value GetBalanceAndNonce(const std::string& address); @@ -762,10 +766,10 @@ class EthRpcMethods { Json::Value GetEthGasPrice() const; uint256_t GetEthGasPriceNum() const; - std::string CreateTransactionEth( - Eth::EthFields const& fields, zbytes const& pubKey, - const unsigned int num_shards, const uint128_t& gasPriceWei, - const CreateTransactionTargetFunc& targetFunc); + std::string CreateTransactionEth(Eth::EthFields const& fields, + zbytes const& pubKey, + const unsigned int num_shards, + const uint128_t& gasPriceWei); Json::Value GetEthBlockTransactionCountByHash(const std::string& blockHash); Json::Value GetEthBlockTransactionCountByNumber( @@ -791,21 +795,27 @@ class EthRpcMethods { Json::Value GetEthBlockReceipts(const std::string& blockId); Json::Value DebugTraceTransaction(const std::string& txHash, const Json::Value& json); - Json::Value OtterscanGetInternalOperations(const std::string& txHash, const std::string &tracer); - Json::Value OtterscanSearchTransactions(const std::string& address, unsigned long blockNumber, unsigned long pageSize, bool before); - Json::Value OtterscanGetTransactionBySenderAndNonce(const std::string& address, uint64_t nonce); + Json::Value OtterscanGetInternalOperations(const std::string& txHash, + const std::string& tracer); + Json::Value OtterscanSearchTransactions(const std::string& address, + unsigned long blockNumber, + unsigned long pageSize, bool before); + Json::Value OtterscanGetTransactionBySenderAndNonce( + const std::string& address, uint64_t nonce); Json::Value DebugTraceBlockByNumber(const std::string& blockNum, const Json::Value& json); Json::Value GetHeaderByNumber(const uint64_t blockNumber); bool HasCode(const std::string& address, const std::string& block); Json::Value GetBlockDetails(const uint64_t blockNumber); - Json::Value GetBlockTransactions(const uint64_t blockNumber, const uint32_t pageNumber, const uint32_t pageSize); + Json::Value GetBlockTransactions(const uint64_t blockNumber, + const uint32_t pageNumber, + const uint32_t pageSize); Json::Value GetContractCreator(const std::string& address); Json::Value GetDSLeaderTxnPool(); void EnsureEvmAndLookupEnabled(); - static bool UnpackRevert(const std::string &data_in, std::string &message); + static bool UnpackRevert(const std::string& data_in, std::string& message); public: Mediator& m_sharedMediator; diff --git a/src/libServer/LookupServer.cpp b/src/libServer/LookupServer.cpp index 7f1b0ad758..9a5775d6c1 100644 --- a/src/libServer/LookupServer.cpp +++ b/src/libServer/LookupServer.cpp @@ -319,8 +319,8 @@ LookupServer::LookupServer(Mediator& mediator, jsonrpc::JSON_STRING, NULL), &LookupServer::GetStateProofI); this->bindAndAddMethod( - jsonrpc::Procedure("GetVersion", jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_OBJECT, - NULL), + jsonrpc::Procedure("GetVersion", jsonrpc::PARAMS_BY_POSITION, + jsonrpc::JSON_OBJECT, NULL), &Server::GetVersionI); m_StartTimeTx = 0; @@ -364,14 +364,12 @@ bool LookupServer::StartCollectorThread() { auto collectorThread = [this]() mutable -> void { this_thread::sleep_for(chrono::seconds(POW_WINDOW_IN_SECONDS)); - vector txnsShard; - vector txnsDS; + vector txnsToSend; LOG_GENERAL(INFO, "[ARCHLOOK]" << "Start thread"); while (true) { this_thread::sleep_for(chrono::seconds(SEED_TXN_COLLECTION_TIME_IN_SEC)); - txnsShard.clear(); - txnsDS.clear(); + txnsToSend.clear(); if (m_mediator.m_disableTxns) { LOG_GENERAL(INFO, "Txns disabled - skipping forwarding to upper seed"); @@ -385,55 +383,15 @@ bool LookupServer::StartCollectorThread() { if (USE_REMOTE_TXN_CREATOR && !m_mediator.m_lookup->GenTxnToSend(NUM_TXN_TO_SEND_PER_ACCOUNT, - txnsShard, txnsDS)) { + txnsToSend)) { LOG_GENERAL(WARNING, "GenTxnToSend failed"); } - if (!txnsShard.empty()) { - for (const auto& tx : txnsShard) { - m_mediator.m_lookup->AddToTxnShardMap(tx, - SEND_TYPE::ARCHIVAL_SEND_SHARD); - } - LOG_GENERAL(INFO, "Size of txns to shard: " << txnsShard.size()); - } - - if (!txnsDS.empty()) { - for (const auto& tx : txnsDS) { - m_mediator.m_lookup->AddToTxnShardMap(tx, - SEND_TYPE::ARCHIVAL_SEND_DS); - } - LOG_GENERAL(INFO, "Size of txns to DS: " << txnsDS.size()); - } - - for (auto const& i : - {SEND_TYPE::ARCHIVAL_SEND_SHARD, SEND_TYPE::ARCHIVAL_SEND_DS}) { - { - lock_guard g(m_mediator.m_lookup->m_txnShardMapMutex); - if (m_mediator.m_lookup->GetTxnFromShardMap(i).empty()) { - continue; - } - } - } - - zbytes msg = {MessageType::LOOKUP, LookupInstructionType::FORWARDTXN}; + LOG_GENERAL(INFO, "Size of generated txns to DS: " << txnsToSend.size()); - { - lock_guard g(m_mediator.m_lookup->m_txnShardMapMutex); - if (!Messenger::SetForwardTxnBlockFromSeed( - msg, MessageOffset::BODY, - m_mediator.m_lookup->GetTxnFromShardMap( - SEND_TYPE::ARCHIVAL_SEND_SHARD), - m_mediator.m_lookup->GetTxnFromShardMap( - SEND_TYPE::ARCHIVAL_SEND_DS))) { - continue; - } - for (auto const& i : - {SEND_TYPE::ARCHIVAL_SEND_SHARD, SEND_TYPE::ARCHIVAL_SEND_DS}) { - m_mediator.m_lookup->DeleteTxnShardMap(i); - } + for (const auto& tx : txnsToSend) { + m_mediator.m_lookup->AddTxnToMemPool(tx); } - - m_mediator.m_lookup->SendMessageToRandomSeedNode(msg); } }; DetachedFunction(1, collectorThread); @@ -552,9 +510,8 @@ bool ValidateTxn(const Transaction& tx, const Address& fromAddr, return true; } -Json::Value LookupServer::CreateTransaction( - const Json::Value& _json, const unsigned int num_shards, - const uint128_t& gasPrice, const CreateTransactionTargetFunc& targetFunc) { +Json::Value LookupServer::CreateTransaction(const Json::Value& _json, + const uint128_t& gasPrice) { INC_CALLS(GetCallsCounter()); LOG_MARKER(); @@ -605,17 +562,8 @@ Json::Value LookupServer::CreateTransaction( toAccountIsContract = toAccountExist && toAccount->isContract(); } - const unsigned int shard = Transaction::GetShardIndex(fromAddr, num_shards); - unsigned int mapIndex = shard; - bool priority = false; - if (_json.isMember("priority")) { - priority = _json["priority"].asBool(); - } switch (Transaction::GetTransactionType(tx)) { case Transaction::ContractType::NON_CONTRACT: - if (ARCHIVAL_LOOKUP) { - mapIndex = SEND_TYPE::ARCHIVAL_SEND_SHARD; - } if (toAccountExist) { if (toAccountIsContract) { throw JsonRpcException(ServerBase::RPC_INVALID_PARAMETER, @@ -624,28 +572,17 @@ Json::Value LookupServer::CreateTransaction( } } - ret["Info"] = "Non-contract txn, sent to shard"; + ret["Info"] = "Non-contract txn, sent to Ds"; break; case Transaction::ContractType::CONTRACT_CREATION: { - // We use the same logic for CONTRACT_CREATION and CONTRACT_CALL. - // TODO(valeryz): once we stop using Zilliqa APIs for EVM, revert - // to the old behavior where CONTRACT_CREATION can be sharded. - auto check = - CheckContractTxnShards(priority, shard, tx, num_shards, - toAccountExist, toAccountIsContract); - ret["Info"] = check.first; + ret["Info"] = CheckContractTxn(tx, toAccountExist, toAccountIsContract); ret["ContractAddress"] = Account::GetAddressForContract(fromAddr, tx.GetNonce() - 1, tx.GetVersionIdentifier()) .hex(); - mapIndex = check.second; } break; case Transaction::ContractType::CONTRACT_CALL: { - auto check = - CheckContractTxnShards(priority, shard, tx, num_shards, - toAccountExist, toAccountIsContract); - ret["Info"] = check.first; - mapIndex = check.second; + ret["Info"] = CheckContractTxn(tx, toAccountExist, toAccountIsContract); } break; case Transaction::ContractType::ERROR: throw JsonRpcException(RPC_INVALID_ADDRESS_OR_KEY, @@ -654,14 +591,8 @@ Json::Value LookupServer::CreateTransaction( default: throw JsonRpcException(RPC_MISC_ERROR, "Txn type unexpected"); } - if (m_mediator.m_lookup->m_sendAllToDS) { - if (ARCHIVAL_LOOKUP) { - mapIndex = SEND_TYPE::ARCHIVAL_SEND_DS; - } else { - mapIndex = num_shards; - } - } - if (!targetFunc(tx, mapIndex)) { + + if (!m_mediator.m_lookup->AddTxnToMemPool(tx)) { throw JsonRpcException(RPC_DATABASE_ERROR, "Txn could not be added as database exceeded " "limit or the txn was already present"); @@ -981,13 +912,15 @@ Json::Value LookupServer::GetSmartContractInit(const string& address) { string initDataStr; Json::Value initDataJson; - // If the contract is EVM, represent the init data as a hex string. Otherwise, it is JSON. + // If the contract is EVM, represent the init data as a hex string. + // Otherwise, it is JSON. if (EvmUtils::isEvm(code)) { initDataStr = DataConversion::Uint8VecToHexStrRet(initData); initDataJson = initDataStr; } else { initDataStr = DataConversion::CharArrayToString(initData); - if (!JSONUtils::GetInstance().convertStrtoJson(initDataStr, initDataJson)) { + if (!JSONUtils::GetInstance().convertStrtoJson(initDataStr, + initDataJson)) { throw JsonRpcException(RPC_PARSE_ERROR, "Unable to convert initData into Json"); } @@ -1259,8 +1192,7 @@ double LookupServer::GetTransactionRate() { refBlockNum = refBlockNum - REF_BLOCK_DIFF; } - mp::cpp_dec_float_50 numTxns( - LookupServer::GetNumTransactions(refBlockNum)); + mp::cpp_dec_float_50 numTxns(LookupServer::GetNumTransactions(refBlockNum)); LOG_GENERAL(INFO, "Num Txns: " << numTxns); try { @@ -2349,17 +2281,13 @@ Json::Value LookupServer::GetStateProof(const string& address, return ret; } - -std::pair LookupServer::CheckContractTxnShards( - bool priority, unsigned int shard, const Transaction& tx, - unsigned int num_shards, bool toAccountExist, bool toAccountIsContract) { +std::string LookupServer::CheckContractTxn(const Transaction& tx, + bool toAccountExist, + bool toAccountIsContract) { TRACE(zil::trace::FilterClass::DEMO); INC_CALLS(GetCallsCounter()); - unsigned int mapIndex = shard; - std::string resultStr; - if (!ENABLE_SC) { throw JsonRpcException(ServerBase::RPC_MISC_ERROR, "Smart contract is disabled"); @@ -2376,49 +2304,10 @@ std::pair LookupServer::CheckContractTxnShards( "Non - contract address called"); } - Transaction::ContractType scType = Transaction::GetTransactionType(tx); - - // Use m_sendSCCallsToDS as initial setting - bool sendToDs = priority || m_sharedMediator.m_lookup->m_sendSCCallsToDS; - if (!tx.IsEth() && scType == Transaction::CONTRACT_CREATION) { - // Scilla smart CONTRACT_CREATION call should be executed in shard rather - // than DS. - if (ARCHIVAL_LOOKUP) { - mapIndex = SEND_TYPE::ARCHIVAL_SEND_SHARD; - } - resultStr = "Contract Creation txn, sent to shard"; - } else { - // CONTRACT_CALL - scilla and EVM , CONTRACT_CREATION - EVM - Address affectedAddress = tx.GetToAddr(); - unsigned int to_shard = - Transaction::GetShardIndex(affectedAddress, num_shards); - if ((to_shard == shard) && !sendToDs) { - if (tx.GetGasLimitZil() > SHARD_MICROBLOCK_GAS_LIMIT) { - throw JsonRpcException(ServerBase::RPC_INVALID_PARAMETER, - "txn gas limit exceeding shard maximum limit"); - } - if (ARCHIVAL_LOOKUP) { - mapIndex = SEND_TYPE::ARCHIVAL_SEND_SHARD; - } - resultStr = - "Contract Creation/Call Txn, Shards Match of the sender " - "and receiver"; - } else { - if (tx.GetGasLimitZil() > DS_MICROBLOCK_GAS_LIMIT) { - throw JsonRpcException( - ServerBase::RPC_INVALID_PARAMETER, - (boost::format( - "txn gas limit exceeding ds maximum limit! Tx: %i DS: %i") % - tx.GetGasLimitZil() % DS_MICROBLOCK_GAS_LIMIT) - .str()); - } - if (ARCHIVAL_LOOKUP) { - mapIndex = SEND_TYPE::ARCHIVAL_SEND_DS; - } else { - mapIndex = num_shards; - } - resultStr = "Contract Creation/Call Txn, Sent To Ds"; - } + if (tx.GetGasLimitZil() > DS_MICROBLOCK_GAS_LIMIT) { + throw JsonRpcException(ServerBase::RPC_INVALID_PARAMETER, + "txn gas limit exceeding shard maximum limit"); } - return make_pair(resultStr, mapIndex); + + return "Contract Creation/Call Txn, Sent To Ds"; } diff --git a/src/libServer/LookupServer.h b/src/libServer/LookupServer.h index 73bb48e233..cdf5b68290 100644 --- a/src/libServer/LookupServer.h +++ b/src/libServer/LookupServer.h @@ -51,17 +51,11 @@ class LookupServer : public Server, Z_I64METRIC m_callCount{Z_FL::API_SERVER, "lookup_invocation_count", "Calls to Lookup Server", "Calls"}; - CreateTransactionTargetFunc m_createTransactionTarget = - [this](const Transaction& tx, uint32_t shardId) -> bool { - return m_mediator.m_lookup->AddToTxnShardMap(tx, shardId); - }; - Json::Value GetTransactionsForTxBlock(const std::string& txBlockNum, const std::string& pageNumber); - std::pair CheckContractTxnShards( - bool priority, unsigned int shard, const Transaction& tx, - unsigned int num_shards, bool toAccountExist, bool toAccountIsContract); + std::string CheckContractTxn(const Transaction& tx, bool toAccountExist, + bool toAccountIsContract); mp::cpp_dec_float_50 CalculateTotalSupply(); public: @@ -82,9 +76,8 @@ class LookupServer : public Server, inline virtual void CreateTransactionI(const Json::Value& request, Json::Value& response) { response = CreateTransaction( - request[0u], m_mediator.m_lookup->GetShardPeers().size(), - m_mediator.m_dsBlockChain.GetLastBlock().GetHeader().GetGasPrice(), - m_createTransactionTarget); + request[0u], + m_mediator.m_dsBlockChain.GetLastBlock().GetHeader().GetGasPrice()); } inline virtual void GetTransactionI(const Json::Value& request, @@ -296,7 +289,7 @@ class LookupServer : public Server, } inline virtual void GetTotalCoinSupplyAsIntI(const Json::Value& request, - Json::Value& response) { + Json::Value& response) { (void)request; static_assert(sizeof(unsigned long) <= sizeof(Json::UInt64)); response = static_cast(this->GetTotalCoinSupplyAsInt()); @@ -325,9 +318,7 @@ class LookupServer : public Server, } std::string GetNetworkId(); Json::Value CreateTransaction(const Json::Value& _json, - const unsigned int num_shards, - const uint128_t& gasPrice, - const CreateTransactionTargetFunc& targetFunc); + const uint128_t& gasPrice); Json::Value GetTransaction(const std::string& transactionHash); Json::Value GetSoftConfirmedTransaction(const std::string& txnHash); diff --git a/src/libUtils/GetTxnFromFile.h b/src/libUtils/GetTxnFromFile.h index 30233df0bf..4924b8c588 100644 --- a/src/libUtils/GetTxnFromFile.h +++ b/src/libUtils/GetTxnFromFile.h @@ -98,7 +98,6 @@ class GetTxnFromFile { const auto num_txn = NUM_TXN_TO_SEND_PER_ACCOUNT; std::fstream file; - txns.clear(); if (totalNum > num_txn) { LOG_GENERAL(WARNING, "A single file is holding too many txns (" diff --git a/src/libValidator/Validator.cpp b/src/libValidator/Validator.cpp index d40132c27c..4347dabd7b 100644 --- a/src/libValidator/Validator.cpp +++ b/src/libValidator/Validator.cpp @@ -137,8 +137,6 @@ bool Validator::CheckCreatedTransactionFromLookup(const Transaction& tx, // Check if from account is sharded here const Address fromAddr = tx.GetSenderAddr(); - unsigned int shardId = m_mediator.m_node->GetShardId(); - unsigned int numShards = m_mediator.m_node->getNumShards(); if (tx.GetGasLimitZil() > (m_mediator.m_ds->m_mode == DirectoryService::Mode::IDLE @@ -156,37 +154,6 @@ bool Validator::CheckCreatedTransactionFromLookup(const Transaction& tx, return false; } - if (m_mediator.m_ds->m_mode == DirectoryService::Mode::IDLE) { - unsigned int correct_shard_from = - Transaction::GetShardIndex(fromAddr, numShards); - if (correct_shard_from != shardId) { - LOG_EPOCH(WARNING, m_mediator.m_currentEpochNum, - "This tx is not sharded to me!" - << " From Account = 0x" << fromAddr - << " Correct shard = " << correct_shard_from - << " This shard = " << m_mediator.m_node->GetShardId()); - error_code = TxnStatus::INCORRECT_SHARD; - return false; - // // Transaction created from the GenTransactionBulk will be rejected - // // by all shards but one. Next line is commented to avoid this - // return false; - } - - if (Transaction::GetTransactionType(tx) == Transaction::CONTRACT_CALL) { - unsigned int correct_shard_to = - Transaction::GetShardIndex(tx.GetToAddr(), numShards); - if (correct_shard_to != correct_shard_from) { - LOG_EPOCH(WARNING, m_mediator.m_currentEpochNum, - "The fromShard " << correct_shard_from << " and toShard " - << correct_shard_to - << " is different for the call SC txn"); - // Already checked at lookup - error_code = TxnStatus::CONTRACT_CALL_WRONG_SHARD; - return false; - } - } - } - if (tx.GetCode().size() > MAX_CODE_SIZE_IN_BYTES) { LOG_EPOCH(WARNING, m_mediator.m_currentEpochNum, "Code size " << tx.GetCode().size() diff --git a/src/libZilliqa/Zilliqa.cpp b/src/libZilliqa/Zilliqa.cpp index 3e87129a54..52b488f1cd 100644 --- a/src/libZilliqa/Zilliqa.cpp +++ b/src/libZilliqa/Zilliqa.cpp @@ -182,7 +182,8 @@ void Zilliqa::ProcessMessage(Zilliqa::Msg &message) { auto span = zil::trace::Tracing::CreateChildSpanOfRemoteTrace( zil::trace::FilterClass::NODE, "Dispatch", message->traceContext); #endif - + LOG_GENERAL(WARNING, "BZ Executing message with type: " + << std::string(MsgTypeToStr(msg_type))); bool result = msg_handlers[msg_type]->Execute( message->msg, MessageOffset::INST, message->from, message->startByte); diff --git a/tests/Lookup/Test_txn_send.cpp b/tests/Lookup/Test_txn_send.cpp index 0e211b640f..8208c85a9b 100644 --- a/tests/Lookup/Test_txn_send.cpp +++ b/tests/Lookup/Test_txn_send.cpp @@ -45,7 +45,7 @@ void test_transaction(const map>& mp, const unsigned newShardNum, Lookup& lk) { for (const auto& shard : mp) { for (const auto& tx : shard.second) { - lk.AddToTxnShardMap(tx, shard.first); + lk.AddTxnToMemPool(tx, shard.first); } } lk.RectifyTxnShardMap(oldNumShard, newShardNum); @@ -69,7 +69,7 @@ void test_transaction(const map>& mp, << k << " and actual index " << index << " does not match"); } - lk.DeleteTxnShardMap(k); + lk.ClearTxnMemPool(k); } }