[master] ZIL-5447: Implement header hash customisation for remote min…

…ers (#3857)

* ZIL-5447: Implement header hash customisation for remote miners

* ZIL-5447: Fix out-of-bounds iterator access

src/common/Constants.cpp

@@ -509,6 +509,7 @@ const bool OPENCL_GPU_MINE{ReadConstantString("OPENCL_GPU_MINE", "node.pow.") ==
                            "true"};
 const bool REMOTE_MINE{ReadConstantString("REMOTE_MINE", "node.pow.") ==
                        "true"};
+const bool REMOTE_MINE_EXTRA_DATA{ReadConstantString("REMOTE_MINE_EXTRA_DATA", "node.pow.") == "true"};
 const std::string MINING_PROXY_URL{
     ReadConstantString("MINING_PROXY_URL", "node.pow.")};
 const unsigned int MINING_PROXY_TIMEOUT_IN_MS{

src/common/Constants.h

@@ -355,6 +355,7 @@ extern const unsigned int RECONNECT_INTERVAL_IN_MS;
 extern const bool FULL_DATASET_MINE;
 extern const bool OPENCL_GPU_MINE;
 extern const bool REMOTE_MINE;
+extern const bool REMOTE_MINE_EXTRA_DATA;
 extern const std::string MINING_PROXY_URL;
 extern const unsigned int MINING_PROXY_TIMEOUT_IN_MS;
 extern const unsigned int MAX_RETRY_SEND_POW_TIME;

src/libData/MiningData/DSPowSolution.cpp

@@ -30,6 +30,7 @@ DSPowSolution::DSPowSolution(const DSPowSolution& src)
       m_nonce(src.m_nonce),
       m_resultingHash(src.m_resultingHash),
       m_mixHash(src.m_mixHash),
+      m_extraData(src.m_extraData),
       m_lookupId(src.m_lookupId),
       m_gasPrice(src.m_gasPrice),
       m_govProposal(src.m_govProposal),
@@ -39,7 +40,7 @@ DSPowSolution::DSPowSolution(
     const uint64_t& blockNumberInput, const uint8_t& difficultyLevelInput,
     const Peer& submitterPeerInput, const PubKey& submitterKeyInput,
     const uint64_t& nonceInput, const std::string& resultingHashInput,
-    const std::string& mixHashInput, const uint32_t& lookupIdInput,
+    const std::string& mixHashInput, const zbytes& extraDataInput, const uint32_t& lookupIdInput,
     const uint128_t& gasPriceInput,
     const std::pair<uint32_t, uint32_t>& govProposalInput,
     const Signature& signatureInput)
@@ -50,6 +51,7 @@ DSPowSolution::DSPowSolution(
       m_nonce(nonceInput),
       m_resultingHash(resultingHashInput),
       m_mixHash(mixHashInput),
+      m_extraData(extraDataInput),
       m_lookupId(lookupIdInput),
       m_gasPrice(gasPriceInput),
       m_govProposal(govProposalInput),
@@ -80,6 +82,8 @@ const std::string& DSPowSolution::GetResultingHash() const {
 /// Returns mix hash
 const std::string& DSPowSolution::GetMixHash() const { return m_mixHash; }
 
+const zbytes& DSPowSolution::GetExtraData() const { return m_extraData; }
+
 /// Returns lookupid
 const uint32_t& DSPowSolution::GetLookupId() const { return m_lookupId; }
 
@@ -100,7 +104,9 @@ bool DSPowSolution::operator==(const DSPowSolution& sol) const {
           (m_submitterPeer == sol.m_submitterPeer) &&
           (m_submitterKey == sol.m_submitterKey) && (m_nonce == sol.m_nonce) &&
           (m_resultingHash == sol.m_resultingHash) &&
-          (m_mixHash == sol.m_mixHash) && (m_lookupId == sol.m_lookupId) &&
+          (m_mixHash == sol.m_mixHash) &&
+          (m_extraData == sol.m_extraData) &&
+          (m_lookupId == sol.m_lookupId) &&
           (m_gasPrice == sol.m_gasPrice) &&
           (m_govProposal == sol.m_govProposal) &&
           (m_signature == sol.m_signature));
@@ -114,6 +120,7 @@ DSPowSolution& DSPowSolution::operator=(const DSPowSolution& src) {
   m_nonce = src.m_nonce;
   m_resultingHash = src.m_resultingHash;
   m_mixHash = src.m_mixHash;
+  m_extraData = src.m_extraData;
   m_lookupId = src.m_lookupId;
   m_gasPrice = src.m_gasPrice;
   m_govProposal = src.m_govProposal;

src/libData/MiningData/DSPowSolution.h

@@ -31,6 +31,7 @@ class DSPowSolution {
   uint64_t m_nonce{};
   std::string m_resultingHash;
   std::string m_mixHash;
+  zbytes m_extraData;
   uint32_t m_lookupId{};
   uint128_t m_gasPrice;
   GovProposalIdVotePair
@@ -50,7 +51,7 @@ class DSPowSolution {
                 const Peer& submitterPeerInput, const PubKey& submitterKeyInput,
                 const uint64_t& nonceInput,
                 const std::string& resultingHashInput,
-                const std::string& mixHashInput, const uint32_t& lookupIdInput,
+                const std::string& mixHashInput, const zbytes& extraDataInput, const uint32_t& lookupIdInput,
                 const uint128_t& gasPriceInput,
                 const std::pair<uint32_t, uint32_t>& govProposalInput,
                 const Signature& signatureInput);
@@ -79,6 +80,8 @@ class DSPowSolution {
   /// Returns mix hash
   const std::string& GetMixHash() const;
 
+  const zbytes& GetExtraData() const;
+
   /// Returns lookupid
   const uint32_t& GetLookupId() const;
 

src/libDirectoryService/DSBlockPreProcessing.cpp

@@ -231,7 +231,8 @@ bool DirectoryService::VerifyPoWWinner(
 
         auto headerHash = POW::GenHeaderHash(
             m_mediator.m_dsBlockRand, m_mediator.m_txBlockRand, peer,
-            DSPowWinner.first, dsPowSoln.m_lookupId, dsPowSoln.m_gasPrice);
+            DSPowWinner.first, dsPowSoln.m_lookupId, dsPowSoln.m_gasPrice,
+            dsPowSoln.m_extraData);
 
         string resultStr, mixHashStr;
         if (!DataConversion::charArrToHexStr(dsPowSoln.m_result, resultStr)) {
@@ -521,7 +522,7 @@ bool DirectoryService::VerifyPoWFromLeader(const Peer& peer,
                                            const PoWSolution& powSoln) {
   auto headerHash =
       POW::GenHeaderHash(m_mediator.m_dsBlockRand, m_mediator.m_txBlockRand,
-                         peer, pubKey, powSoln.m_lookupId, powSoln.m_gasPrice);
+                         peer, pubKey, powSoln.m_lookupId, powSoln.m_gasPrice, powSoln.m_extraData);
 
   auto difficulty =
       (GUARD_MODE && Guard::GetInstance().IsNodeInShardGuardList(pubKey))

src/libDirectoryService/DirectoryService.h

@@ -36,30 +36,34 @@ struct PoWSolution {
   uint32_t m_lookupId;
   uint128_t m_gasPrice;
   GovProposalIdVotePair m_govProposal;  // proposal id and vote value pair
+  zbytes m_extraData;
 
   PoWSolution()
       : m_nonce(0),
         m_result({{0}}),
         m_mixhash({{0}}),
         m_lookupId(uint32_t() - 1),
         m_gasPrice(0),
-        m_govProposal({0, 0}) {
+        m_govProposal({0, 0}),
+        m_extraData() {
 
   }  // The oldest DS (and now new shard node) will have this default value
   PoWSolution(const uint64_t n, const std::array<unsigned char, 32>& r,
               const std::array<unsigned char, 32>& m, uint32_t l,
-              const uint128_t& gp, const std::pair<uint32_t, uint32_t>& gvp)
+              const uint128_t& gp, const std::pair<uint32_t, uint32_t>& gvp,
+              const zbytes& extraData)
       : m_nonce(n),
         m_result(r),
         m_mixhash(m),
         m_lookupId(l),
         m_gasPrice(gp),
-        m_govProposal(gvp) {}
+        m_govProposal(gvp),
+        m_extraData(extraData) {}
   bool operator==(const PoWSolution& rhs) const {
     return std::tie(m_nonce, m_result, m_mixhash, m_lookupId, m_gasPrice,
-                    m_govProposal) ==
+                    m_govProposal, m_extraData) ==
            std::tie(rhs.m_nonce, rhs.m_result, rhs.m_mixhash, rhs.m_lookupId,
-                    rhs.m_gasPrice, rhs.m_govProposal);
+                    rhs.m_gasPrice, rhs.m_govProposal, rhs.m_extraData);
   }
 };
 

src/libDirectoryService/PoWProcessing.cpp

@@ -165,6 +165,7 @@ bool DirectoryService::ProcessPoWSubmission(
   uint64_t nonce;
   std::string resultingHash;
   std::string mixHash;
+  zbytes extraData;
   uint32_t lookupId;
   uint128_t gasPrice;
   Signature signature;
@@ -173,7 +174,7 @@ bool DirectoryService::ProcessPoWSubmission(
   string version;
   if (!Messenger::GetDSPoWSubmission(
           message, offset, blockNumber, difficultyLevel, submitterPeer,
-          submitterKey, nonce, resultingHash, mixHash, signature, lookupId,
+          submitterKey, nonce, resultingHash, mixHash, extraData, signature, lookupId,
           gasPrice, govProposalId, govVoteValue, version)) {
     LOG_EPOCH(WARNING, m_mediator.m_currentEpochNum,
               "DirectoryService::ProcessPowSubmission failed.");
@@ -231,7 +232,7 @@ bool DirectoryService::ProcessPoWSubmission(
   }
 
   DSPowSolution powSoln(blockNumber, difficultyLevel, submitterPeer,
-                        submitterKey, nonce, resultingHash, mixHash, lookupId,
+                        submitterKey, nonce, resultingHash, mixHash, extraData, lookupId,
                         gasPrice, std::make_pair(govProposalId, govVoteValue),
                         signature);
 
@@ -287,6 +288,7 @@ bool DirectoryService::VerifyPoWSubmission(const DSPowSolution& sol) {
   uint64_t nonce = sol.GetNonce();
   const string& resultingHash = sol.GetResultingHash();
   const string& mixHash = sol.GetMixHash();
+  const zbytes& extraData = sol.GetExtraData();
   uint32_t lookupId = sol.GetLookupId();
   const uint128_t& gasPrice = sol.GetGasPrice();
   const uint32_t& govProposalId = sol.GetGovProposalId();
@@ -382,7 +384,7 @@ bool DirectoryService::VerifyPoWSubmission(const DSPowSolution& sol) {
   // m_timespec = r_timer_start();
 
   auto headerHash = POW::GenHeaderHash(rand1, rand2, submitterPeer,
-                                       submitterPubKey, lookupId, gasPrice);
+                                       submitterPubKey, lookupId, gasPrice, extraData);
   bool result = POW::GetInstance().PoWVerify(
       blockNumber, difficultyLevel, headerHash, nonce, resultingHash, mixHash);
 
@@ -403,7 +405,7 @@ bool DirectoryService::VerifyPoWSubmission(const DSPowSolution& sol) {
       DataConversion::HexStrToStdArray(resultingHash, resultingHashArr);
       DataConversion::HexStrToStdArray(mixHash, mixHashArr);
       PoWSolution soln(nonce, resultingHashArr, mixHashArr, lookupId, gasPrice,
-                       std::make_pair(govProposalId, govVoteValue));
+                       std::make_pair(govProposalId, govVoteValue), extraData);
 
       m_allPoWConns.emplace(submitterPubKey, submitterPeer);
       if (m_allPoWs.find(submitterPubKey) == m_allPoWs.end()) {

src/libMessage/Messenger.cpp

@@ -704,6 +704,8 @@ void AnnouncementShardingStructureToProtobuf(
       proto_soln->mutable_govdata()->set_votevalue(
           soln->second.m_govProposal.second);
     }
+    proto_soln->set_extradata(soln->second.m_extraData.data(),
+                              soln->second.m_extraData.size());
   }
 }
 
@@ -715,6 +717,7 @@ bool ProtobufToShardingStructureAnnouncement(
   uint128_t gasPrice;
   uint32_t govProposalId{};
   uint32_t govVoteValue{};
+  zbytes extraData;
   for (const auto& proto_shard : protoShardingStructure.shards()) {
     for (const auto& proto_member : proto_shard.members()) {
       PubKey key;
@@ -741,10 +744,16 @@ bool ProtobufToShardingStructureAnnouncement(
         govProposalId = proto_member.powsoln().govdata().proposalid();
         govVoteValue = proto_member.powsoln().govdata().votevalue();
       }
+      if (proto_member.powsoln().extradata().size() > 32) {
+        LOG_GENERAL(WARNING, "extra data is too large");
+        return false;
+      }
+      zbytes extraData(proto_member.powsoln().extradata().begin(), proto_member.powsoln().extradata().end());
       allPoWs.emplace(
           key, PoWSolution(proto_member.powsoln().nonce(), result, mixhash,
                            proto_member.powsoln().lookupid(), gasPrice,
-                           std::make_pair(govProposalId, govVoteValue)));
+                           std::make_pair(govProposalId, govVoteValue),
+                           extraData));
     }
   }
 
@@ -1034,6 +1043,7 @@ void DSPowSolutionToProtobuf(const DSPowSolution& powSolution,
   dsPowSubmission.mutable_data()->set_resultinghash(
       powSolution.GetResultingHash());
   dsPowSubmission.mutable_data()->set_mixhash(powSolution.GetMixHash());
+  dsPowSubmission.mutable_data()->set_extradata(powSolution.GetExtraData().data(), powSolution.GetExtraData().size());
   dsPowSubmission.mutable_data()->set_lookupid(powSolution.GetLookupId());
   if (dsPowSubmission.mutable_data()->govdata().IsInitialized()) {
     dsPowSubmission.mutable_data()->mutable_govdata()->set_proposalid(
@@ -1063,6 +1073,11 @@ bool ProtobufToDSPowSolution(const DSPoWSubmission& dsPowSubmission,
   const uint64_t& nonce = dsPowSubmission.data().nonce();
   const std::string& resultingHash = dsPowSubmission.data().resultinghash();
   const std::string& mixHash = dsPowSubmission.data().mixhash();
+  if (dsPowSubmission.data().extradata().size() > 32) {
+    LOG_GENERAL(WARNING, "extra data is too large");
+    return false;
+  }
+  zbytes extraData(dsPowSubmission.data().extradata().begin(), dsPowSubmission.data().extradata().end());
   const uint32_t& lookupId = dsPowSubmission.data().lookupid();
   uint128_t gasPrice;
   ProtobufByteArrayToNumber<uint128_t, UINT128_SIZE>(
@@ -1074,7 +1089,7 @@ bool ProtobufToDSPowSolution(const DSPoWSubmission& dsPowSubmission,
   const uint32_t& govVoteValue = dsPowSubmission.data().govdata().votevalue();
 
   DSPowSolution result(blockNumber, difficultyLevel, submitterPeer,
-                       submitterKey, nonce, resultingHash, mixHash, lookupId,
+                       submitterKey, nonce, resultingHash, mixHash, extraData, lookupId,
                        gasPrice, std::make_pair(govProposalId, govVoteValue),
                        signature);
   powSolution = result;
@@ -2447,7 +2462,7 @@ bool Messenger::SetDSPoWSubmission(
     zbytes& dst, const unsigned int offset, const uint64_t blockNumber,
     const uint8_t difficultyLevel, const Peer& submitterPeer,
     const PairOfKey& submitterKey, const uint64_t nonce,
-    const string& resultingHash, const string& mixHash,
+    const string& resultingHash, const string& mixHash, const zbytes& extraData,
     const uint32_t& lookupId, const uint128_t& gasPrice,
     const GovProposalIdVotePair& govProposal, const string& version) {
   DSPoWSubmission result;
@@ -2463,6 +2478,7 @@ bool Messenger::SetDSPoWSubmission(
   result.mutable_data()->set_nonce(nonce);
   result.mutable_data()->set_resultinghash(resultingHash);
   result.mutable_data()->set_mixhash(mixHash);
+  result.mutable_data()->set_extradata(extraData.data(), extraData.size());
   result.mutable_data()->set_lookupid(lookupId);
 
   NumberToProtobufByteArray<uint128_t, UINT128_SIZE>(
@@ -2508,7 +2524,7 @@ bool Messenger::SetDSPoWSubmission(
 bool Messenger::GetDSPoWSubmission(
     const zbytes& src, const unsigned int offset, uint64_t& blockNumber,
     uint8_t& difficultyLevel, Peer& submitterPeer, PubKey& submitterPubKey,
-    uint64_t& nonce, string& resultingHash, string& mixHash,
+    uint64_t& nonce, string& resultingHash, string& mixHash, zbytes& extraData,
     Signature& signature, uint32_t& lookupId, uint128_t& gasPrice,
     uint32_t& govProposalId, uint32_t& govVoteValue, string& version) {
   if (offset >= src.size()) {
@@ -2534,6 +2550,12 @@ bool Messenger::GetDSPoWSubmission(
   nonce = result.data().nonce();
   resultingHash = result.data().resultinghash();
   mixHash = result.data().mixhash();
+  if (result.data().extradata().size() > 32) {
+    LOG_GENERAL(WARNING, "extra data is too large");
+    return false;
+  }
+  extraData.resize(result.data().extradata().size());
+  std::copy(result.data().extradata().begin(), result.data().extradata().end(), extraData.begin());
   lookupId = result.data().lookupid();
   PROTOBUFBYTEARRAYTOSERIALIZABLE(result.signature(), signature);
 
@@ -2745,6 +2767,7 @@ bool Messenger::SetDSDSBlockAnnouncement(
         soln.m_gasPrice, *proto_soln->mutable_gasprice());
     proto_soln->mutable_govdata()->set_proposalid(soln.m_govProposal.first);
     proto_soln->mutable_govdata()->set_votevalue(soln.m_govProposal.second);
+    proto_soln->set_extradata(soln.m_extraData.data(), soln.m_extraData.size());
   }
 
   if (!dsblock->IsInitialized()) {
@@ -2854,10 +2877,15 @@ bool Messenger::GetDSDSBlockAnnouncement(
       govProposalId = protoDSWinnerPoW.powsoln().govdata().proposalid();
       govVoteValue = protoDSWinnerPoW.powsoln().govdata().votevalue();
     }
+    if (protoDSWinnerPoW.powsoln().extradata().size() > 32) {
+      LOG_GENERAL(WARNING, "extra data is too large");
+      return false;
+    }
+    zbytes extraData(protoDSWinnerPoW.powsoln().extradata().begin(), protoDSWinnerPoW.powsoln().extradata().end());
     dsWinnerPoWs.emplace(
         key, PoWSolution(protoDSWinnerPoW.powsoln().nonce(), result, mixhash,
                          protoDSWinnerPoW.powsoln().lookupid(), gasPrice,
-                         std::make_pair(govProposalId, govVoteValue)));
+                         std::make_pair(govProposalId, govVoteValue), extraData));
   }
 
   // Get the part of the announcement that should be co-signed during the first

src/libMessage/Messenger.h

@@ -194,14 +194,14 @@ class Messenger {
       zbytes& dst, const unsigned int offset, const uint64_t blockNumber,
       const uint8_t difficultyLevel, const Peer& submitterPeer,
       const PairOfKey& submitterKey, const uint64_t nonce,
-      const std::string& resultingHash, const std::string& mixHash,
+      const std::string& resultingHash, const std::string& mixHash, const zbytes& extraData,
       const uint32_t& lookupId, const uint128_t& gasPrice,
       const GovProposalIdVotePair& govProposal, const std::string& version);
 
   static bool GetDSPoWSubmission(
       const zbytes& src, const unsigned int offset, uint64_t& blockNumber,
       uint8_t& difficultyLevel, Peer& submitterPeer, PubKey& submitterPubKey,
-      uint64_t& nonce, std::string& resultingHash, std::string& mixHash,
+      uint64_t& nonce, std::string& resultingHash, std::string& mixHash, zbytes& extraData,
       Signature& signature, uint32_t& lookupId, uint128_t& gasPrice,
       uint32_t& proposalId, uint32_t& voteValue, std::string& version);
 

src/libMessage/ZilliqaMessage.proto

@@ -356,6 +356,7 @@ message ProtoPoWSolution
     oneof oneof4 { uint32 lookupid     = 4; }
     ByteArray gasprice  = 5;
     GovernanceData govdata  = 6;
+    bytes extradata     = 7;
 }
 
 message ProtoCommittee
@@ -516,6 +517,7 @@ message DSPoWSubmission
         ByteArray gasprice          = 9;
         GovernanceData govdata      = 10;
         string version              = 11;
+        bytes extraData             = 12;
     }
     Data data                       = 1;
     ByteArray signature             = 2;

src/libNode/Node.h

@@ -625,6 +625,7 @@ class Node : public Executable {
                              const uint64_t winningNonce,
                              const std::string& powResultHash,
                              const std::string& powMixhash,
+                             const zbytes& extraData,
                              const uint32_t& lookupId,
                              const uint128_t& gasPrice);