Implement integration tests for DKG error handling (dashpay#2905)

* Allow modifying simulate DKG error rates via RPC

* Don't lie to yourself :)

* Add some missing new-lines in LogPrintf calls

* More fine grained control over which messages to expect in mine_quorum

* Implement llmq-dkgerrors.py integration tests

These test DKG errors and malicious behavior.

qa/pull-tester/rpc-tests.py

@@ -48,6 +48,7 @@
     'llmq-chainlocks.py', # NOTE: needs dash_hash to pass
     'llmq-simplepose.py', # NOTE: needs dash_hash to pass
     'llmq-is-cl-conflicts.py', # NOTE: needs dash_hash to pass
+    'llmq-dkgerrors.py', # NOTE: needs dash_hash to pass
     'dip4-coinbasemerkleroots.py', # NOTE: needs dash_hash to pass
     # vv Tests less than 60s vv
     'sendheaders.py', # NOTE: needs dash_hash to pass

qa/rpc-tests/llmq-dkgerrors.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015-2018 The Dash Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+from test_framework.test_framework import DashTestFramework
+from test_framework.util import *
+
+'''
+llmq-dkgerrors.py
+
+Simulate and check DKG errors
+
+'''
+
+class LLMQDKGErrors(DashTestFramework):
+    def __init__(self):
+        super().__init__(6, 5, [], fast_dip3_enforcement=True)
+
+    def run_test(self):
+
+        while self.nodes[0].getblockchaininfo()["bip9_softforks"]["dip0008"]["status"] != "active":
+            self.nodes[0].generate(10)
+        sync_blocks(self.nodes, timeout=60*5)
+
+        self.nodes[0].spork("SPORK_17_QUORUM_DKG_ENABLED", 0)
+        self.wait_for_sporks_same()
+
+        # Mine one quorum without simulating any errors
+        qh = self.mine_quorum()
+        self.assert_member_valid(qh, self.mninfo[0].proTxHash, True)
+
+        # Lets omit the contribution
+        self.mninfo[0].node.quorum('dkgsimerror', 'contribution-omit', '1')
+        qh = self.mine_quorum(expected_contributions=4)
+        self.assert_member_valid(qh, self.mninfo[0].proTxHash, False)
+
+        # Lets lie in the contribution but provide a correct justification
+        self.mninfo[0].node.quorum('dkgsimerror', 'contribution-omit', '0')
+        self.mninfo[0].node.quorum('dkgsimerror', 'contribution-lie', '1')
+        qh = self.mine_quorum(expected_contributions=5, expected_complaints=4, expected_justifications=1)
+        self.assert_member_valid(qh, self.mninfo[0].proTxHash, True)
+
+        # Lets lie in the contribution and then omit the justification
+        self.mninfo[0].node.quorum('dkgsimerror', 'justify-omit', '1')
+        qh = self.mine_quorum(expected_contributions=4, expected_complaints=4)
+        self.assert_member_valid(qh, self.mninfo[0].proTxHash, False)
+
+        # Heal some damage (don't get PoSe banned)
+        self.heal_masternodes(33)
+
+        # Lets lie in the contribution and then also lie in the justification
+        self.mninfo[0].node.quorum('dkgsimerror', 'justify-omit', '0')
+        self.mninfo[0].node.quorum('dkgsimerror', 'justify-lie', '1')
+        qh = self.mine_quorum(expected_contributions=4, expected_complaints=4, expected_justifications=1)
+        self.assert_member_valid(qh, self.mninfo[0].proTxHash, False)
+
+        # Lets lie about another MN
+        self.mninfo[0].node.quorum('dkgsimerror', 'contribution-lie', '0')
+        self.mninfo[0].node.quorum('dkgsimerror', 'justify-lie', '0')
+        self.mninfo[0].node.quorum('dkgsimerror', 'complain-lie', '1')
+        qh = self.mine_quorum(expected_contributions=5, expected_complaints=1, expected_justifications=4)
+        self.assert_member_valid(qh, self.mninfo[0].proTxHash, True)
+
+        # Lets omit 2 premature commitments
+        self.mninfo[0].node.quorum('dkgsimerror', 'complain-lie', '0')
+        self.mninfo[0].node.quorum('dkgsimerror', 'commit-omit', '1')
+        self.mninfo[1].node.quorum('dkgsimerror', 'commit-omit', '1')
+        qh = self.mine_quorum(expected_contributions=5, expected_complaints=0, expected_justifications=0, expected_commitments=3)
+        self.assert_member_valid(qh, self.mninfo[0].proTxHash, True)
+
+        # Lets lie in 2 premature commitments
+        self.mninfo[0].node.quorum('dkgsimerror', 'commit-omit', '0')
+        self.mninfo[1].node.quorum('dkgsimerror', 'commit-omit', '0')
+        self.mninfo[0].node.quorum('dkgsimerror', 'commit-lie', '1')
+        self.mninfo[1].node.quorum('dkgsimerror', 'commit-lie', '1')
+        qh = self.mine_quorum(expected_contributions=5, expected_complaints=0, expected_justifications=0, expected_commitments=3)
+        self.assert_member_valid(qh, self.mninfo[0].proTxHash, True)
+
+    def assert_member_valid(self, quorumHash, proTxHash, expectedValid):
+        q = self.nodes[0].quorum('info', 100, quorumHash, True)
+        for m in q['members']:
+            if m['proTxHash'] == proTxHash:
+                if expectedValid:
+                    assert(m['valid'])
+                else:
+                    assert(not m['valid'])
+            else:
+                assert(m['valid'])
+
+    def heal_masternodes(self, blockCount):
+        # We're not testing PoSe here, so lets heal the MNs :)
+        self.nodes[0].spork("SPORK_17_QUORUM_DKG_ENABLED", 4070908800)
+        self.wait_for_sporks_same()
+        for i in range(blockCount):
+            set_mocktime(get_mocktime() + 1)
+            set_node_times(self.nodes, get_mocktime())
+            self.nodes[0].generate(1)
+        self.sync_all()
+        self.nodes[0].spork("SPORK_17_QUORUM_DKG_ENABLED", 0)
+        self.wait_for_sporks_same()
+
+
+if __name__ == '__main__':
+    LLMQDKGErrors().main()

qa/rpc-tests/llmq-simplepose.py

@@ -38,7 +38,7 @@ def run_test(self):
 
             t = time()
             while (not self.check_punished(mn) or not self.check_banned(mn)) and (time() - t) < 120:
-                self.mine_quorum(expected_valid_count=i-1)
+                self.mine_quorum(expected_contributions=i-1, expected_complaints=i-1, expected_commitments=i-1)
 
             assert(self.check_punished(mn) and self.check_banned(mn))
 

qa/rpc-tests/test_framework/test_framework.py

@@ -622,7 +622,7 @@ def wait_for_quorum_commitment(self, timeout = 15):
             sleep(0.1)
         raise AssertionError("wait_for_quorum_commitment timed out")
 
-    def mine_quorum(self, expected_valid_count=5):
+    def mine_quorum(self, expected_contributions=5, expected_complaints=0, expected_justifications=0, expected_commitments=5):
         quorums = self.nodes[0].quorum("list")
 
         # move forward to next DKG
@@ -643,28 +643,28 @@ def mine_quorum(self, expected_valid_count=5):
         sync_blocks(self.nodes)
 
         # Make sure all reached phase 2 (contribute) and received all contributions
-        self.wait_for_quorum_phase(2, "receivedContributions", expected_valid_count)
+        self.wait_for_quorum_phase(2, "receivedContributions", expected_contributions)
         set_mocktime(get_mocktime() + 1)
         set_node_times(self.nodes, get_mocktime())
         self.nodes[0].generate(2)
         sync_blocks(self.nodes)
 
         # Make sure all reached phase 3 (complain) and received all complaints
-        self.wait_for_quorum_phase(3, "receivedComplaints" if expected_valid_count != 5 else None, expected_valid_count)
+        self.wait_for_quorum_phase(3, "receivedComplaints", expected_complaints)
         set_mocktime(get_mocktime() + 1)
         set_node_times(self.nodes, get_mocktime())
         self.nodes[0].generate(2)
         sync_blocks(self.nodes)
 
         # Make sure all reached phase 4 (justify)
-        self.wait_for_quorum_phase(4, None, 0)
+        self.wait_for_quorum_phase(4, "receivedJustifications", expected_justifications)
         set_mocktime(get_mocktime() + 1)
         set_node_times(self.nodes, get_mocktime())
         self.nodes[0].generate(2)
         sync_blocks(self.nodes)
 
         # Make sure all reached phase 5 (commit)
-        self.wait_for_quorum_phase(5, "receivedPrematureCommitments", expected_valid_count)
+        self.wait_for_quorum_phase(5, "receivedPrematureCommitments", expected_commitments)
         set_mocktime(get_mocktime() + 1)
         set_node_times(self.nodes, get_mocktime())
         self.nodes[0].generate(2)

src/llmq/quorums_dkgsession.cpp

@@ -25,13 +25,39 @@
 namespace llmq
 {
 
-double contributionOmitRate = 0;
-double contributionLieRate = 0;
-double complainLieRate = 0;
-double justifyOmitRate = 0;
-double justifyLieRate = 0;
-double commitOmitRate = 0;
-double commitLieRate = 0;
+// Supported error types:
+// - contribution-omit
+// - contribution-lie
+// - complain-lie
+// - justify-lie
+// - justify-omit
+// - commit-omit
+// - commit-lie
+
+static CCriticalSection cs_simDkgError;
+static std::map<std::string, double> simDkgErrorMap;
+
+void SetSimulatedDKGErrorRate(const std::string& type, double rate)
+{
+    LOCK(cs_simDkgError);
+    simDkgErrorMap[type] = rate;
+}
+
+static double GetSimulatedErrorRate(const std::string& type)
+{
+    LOCK(cs_simDkgError);
+    auto it = simDkgErrorMap.find(type);
+    if (it != simDkgErrorMap.end()) {
+        return it->second;
+    }
+    return 0;
+}
+
+static bool ShouldSimulateError(const std::string& type)
+{
+    double rate = GetSimulatedErrorRate(type);
+    return GetRandBool(rate);
+}
 
 CDKGLogger::CDKGLogger(const CDKGSession& _quorumDkg, const std::string& _func) :
     CDKGLogger(_quorumDkg.params.type, _quorumDkg.quorumHash, _quorumDkg.height, _quorumDkg.AreWeMember(), _func)
@@ -137,7 +163,7 @@ void CDKGSession::SendContributions(CDKGPendingMessages& pendingMessages)
 
     logger.Batch("sending contributions");
 
-    if (GetRandBool(contributionOmitRate)) {
+    if (ShouldSimulateError("contribution-omit")) {
         logger.Batch("omitting");
         return;
     }
@@ -156,7 +182,7 @@ void CDKGSession::SendContributions(CDKGPendingMessages& pendingMessages)
         auto& m = members[i];
         CBLSSecretKey skContrib = skContributions[i];
 
-        if (GetRandBool(contributionLieRate)) {
+        if (i != myIdx && ShouldSimulateError("contribution-lie")) {
             logger.Batch("lying for %s", m->dmn->proTxHash.ToString());
             skContrib.MakeNewKey();
         }
@@ -297,7 +323,7 @@ void CDKGSession::ReceiveMessage(const uint256& hash, const CDKGContribution& qc
     if (!qc.contributions->Decrypt(myIdx, *activeMasternodeInfo.blsKeyOperator, skContribution, PROTOCOL_VERSION)) {
         logger.Batch("contribution from %s could not be decrypted", member->dmn->proTxHash.ToString());
         complain = true;
-    } else if (GetRandBool(complainLieRate)) {
+    } else if (member->idx != myIdx && ShouldSimulateError("complain-lie")) {
         logger.Batch("lying/complaining for %s", member->dmn->proTxHash.ToString());
         complain = true;
     }
@@ -630,15 +656,15 @@ void CDKGSession::SendJustification(CDKGPendingMessages& pendingMessages, const
 
         CBLSSecretKey skContribution = skContributions[i];
 
-        if (GetRandBool(justifyLieRate)) {
+        if (i != myIdx && ShouldSimulateError("justify-lie")) {
             logger.Batch("lying for %s", m->dmn->proTxHash.ToString());
             skContribution.MakeNewKey();
         }
 
         qj.contributions.emplace_back(i, skContribution);
     }
 
-    if (GetRandBool(justifyOmitRate)) {
+    if (ShouldSimulateError("justify-omit")) {
         logger.Batch("omitting");
         return;
     }
@@ -888,7 +914,7 @@ void CDKGSession::SendCommitment(CDKGPendingMessages& pendingMessages)
         return;
     }
 
-    if (GetRandBool(commitOmitRate)) {
+    if (ShouldSimulateError("commit-omit")) {
         logger.Batch("omitting");
         return;
     }
@@ -926,7 +952,7 @@ void CDKGSession::SendCommitment(CDKGPendingMessages& pendingMessages)
     qc.quorumVvecHash = ::SerializeHash(*vvec);
 
     int lieType = -1;
-    if (GetRandBool(commitLieRate)) {
+    if (ShouldSimulateError("commit-lie")) {
         lieType = GetRandInt(5);
         logger.Batch("lying on commitment. lieType=%d", lieType);
     }

src/llmq/quorums_dkgsession.h

@@ -341,6 +341,8 @@ class CDKGSession
     CDKGMember* GetMember(const uint256& proTxHash) const;
 };
 
+void SetSimulatedDKGErrorRate(const std::string& type, double rate);
+
 }
 
 #endif //DASH_QUORUMS_DKGSESSION_H

src/llmq/quorums_dkgsessionhandler.cpp

@@ -394,13 +394,13 @@ bool ProcessPendingMessageBatch(CDKGSession& session, CDKGPendingMessages& pendi
         bool ban = false;
         if (!session.PreVerifyMessage(hash, msg, ban)) {
             if (ban) {
-                LogPrintf("%s -- banning node due to failed preverification, peer=%d", __func__, p.first);
+                LogPrintf("%s -- banning node due to failed preverification, peer=%d\n", __func__, p.first);
                 {
                     LOCK(cs_main);
                     Misbehaving(p.first, 100);
                 }
             }
-            LogPrintf("%s -- skipping message due to failed preverification, peer=%d", __func__, p.first);
+            LogPrintf("%s -- skipping message due to failed preverification, peer=%d\n", __func__, p.first);
             continue;
         }
         hashes.emplace_back(hash);
@@ -414,7 +414,7 @@ bool ProcessPendingMessageBatch(CDKGSession& session, CDKGPendingMessages& pendi
     if (!badNodes.empty()) {
         LOCK(cs_main);
         for (auto nodeId : badNodes) {
-            LogPrintf("%s -- failed to verify signature, peer=%d", __func__, nodeId);
+            LogPrintf("%s -- failed to verify signature, peer=%d\n", __func__, nodeId);
             Misbehaving(nodeId, 100);
         }
     }
@@ -428,7 +428,7 @@ bool ProcessPendingMessageBatch(CDKGSession& session, CDKGPendingMessages& pendi
         bool ban = false;
         session.ReceiveMessage(hashes[i], msg, ban);
         if (ban) {
-            LogPrintf("%s -- banning node after ReceiveMessage failed, peer=%d", __func__, nodeId);
+            LogPrintf("%s -- banning node after ReceiveMessage failed, peer=%d\n", __func__, nodeId);
             LOCK(cs_main);
             Misbehaving(nodeId, 100);
             badNodes.emplace(nodeId);

src/rpc/rpcquorums.cpp

@@ -273,6 +273,38 @@ UniValue quorum_sigs_cmd(const JSONRPCRequest& request)
     }
 }
 
+void quorum_dkgsimerror_help()
+{
+    throw std::runtime_error(
+            "quorum dkgsimerror \"type\" rate\n"
+            "This enables simulation of errors and malicious behaviour in the DKG. Do NOT use this on mainnet\n"
+            "as you will get yourself very likely PoSe banned for this.\n"
+            "\nArguments:\n"
+            "1. \"type\"                (string, required) Error type.\n"
+            "2. rate                  (number, required) Rate at which to simulate this error type.\n"
+    );
+}
+
+UniValue quorum_dkgsimerror(const JSONRPCRequest& request)
+{
+    auto cmd = request.params[0].get_str();
+    if (request.fHelp || (request.params.size() != 3)) {
+        quorum_dkgsimerror_help();
+    }
+
+    std::string type = request.params[1].get_str();
+    double rate = ParseDoubleV(request.params[2], "rate");
+
+    if (rate < 0 || rate > 1) {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, "invalid rate. Must be between 0 and 1");
+    }
+
+    llmq::SetSimulatedDKGErrorRate(type, rate);
+
+    return UniValue();
+}
+
+
 [[ noreturn ]] void quorum_help()
 {
     throw std::runtime_error(
@@ -284,6 +316,7 @@ UniValue quorum_sigs_cmd(const JSONRPCRequest& request)
             "\nAvailable commands:\n"
             "  list              - List of on-chain quorums\n"
             "  info              - Return information about a quorum\n"
+            "  dkgsimerror       - Simulates DKG errors and malicious behavior.\n"
             "  dkgstatus         - Return the status of the current DKG process\n"
             "  sign              - Threshold-sign a message\n"
             "  hasrecsig         - Test if a valid recovered signature is present\n"
@@ -311,6 +344,8 @@ UniValue quorum(const JSONRPCRequest& request)
         return quorum_dkgstatus(request);
     } else if (command == "sign" || command == "hasrecsig" || command == "getrecsig" || command == "isconflicting") {
         return quorum_sigs_cmd(request);
+    } else if (command == "dkgsimerror") {
+        return quorum_dkgsimerror(request);
     } else {
         quorum_help();
     }