Properly handle conflicts between ChainLocks and InstantSend (#2904)

* Move code to write archived ISLOCKs into its own method

We'll need this from another method as well later.

* Return ISLOCK instead of conflicting txid in GetConflictingTx/GetConflictingLock

* Implement GetInstantSendLocksByParent and RemoveChainedInstantSendLocks

These allow to easily delete multiple chains (actually trees) of ISLOCKs
in one go.

* Implement RemoveConflictedTx and call it from RemoveMempoolConflictsForLock

Also add "retryChildren" parameter to RemoveNonLockedTx so that we can
skip retrying of non-locked children TXs.

* Properly handle/remove conflicted TXs (between mempool and new blocks)

* Track non-locked TXs by inputs

* Implement and call ResolveBlockConflicts

* Also call ResolveBlockConflicts from ConnectBlock

But only when a block is known to have a conflict and at the same time is
ChainLocked, which causes the ISLOCK to be pruned.

* Split out RemoveChainLockConflictingLock from ResolveBlockConflicts

* Implement "quorum getrecsig" RPC

* Include decoded TX data in result of create_raw_tx

* Implement support for CLSIG in mininode.py

* Fix condition for update of nonLockedTxs.pindexMined

* Only add entries to nonLockedTxsByInputs when AddNonLockedTx is called for the first time

* Implement support for ISLOCK in mininode.py

* Implement tests for ChainLock vs InstantSend lock conflict resolution

* Handle review comment

Bail out (continue) early

qa/pull-tester/rpc-tests.py

@@ -47,6 +47,7 @@
     'llmq-signing.py', # NOTE: needs dash_hash to pass
     'llmq-chainlocks.py', # NOTE: needs dash_hash to pass
     'llmq-simplepose.py', # NOTE: needs dash_hash to pass
+    'llmq-is-cl-conflicts.py', # NOTE: needs dash_hash to pass
     'dip4-coinbasemerkleroots.py', # NOTE: needs dash_hash to pass
     # vv Tests less than 60s vv
     'sendheaders.py', # NOTE: needs dash_hash to pass

qa/rpc-tests/llmq-is-cl-conflicts.py

@@ -0,0 +1,338 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015-2018 The Dash Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+from test_framework.blocktools import get_masternode_payment, create_coinbase, create_block
+from test_framework.mininode import *
+from test_framework.test_framework import DashTestFramework
+from test_framework.util import *
+from time import *
+
+'''
+llmq-is-cl-conflicts.py
+
+Checks conflict handling between ChainLocks and InstantSend
+
+'''
+
+class TestNode(SingleNodeConnCB):
+    def __init__(self):
+        SingleNodeConnCB.__init__(self)
+        self.clsigs = {}
+        self.islocks = {}
+
+    def send_clsig(self, clsig):
+        hash = uint256_from_str(hash256(clsig.serialize()))
+        self.clsigs[hash] = clsig
+
+        inv = msg_inv([CInv(29, hash)])
+        self.send_message(inv)
+
+    def send_islock(self, islock):
+        hash = uint256_from_str(hash256(islock.serialize()))
+        self.islocks[hash] = islock
+
+        inv = msg_inv([CInv(30, hash)])
+        self.send_message(inv)
+
+    def on_getdata(self, conn, message):
+        for inv in message.inv:
+            if inv.hash in self.clsigs:
+                self.send_message(self.clsigs[inv.hash])
+            if inv.hash in self.islocks:
+                self.send_message(self.islocks[inv.hash])
+
+
+class LLMQ_IS_CL_Conflicts(DashTestFramework):
+    def __init__(self):
+        super().__init__(6, 5, [], fast_dip3_enforcement=True)
+        #disable_mocktime()
+
+    def run_test(self):
+
+        while self.nodes[0].getblockchaininfo()["bip9_softforks"]["dip0008"]["status"] != "active":
+            self.nodes[0].generate(10)
+        sync_blocks(self.nodes, timeout=60*5)
+
+        self.test_node = TestNode()
+        self.test_node.add_connection(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
+        NetworkThread().start()  # Start up network handling in another thread
+        self.test_node.wait_for_verack()
+
+        self.nodes[0].spork("SPORK_17_QUORUM_DKG_ENABLED", 0)
+        self.nodes[0].spork("SPORK_19_CHAINLOCKS_ENABLED", 0)
+        self.nodes[0].spork("SPORK_20_INSTANTSEND_LLMQ_BASED", 0)
+        self.wait_for_sporks_same()
+
+        self.mine_quorum()
+
+        # mine single block, wait for chainlock
+        self.nodes[0].generate(1)
+        self.wait_for_chainlock_tip_all_nodes()
+
+        self.test_chainlock_overrides_islock(False)
+        self.test_chainlock_overrides_islock(True)
+        self.test_islock_overrides_nonchainlock()
+
+    def test_chainlock_overrides_islock(self, test_block_conflict):
+        # create three raw TXs, they will conflict with each other
+        rawtx1 = self.create_raw_tx(self.nodes[0], self.nodes[0], 1, 1, 100)['hex']
+        rawtx2 = self.create_raw_tx(self.nodes[0], self.nodes[0], 1, 1, 100)['hex']
+        rawtx3 = self.create_raw_tx(self.nodes[0], self.nodes[0], 1, 1, 100)['hex']
+        rawtx1_obj = FromHex(CTransaction(), rawtx1)
+        rawtx2_obj = FromHex(CTransaction(), rawtx2)
+        rawtx3_obj = FromHex(CTransaction(), rawtx3)
+
+        rawtx1_txid = self.nodes[0].sendrawtransaction(rawtx1)
+        rawtx2_txid = encode(hash256(hex_str_to_bytes(rawtx2))[::-1], 'hex_codec').decode('ascii')
+        rawtx3_txid = encode(hash256(hex_str_to_bytes(rawtx3))[::-1], 'hex_codec').decode('ascii')
+
+        # Create a chained TX on top of tx1
+        inputs = []
+        n = 0
+        for out in rawtx1_obj.vout:
+            if out.nValue == 100000000:
+                inputs.append({"txid": rawtx1_txid, "vout": n})
+            n += 1
+        rawtx4 = self.nodes[0].createrawtransaction(inputs, {self.nodes[0].getnewaddress(): 0.999})
+        rawtx4 = self.nodes[0].signrawtransaction(rawtx4)['hex']
+        rawtx4_txid = self.nodes[0].sendrawtransaction(rawtx4)
+
+        for node in self.nodes:
+            self.wait_for_instantlock(rawtx1_txid, node)
+            self.wait_for_instantlock(rawtx4_txid, node)
+
+        block = self.create_block(self.nodes[0], [rawtx2_obj])
+        if test_block_conflict:
+            submit_result = self.nodes[0].submitblock(ToHex(block))
+            assert(submit_result == "conflict-tx-lock")
+
+        cl = self.create_chainlock(self.nodes[0].getblockcount() + 1, block.sha256)
+        self.test_node.send_clsig(cl)
+
+        # Give the CLSIG some time to propagate. We unfortunately can't check propagation here as "getblock/getblockheader"
+        # is required to check for CLSIGs, but this requires the block header to be propagated already
+        sleep(1)
+
+        # The block should get accepted now, and at the same time prune the conflicting ISLOCKs
+        submit_result = self.nodes[1].submitblock(ToHex(block))
+        if test_block_conflict:
+            assert(submit_result == "duplicate")
+        else:
+            assert(submit_result is None)
+
+        for node in self.nodes:
+            self.wait_for_chainlock(node, "%064x" % block.sha256)
+
+        # Create a chained TX on top of tx2
+        inputs = []
+        n = 0
+        for out in rawtx2_obj.vout:
+            if out.nValue == 100000000:
+                inputs.append({"txid": rawtx2_txid, "vout": n})
+            n += 1
+        rawtx5 = self.nodes[0].createrawtransaction(inputs, {self.nodes[0].getnewaddress(): 0.999})
+        rawtx5 = self.nodes[0].signrawtransaction(rawtx5)['hex']
+        rawtx5_txid = self.nodes[0].sendrawtransaction(rawtx5)
+        for node in self.nodes:
+            self.wait_for_instantlock(rawtx5_txid, node)
+
+        # Lets verify that the ISLOCKs got pruned
+        for node in self.nodes:
+            assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx1_txid, True)
+            assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx4_txid, True)
+            rawtx = node.getrawtransaction(rawtx2_txid, True)
+            assert(rawtx['chainlock'])
+            assert(rawtx['instantlock'])
+            assert(not rawtx['instantlock_internal'])
+
+    def test_islock_overrides_nonchainlock(self):
+        # create two raw TXs, they will conflict with each other
+        rawtx1 = self.create_raw_tx(self.nodes[0], self.nodes[0], 1, 1, 100)['hex']
+        rawtx2 = self.create_raw_tx(self.nodes[0], self.nodes[0], 1, 1, 100)['hex']
+
+        rawtx1_txid = encode(hash256(hex_str_to_bytes(rawtx1))[::-1], 'hex_codec').decode('ascii')
+        rawtx2_txid = encode(hash256(hex_str_to_bytes(rawtx2))[::-1], 'hex_codec').decode('ascii')
+
+        # Create an ISLOCK but don't broadcast it yet
+        islock = self.create_islock(rawtx2)
+
+        # Stop enough MNs so that ChainLocks don't work anymore
+        for i in range(3):
+            self.stop_node(len(self.nodes) - 1)
+            self.nodes.pop(len(self.nodes) - 1)
+            self.mninfo.pop(len(self.mninfo) - 1)
+
+        # Send tx1, which will later conflict with the ISLOCK
+        self.nodes[0].sendrawtransaction(rawtx1)
+
+        # fast forward 11 minutes, so that the TX is considered safe and included in the next block
+        set_mocktime(get_mocktime() + int(60 * 11))
+        set_node_times(self.nodes, get_mocktime())
+
+        # Mine the conflicting TX into a block
+        good_tip = self.nodes[0].getbestblockhash()
+        self.nodes[0].generate(2)
+        self.sync_all()
+
+        # Assert that the conflicting tx got mined and the locked TX is not valid
+        assert(self.nodes[0].getrawtransaction(rawtx1_txid, True)['confirmations'] > 0)
+        assert_raises_jsonrpc(-25, "Missing inputs", self.nodes[0].sendrawtransaction, rawtx2)
+
+        # Send the ISLOCK, which should result in the last 2 blocks to be invalidated, even though the nodes don't know
+        # the locked transaction yet
+        self.test_node.send_islock(islock)
+        sleep(5)
+
+        assert(self.nodes[0].getbestblockhash() == good_tip)
+        assert(self.nodes[1].getbestblockhash() == good_tip)
+
+        # Send the actual transaction and mine it
+        self.nodes[0].sendrawtransaction(rawtx2)
+        self.nodes[0].generate(1)
+        self.sync_all()
+
+        assert(self.nodes[0].getrawtransaction(rawtx2_txid, True)['confirmations'] > 0)
+        assert(self.nodes[1].getrawtransaction(rawtx2_txid, True)['confirmations'] > 0)
+        assert(self.nodes[0].getrawtransaction(rawtx2_txid, True)['instantlock'])
+        assert(self.nodes[1].getrawtransaction(rawtx2_txid, True)['instantlock'])
+        assert(self.nodes[0].getbestblockhash() != good_tip)
+        assert(self.nodes[1].getbestblockhash() != good_tip)
+
+    def wait_for_chainlock_tip_all_nodes(self):
+        for node in self.nodes:
+            tip = node.getbestblockhash()
+            self.wait_for_chainlock(node, tip)
+
+    def wait_for_chainlock_tip(self, node):
+        tip = node.getbestblockhash()
+        self.wait_for_chainlock(node, tip)
+
+    def wait_for_chainlock(self, node, block_hash):
+        t = time()
+        while time() - t < 15:
+            try:
+                block = node.getblockheader(block_hash)
+                if block["confirmations"] > 0 and block["chainlock"]:
+                    return
+            except:
+                # block might not be on the node yet
+                pass
+            sleep(0.1)
+        raise AssertionError("wait_for_chainlock timed out")
+
+    def create_block(self, node, vtx=[]):
+        bt = node.getblocktemplate()
+        height = bt['height']
+        tip_hash = bt['previousblockhash']
+
+        coinbasevalue = bt['coinbasevalue']
+        miner_address = node.getnewaddress()
+        mn_payee = bt['masternode'][0]['payee']
+
+        # calculate fees that the block template included (we'll have to remove it from the coinbase as we won't
+        # include the template's transactions
+        bt_fees = 0
+        for tx in bt['transactions']:
+            bt_fees += tx['fee']
+
+        new_fees = 0
+        for tx in vtx:
+            in_value = 0
+            out_value = 0
+            for txin in tx.vin:
+                txout = node.gettxout("%064x" % txin.prevout.hash, txin.prevout.n, False)
+                in_value += int(txout['value'] * COIN)
+            for txout in tx.vout:
+                out_value += txout.nValue
+            new_fees += in_value - out_value
+
+        # fix fees
+        coinbasevalue -= bt_fees
+        coinbasevalue += new_fees
+
+        mn_amount = get_masternode_payment(height, coinbasevalue)
+        miner_amount = coinbasevalue - mn_amount
+
+        outputs = {miner_address: str(Decimal(miner_amount) / COIN)}
+        if mn_amount > 0:
+            outputs[mn_payee] = str(Decimal(mn_amount) / COIN)
+
+        coinbase = FromHex(CTransaction(), node.createrawtransaction([], outputs))
+        coinbase.vin = create_coinbase(height).vin
+
+        # We can't really use this one as it would result in invalid merkle roots for masternode lists
+        if len(bt['coinbase_payload']) != 0:
+            cbtx = FromHex(CCbTx(version=1), bt['coinbase_payload'])
+            coinbase.nVersion = 3
+            coinbase.nType = 5 # CbTx
+            coinbase.vExtraPayload = cbtx.serialize()
+
+        coinbase.calc_sha256()
+
+        block = create_block(int(tip_hash, 16), coinbase, nTime=bt['curtime'])
+        block.vtx += vtx
+
+        # Add quorum commitments from template
+        for tx in bt['transactions']:
+            tx2 = FromHex(CTransaction(), tx['data'])
+            if tx2.nType == 6:
+                block.vtx.append(tx2)
+
+        block.hashMerkleRoot = block.calc_merkle_root()
+        block.solve()
+        return block
+
+    def create_chainlock(self, height, blockHash):
+        request_id = "%064x" % uint256_from_str(hash256(ser_string(b"clsig") + struct.pack("<I", height)))
+        message_hash = "%064x" % blockHash
+
+        for mn in self.mninfo:
+            mn.node.quorum('sign', 100, request_id, message_hash)
+
+        recSig = None
+
+        t = time()
+        while time() - t < 10:
+            try:
+                recSig = self.nodes[0].quorum('getrecsig', 100, request_id, message_hash)
+                break
+            except:
+                sleep(0.1)
+        assert(recSig is not None)
+
+        clsig = msg_clsig(height, blockHash, hex_str_to_bytes(recSig['sig']))
+        return clsig
+
+    def create_islock(self, hextx):
+        tx = FromHex(CTransaction(), hextx)
+        tx.rehash()
+
+        request_id_buf = ser_string(b"islock") + ser_compact_size(len(tx.vin))
+        inputs = []
+        for txin in tx.vin:
+            request_id_buf += txin.prevout.serialize()
+            inputs.append(txin.prevout)
+        request_id = "%064x" % uint256_from_str(hash256(request_id_buf))
+        message_hash = "%064x" % tx.sha256
+
+        for mn in self.mninfo:
+            mn.node.quorum('sign', 100, request_id, message_hash)
+
+        recSig = None
+
+        t = time()
+        while time() - t < 10:
+            try:
+                recSig = self.nodes[0].quorum('getrecsig', 100, request_id, message_hash)
+                break
+            except:
+                sleep(0.1)
+        assert(recSig is not None)
+
+        islock = msg_islock(inputs, tx.sha256, hex_str_to_bytes(recSig['sig']))
+        return islock
+
+
+if __name__ == '__main__':
+    LLMQ_IS_CL_Conflicts().main()