Merge pull request #2832 from codablock/pr_dip4_tests

Implement DIP4 integration tests

qa/pull-tester/rpc-tests.py

@@ -47,6 +47,7 @@
     'llmq-signing.py', # NOTE: needs dash_hash to pass
     'llmq-chainlocks.py', # NOTE: needs dash_hash to pass
     'llmq-simplepose.py', # NOTE: needs dash_hash to pass
+    'dip4-coinbasemerkleroots.py', # NOTE: needs dash_hash to pass
     # vv Tests less than 60s vv
     'sendheaders.py', # NOTE: needs dash_hash to pass
     'zapwallettxes.py',

qa/rpc-tests/dip4-coinbasemerkleroots.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015-2018 The Dash Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+from collections import namedtuple
+
+from test_framework.mininode import *
+from test_framework.test_framework import DashTestFramework
+from test_framework.util import *
+from time import *
+
+'''
+dip4-coinbasemerkleroots.py
+
+Checks DIP4 merkle roots in coinbases
+
+'''
+
+class TestNode(SingleNodeConnCB):
+    def __init__(self):
+        SingleNodeConnCB.__init__(self)
+        self.last_mnlistdiff = None
+
+    def on_mnlistdiff(self, conn, message):
+        self.last_mnlistdiff = message
+
+    def wait_for_mnlistdiff(self, timeout=30):
+        self.last_mnlistdiff = None
+        def received_mnlistdiff():
+            return self.last_mnlistdiff is not None
+        return wait_until(received_mnlistdiff, timeout=timeout)
+
+    def getmnlistdiff(self, baseBlockHash, blockHash):
+        msg = msg_getmnlistd(baseBlockHash, blockHash)
+        self.send_message(msg)
+        self.wait_for_mnlistdiff()
+        return self.last_mnlistdiff
+
+
+class LLMQCoinbaseCommitmentsTest(DashTestFramework):
+    def __init__(self):
+        super().__init__(6, 5, [], fast_dip3_enforcement=True)
+
+    def run_test(self):
+        self.test_node = TestNode()
+        self.test_node.add_connection(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
+        NetworkThread().start()  # Start up network handling in another thread
+        self.test_node.wait_for_verack()
+
+        self.confirm_mns()
+
+        null_hash = format(0, "064x")
+
+        # Check if a diff with the genesis block as base returns all MNs
+        expectedUpdated = [mn.proTxHash for mn in self.mninfo]
+        mnList = self.test_getmnlistdiff(null_hash, self.nodes[0].getbestblockhash(), {}, [], expectedUpdated)
+        expectedUpdated2 = expectedUpdated + []
+
+        # Register one more MN, but don't start it (that would fail as DashTestFramework doesn't support this atm)
+        baseBlockHash = self.nodes[0].getbestblockhash()
+        self.prepare_masternode(self.mn_count)
+        new_mn = self.mninfo[self.mn_count]
+
+        # Now test if that MN appears in a diff when the base block is the one just before MN registration
+        expectedDeleted = []
+        expectedUpdated = [new_mn.proTxHash]
+        mnList = self.test_getmnlistdiff(baseBlockHash, self.nodes[0].getbestblockhash(), mnList, expectedDeleted, expectedUpdated)
+        assert(mnList[new_mn.proTxHash].confirmedHash == 0)
+        # Now let the MN get enough confirmations and verify that the MNLISTDIFF now has confirmedHash != 0
+        self.confirm_mns()
+        mnList = self.test_getmnlistdiff(baseBlockHash, self.nodes[0].getbestblockhash(), mnList, expectedDeleted, expectedUpdated)
+        assert(mnList[new_mn.proTxHash].confirmedHash != 0)
+
+        # Spend the collateral of the previously added MN and test if it appears in "deletedMNs"
+        expectedDeleted = [new_mn.proTxHash]
+        expectedUpdated = []
+        baseBlockHash2 = self.nodes[0].getbestblockhash()
+        self.remove_mastermode(self.mn_count)
+        mnList = self.test_getmnlistdiff(baseBlockHash2, self.nodes[0].getbestblockhash(), mnList, expectedDeleted, expectedUpdated)
+
+        # When comparing genesis and best block, we shouldn't see the previously added and then deleted MN
+        mnList = self.test_getmnlistdiff(null_hash, self.nodes[0].getbestblockhash(), {}, [], expectedUpdated2)
+
+    def test_getmnlistdiff(self, baseBlockHash, blockHash, baseMNList, expectedDeleted, expectedUpdated):
+        d = self.test_getmnlistdiff_base(baseBlockHash, blockHash)
+
+        # Assert that the deletedMNs and mnList fields are what we expected
+        assert_equal(set(d.deletedMNs), set([int(e, 16) for e in expectedDeleted]))
+        assert_equal(set([e.proRegTxHash for e in d.mnList]), set(int(e, 16) for e in expectedUpdated))
+
+        # Build a new list based on the old list and the info from the diff
+        newMNList = baseMNList.copy()
+        for e in d.deletedMNs:
+            newMNList.pop(format(e, '064x'))
+        for e in d.mnList:
+            newMNList[format(e.proRegTxHash, '064x')] = e
+
+        cbtx = CCbTx()
+        cbtx.deserialize(BytesIO(d.cbTx.vExtraPayload))
+
+        # Verify that the merkle root matches what we locally calculate
+        hashes = []
+        for mn in sorted(newMNList.values(), key=lambda mn: ser_uint256(mn.proRegTxHash)):
+            hashes.append(hash256(mn.serialize()))
+        merkleRoot = CBlock.get_merkle_root(hashes)
+        assert_equal(merkleRoot, cbtx.merkleRootMNList)
+
+        return newMNList
+
+    def test_getmnlistdiff_base(self, baseBlockHash, blockHash):
+        hexstr = self.nodes[0].getblockheader(blockHash, False)
+        header = FromHex(CBlockHeader(), hexstr)
+
+        d = self.test_node.getmnlistdiff(int(baseBlockHash, 16), int(blockHash, 16))
+        assert_equal(d.baseBlockHash, int(baseBlockHash, 16))
+        assert_equal(d.blockHash, int(blockHash, 16))
+
+        # Check that the merkle proof is valid
+        proof = CMerkleBlock(header, d.merkleProof)
+        proof = proof.serialize().hex()
+        assert_equal(self.nodes[0].verifytxoutproof(proof), [d.cbTx.hash])
+
+        # Check if P2P messages match with RPCs
+        d2 = self.nodes[0].protx("diff", baseBlockHash, blockHash)
+        assert_equal(d2["baseBlockHash"], baseBlockHash)
+        assert_equal(d2["blockHash"], blockHash)
+        assert_equal(d2["cbTxMerkleTree"], d.merkleProof.serialize().hex())
+        assert_equal(d2["cbTx"], d.cbTx.serialize().hex())
+        assert_equal(set([int(e, 16) for e in d2["deletedMNs"]]), set(d.deletedMNs))
+        assert_equal(set([int(e["proRegTxHash"], 16) for e in d2["mnList"]]), set([e.proRegTxHash for e in d.mnList]))
+
+        return d
+
+    def confirm_mns(self):
+        while True:
+            diff = self.nodes[0].protx("diff", 1, self.nodes[0].getblockcount())
+            found_unconfirmed = False
+            for mn in diff["mnList"]:
+                if int(mn["confirmedHash"], 16) == 0:
+                    found_unconfirmed = True
+                    break
+            if not found_unconfirmed:
+                break
+            self.nodes[0].generate(1)
+        sync_blocks(self.nodes)
+
+if __name__ == '__main__':
+    LLMQCoinbaseCommitmentsTest().main()

qa/rpc-tests/test_framework/mininode.py

@@ -201,6 +201,34 @@ def ser_int_vector(l):
         r += struct.pack("<i", i)
     return r
 
+
+def deser_dyn_bitset(f, bytes_based):
+    if bytes_based:
+        nb = deser_compact_size(f)
+        n = nb * 8
+    else:
+        n = deser_compact_size(f)
+        nb = int((n + 7) / 8)
+    b = f.read(nb)
+    r = []
+    for i in range(n):
+        r.append((b[int(i / 8)] & (1 << (i % 8))) != 0)
+    return r
+
+
+def ser_dyn_bitset(l, bytes_based):
+    n = len(l)
+    nb = int((n + 7) / 8)
+    r = [0] * nb
+    for i in range(n):
+        r[int(i / 8)] |= (1 if l[i] else 0) << (i % 8)
+    if bytes_based:
+        r = ser_compact_size(nb) + bytes(r)
+    else:
+        r = ser_compact_size(n) + bytes(r)
+    return r
+
+
 # Deserialize from a hex string representation (eg from RPC)
 def FromHex(obj, hex_string):
     obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
@@ -212,6 +240,25 @@ def ToHex(obj):
 
 # Objects that map to dashd objects, which can be serialized/deserialized
 
+class CService(object):
+    def __init__(self):
+        self.ip = ""
+        self.port = 0
+
+    def deserialize(self, f):
+        self.ip = socket.inet_ntop(socket.AF_INET6, f.read(16))
+        self.port = struct.unpack(">H", f.read(2))[0]
+
+    def serialize(self):
+        r = b""
+        r += socket.inet_pton(socket.AF_INET6, self.ip)
+        r += struct.pack(">H", self.port)
+        return r
+
+    def __repr__(self):
+        return "CService(ip=%s port=%i)" % (self.ip, self.port)
+
+
 class CAddress(object):
     def __init__(self):
         self.nServices = 1
@@ -502,7 +549,8 @@ def serialize(self):
         return r
 
     # Calculate the merkle root given a vector of transaction hashes
-    def get_merkle_root(self, hashes):
+    @staticmethod
+    def get_merkle_root(hashes):
         while len(hashes) > 1:
             newhashes = []
             for i in range(0, len(hashes), 2):
@@ -793,6 +841,47 @@ def __repr__(self):
         return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
 
 
+class CPartialMerkleTree(object):
+    def __init__(self):
+        self.nTransactions = 0
+        self.vBits = []
+        self.vHash = []
+
+    def deserialize(self, f):
+        self.nTransactions = struct.unpack("<I", f.read(4))[0]
+        self.vHash = deser_uint256_vector(f)
+        self.vBits = deser_dyn_bitset(f, True)
+
+    def serialize(self):
+        r = b""
+        r += struct.pack("<I", self.nTransactions)
+        r += ser_uint256_vector(self.vHash)
+        r += ser_dyn_bitset(self.vBits, True)
+        return r
+
+    def __repr__(self):
+        return "CPartialMerkleTree(nTransactions=%d vBits.size=%d vHash.size=%d)" % (self.nTransactions, len(self.vBits), len(self.vHash))
+
+
+class CMerkleBlock(object):
+    def __init__(self, header=CBlockHeader(), txn=CPartialMerkleTree()):
+        self.header = header
+        self.txn = txn
+
+    def deserialize(self, f):
+        self.header.deserialize(f)
+        self.txn.deserialize(f)
+
+    def serialize(self):
+        r = b""
+        r += self.header.serialize()
+        r += self.txn.serialize()
+        return r
+
+    def __repr__(self):
+        return "CMerkleBlock(header=%s txn=%s)" % (repr(self.header), repr(self.txn))
+
+
 class CCbTx(object):
     def __init__(self, version=None, height=None, merkleRootMNList=None):
         self.set_null()
@@ -821,6 +910,37 @@ def serialize(self):
         return r
 
 
+class CSimplifiedMNListEntry(object):
+    def __init__(self):
+        self.set_null()
+
+    def set_null(self):
+        self.proRegTxHash = 0
+        self.confirmedHash = 0
+        self.service = CService()
+        self.pubKeyOperator = b'\\x0' * 48
+        self.keyIDVoting = 0
+        self.isValid = False
+
+    def deserialize(self, f):
+        self.proRegTxHash = deser_uint256(f)
+        self.confirmedHash = deser_uint256(f)
+        self.service.deserialize(f)
+        self.pubKeyOperator = f.read(48)
+        self.keyIDVoting = f.read(20)
+        self.isValid = struct.unpack("<?", f.read(1))[0]
+
+    def serialize(self):
+        r = b""
+        r += ser_uint256(self.proRegTxHash)
+        r += ser_uint256(self.confirmedHash)
+        r += self.service.serialize()
+        r += self.pubKeyOperator
+        r += self.keyIDVoting
+        r += struct.pack("<?", self.isValid)
+        return r
+
+
 # Objects that correspond to messages on the wire
 class msg_version(object):
     command = b"version"
@@ -1315,6 +1435,55 @@ def serialize(self):
     def __repr__(self):
         return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
 
+class msg_getmnlistd(object):
+    command = b"getmnlistd"
+
+    def __init__(self, baseBlockHash=0, blockHash=0):
+        self.baseBlockHash = baseBlockHash
+        self.blockHash = blockHash
+
+    def deserialize(self, f):
+        self.baseBlockHash = deser_uint256(f)
+        self.blockHash = deser_uint256(f)
+
+    def serialize(self):
+        r = b""
+        r += ser_uint256(self.baseBlockHash)
+        r += ser_uint256(self.blockHash)
+        return r
+
+    def __repr__(self):
+        return "msg_getmnlistd(baseBlockHash=%064x, blockHash=%064x)" % (self.baseBlockHash, self.blockHash)
+
+class msg_mnlistdiff(object):
+    command = b"mnlistdiff"
+
+    def __init__(self):
+        self.baseBlockHash = 0
+        self.blockHash = 0
+        self.merkleProof = CPartialMerkleTree()
+        self.cbTx = None
+        self.deletedMNs = []
+        self.mnList = []
+
+    def deserialize(self, f):
+        self.baseBlockHash = deser_uint256(f)
+        self.blockHash = deser_uint256(f)
+        self.merkleProof.deserialize(f)
+        self.cbTx = CTransaction()
+        self.cbTx.deserialize(f)
+        self.cbTx.rehash()
+        self.deletedMNs = deser_uint256_vector(f)
+        self.mnList = []
+        for i in range(deser_compact_size(f)):
+            e = CSimplifiedMNListEntry()
+            e.deserialize(f)
+            self.mnList.append(e)
+
+    def __repr__(self):
+        return "msg_mnlistdiff(baseBlockHash=%064x, blockHash=%064x)" % (self.baseBlockHash, self.blockHash)
+
+
 # This is what a callback should look like for NodeConn
 # Reimplement the on_* functions to provide handling for events
 class NodeConnCB(object):
@@ -1398,6 +1567,7 @@ def on_sendcmpct(self, conn, message): pass
     def on_cmpctblock(self, conn, message): pass
     def on_getblocktxn(self, conn, message): pass
     def on_blocktxn(self, conn, message): pass
+    def on_mnlistdiff(self, conn, message): pass
 
 # More useful callbacks and functions for NodeConnCB's which have a single NodeConn
 class SingleNodeConnCB(NodeConnCB):
@@ -1454,7 +1624,8 @@ class NodeConn(asyncore.dispatcher):
         b"sendcmpct": msg_sendcmpct,
         b"cmpctblock": msg_cmpctblock,
         b"getblocktxn": msg_getblocktxn,
-        b"blocktxn": msg_blocktxn
+        b"blocktxn": msg_blocktxn,
+        b"mnlistdiff": msg_mnlistdiff
     }
     MAGIC_BYTES = {
         "mainnet": b"\xbf\x0c\x6b\xbd",   # mainnet