Fix for verifychain (#634)

* fix coins view instead of coins tip

* recount votes when verifying database

* fix boolean

* write instead of insert if it already exists

* use mapState instead of fState and blockhash

* add missing txblockhash to ser function

* fix log

* getstate string for paid state

* update paidOnblock to stateChangedOnBlock from cfund tests

* check statedb hash in verifychain

* fix v452 fork

* fix %d->%s

* add log

* use formatmoney

* prevent undefined behavior

* show cf supply in getblockheader

* check for paid state

* fix some tests

* fix diff prequest

* GetLastStateBlockIndexForState()

* fix cfund-paymentrequest-hardfork-452.py

* consider status changes from same block

* fix heights for testnet

* notRequestedYet

* notRequestedYet

* detect old db struct

* add statehash to updatetip log

* check old structure bootstrap

* only do reindex-chainstate when rebuilding db struct

* only calculate statehash if debug=statehash

* include cfsupply and cflocked in statehash

* add mine filter to listproposals

* support multiple filters

* remove duplicate declaration

* add cfund test unit

* fix call to old var

* setup consistency

* fix loop condition

* assert statehash

* remove old reference after master merge

* remove old reference after master merge

* fDirty

* Revert "fDirty"

This reverts commit 561339d.

* add extra log

Author: alex v

qa/pull-tester/rpc-tests.py

@@ -190,7 +190,8 @@
     'sendtoaddress.py',
     'stakeimmaturebalance.py',
     'rpc-help.py',
-    'createrawscriptaddress.py'
+    'createrawscriptaddress.py',
+    'cfunddb-statehash.py'
 ]
 #if ENABLE_ZMQ:
 #    testScripts.append('zmq_test.py')

qa/rpc-tests/cfund-fork-reorg-preq.py

@@ -17,7 +17,9 @@ def __init__(self):
         self.num_nodes = 2
 
     def setup_network(self, split=False):
-        self.nodes = self.setup_nodes()
+        self.nodes = []
+        self.nodes.append(start_node(0, self.options.tmpdir, ["-debug=dao"]))
+        self.nodes.append(start_node(1, self.options.tmpdir, ["-debug=dao"]))
         connect_nodes_bi(self.nodes, 0, 1)
         self.is_network_split = False
 
@@ -83,7 +85,7 @@ def run_test(self):
         best_block = self.nodes[1].getbestblockhash()
 
         self.stop_node(0)
-        self.nodes[0] = start_node(0, self.options.tmpdir, [])
+        self.nodes[0] = start_node(0, self.options.tmpdir, ["-debug=dao"])
 
         # Reconnect the nodes
         connect_nodes_bi(self.nodes, 0, 1)
@@ -100,8 +102,8 @@ def run_test(self):
         slow_gen(self.nodes[0], self.nodes[0].cfundstats()["votingPeriod"]["ending"] - self.nodes[0].cfundstats()["votingPeriod"]["current"])
         sync_blocks(self.nodes)
 
-        assert_equal(self.nodes[0].getpaymentrequest(paymentHash0)["status"], "accepted")
-        assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(paymentHash0)["paidOnBlock"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(paymentHash0)["paidOnBlock"]))
+        assert_equal(self.nodes[0].getpaymentrequest(paymentHash0)["status"], "paid")
+        assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(paymentHash0)["stateChangedOnBlock"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(paymentHash0)["stateChangedOnBlock"]))
         assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
         assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(paymentHash0)["blockHash"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(paymentHash0)["blockHash"]))
         assert_equal(self.nodes[0].getpaymentrequest(paymentHash0), self.nodes[1].getpaymentrequest(paymentHash0))
@@ -117,10 +119,10 @@ def create_raw_paymentrequest(self, amount, address, proposal_hash, description)
             [],
             {"6ac1": 1},
             json.dumps({
-              "v": 2, 
-              "h": proposal_hash, 
-              "n": amount, 
-              "s": signature, 
+              "v": 2,
+              "h": proposal_hash,
+              "n": amount,
+              "s": signature,
               "i": description,
               })
         )

qa/rpc-tests/cfund-fork-reorg-proposal.py

@@ -19,7 +19,9 @@ def __init__(self):
         self.num_nodes = 2
 
     def setup_network(self, split=False):
-        self.nodes = self.setup_nodes()
+        self.nodes = []
+        self.nodes.append(start_node(0, self.options.tmpdir, ["-debug=dao"]))
+        self.nodes.append(start_node(1, self.options.tmpdir, ["-debug=dao"]))
         connect_nodes_bi(self.nodes, 0, 1)
         self.is_network_split = False
 
@@ -59,19 +61,19 @@ def run_test(self):
         slow_gen(self.nodes[1], 1)
 
         # Assert that both hashes for payment request are identical
-        assert(proposalHash0 == proposalHash1)    
+        assert(proposalHash0 == proposalHash1)
 
         # Assert that both payment requests have been included in different block hashes
         assert(self.nodes[0].getproposal(proposalHash0)["blockHash"] != self.nodes[1].getproposal(proposalHash1)["blockHash"])
-    
+
         # Make the node vote yes
         self.nodes[1].proposalvote(proposalHash0, "yes")
 
         # End cycle 0
         slow_gen(self.nodes[1], 1)
         end_cycle(self.nodes[1])
 
-        # End cycle 1        
+        # End cycle 1
         slow_gen(self.nodes[1], 1)
         end_cycle(self.nodes[1])
 
@@ -96,7 +98,7 @@ def run_test(self):
         assert_equal(self.nodes[0].getblock(self.nodes[0].getproposal(proposalHash0)["blockHash"]), self.nodes[1].getblock(self.nodes[1].getproposal(proposalHash0)["blockHash"]))
         assert_equal(self.nodes[0].getproposal(proposalHash0), self.nodes[1].getproposal(proposalHash0))
 
-        # End cycle 2        
+        # End cycle 2
         slow_gen(self.nodes[1], 1)
         end_cycle(self.nodes[1])
         sync_blocks(self.nodes)
@@ -132,14 +134,14 @@ def run_test(self):
 
         # Verify both nodes accpeted the payment
 
-        assert_equal(self.nodes[0].getpaymentrequest(preqHash)["status"], "accepted")
-        assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["paidOnBlock"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(preqHash)["paidOnBlock"]))
+        assert_equal(self.nodes[0].getpaymentrequest(preqHash)["status"], "paid")
+        assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["stateChangedOnBlock"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(preqHash)["stateChangedOnBlock"]))
         assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
         assert_equal(self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["blockHash"]), self.nodes[1].getblock(self.nodes[1].getpaymentrequest(preqHash)["blockHash"]))
         assert_equal(self.nodes[0].getpaymentrequest(preqHash), self.nodes[1].getpaymentrequest(preqHash))
 
         # Verify the payment was actually received
-        paidBlock = self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["paidOnBlock"])
+        paidBlock = self.nodes[0].getblock(self.nodes[0].getpaymentrequest(preqHash)["stateChangedOnBlock"])
         unspent = self.nodes[0].listunspent(0, 80)
 
         assert_equal(unspent[0]['address'], paymentAddress)

qa/rpc-tests/cfund-paymentrequest-duplicate.py

@@ -17,6 +17,7 @@ def __init__(self):
         super().__init__()
         self.setup_clean_chain = True
         self.num_nodes = 2
+        self.node_args = [['-debug=dao'], ['-debug=dao']]
 
     def setup_network(self, split=False):
         self.nodes = []
@@ -141,14 +142,11 @@ def run_test(self):
         for x in range(self.num_nodes):
             for proposal in proposals:
                 for preq in proposal["preqs"]:
-                    assert(self.nodes[x].getpaymentrequest(preq["hash"])["state"] == 1)
-                    assert(self.nodes[x].getpaymentrequest(preq["hash"])["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000")
-
-
+                    assert_equal(self.nodes[x].getpaymentrequest(preq["hash"])["state"], 1)
 
         wallet_info1 = self.nodes[0].getwalletinfo()
 
-        while self.nodes[0].getpaymentrequest(proposals[0]["preqs"][0]["hash"])["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000":
+        while self.nodes[0].getpaymentrequest(proposals[0]["preqs"][0]["hash"])["state"] != 6:
             blocks = slow_gen(self.nodes[0], 1)
 
         sync_blocks(self.nodes)
@@ -164,7 +162,7 @@ def run_test(self):
             for proposal in proposals:
                 for preq in proposal["preqs"]:
                     preqsFound += 1
-                    payoutBlockHash = self.nodes[x].getpaymentrequest(preq["hash"])["paidOnBlock"]
+                    payoutBlockHash = self.nodes[x].getpaymentrequest(preq["hash"])["stateChangedOnBlock"]
                     payoutBlock = self.nodes[x].getblock(payoutBlockHash)
                     payoutHex = self.nodes[x].getrawtransaction(payoutBlock["tx"][0])
                     payoutTx = self.nodes[x].decoderawtransaction(payoutHex)

qa/rpc-tests/cfund-paymentrequest-extract-funds.py

@@ -112,7 +112,7 @@ def run_test(self):
         # the 5 payment requests should be accepted
         allAccepted = True
         for paymentReq in paymentRequests:
-            if self.nodes[0].getpaymentrequest(paymentReq)["state"] != 1:
+            if self.nodes[0].getpaymentrequest(paymentReq)["state"] != 6:
                 allAccepted = False
 
         # all the payment requests should have been validated

qa/rpc-tests/cfund-paymentrequest-hardfork-452.py

@@ -26,7 +26,7 @@ def run_test(self):
 
         payoutTxID_1 = self.create_double_preq_payout()
         print("Second payment request payout:\n"+str(self.nodes[0].gettxout(payoutTxID_1, 1)))
-        
+
         # If the payment request is paid out again, the coinbase tx will have 2 vouts, so check txout at index 1
         assert(self.nodes[0].gettxout(payoutTxID_1, 1) != None)
 
@@ -36,11 +36,11 @@ def run_test(self):
 
         payoutTxID_2 = self.create_double_preq_payout()
         print("Second payment request payout:\n"+str(self.nodes[0].gettxout(payoutTxID_2, 1)))
-        
+
 
         # If the payment request is paid out again, the coinbase tx will have 2 vouts, so check txout at index 1
         assert(self.nodes[0].gettxout(payoutTxID_2, 1) == None)
-        
+
 
     def create_double_preq_payout(self):
         # Creates a proposal and payment request that is paid out twice
@@ -53,11 +53,11 @@ def create_double_preq_payout(self):
 
         proposalid0 = self.nodes[0].createproposal(paymentAddress, proposalAmount, 36000, "test")["hash"]
         start_new_cycle(self.nodes[0])
-        
+
         self.nodes[0].proposalvote(proposalid0, "yes")
-        
+
         start_new_cycle(self.nodes[0])
-        
+
         # Proposal should be accepted
         assert(self.nodes[0].getproposal(proposalid0)["state"] == 1)
         assert(self.nodes[0].getproposal(proposalid0)["status"] == "accepted")
@@ -74,9 +74,9 @@ def create_double_preq_payout(self):
 
         start_new_cycle(self.nodes[0])
         assert(self.nodes[0].getpaymentrequest(paymentReq)["state"] == 1)
-        assert(self.nodes[0].getpaymentrequest(paymentReq)["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000")
+        assert(self.nodes[0].getpaymentrequest(paymentReq)["stateChangedOnBlock"] != "0000000000000000000000000000000000000000000000000000000000000000")
 
-        while self.nodes[0].getpaymentrequest(paymentReq)["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000":
+        while self.nodes[0].getpaymentrequest(paymentReq)["state"] != 6:
             blocks = slow_gen(self.nodes[0], 1)
 
         slow_gen(self.nodes[0], 1)
@@ -87,11 +87,11 @@ def create_double_preq_payout(self):
 
         self.nodes[0].coinbaseoutputs([rawOutput])
         self.nodes[0].setcoinbasestrdzeel('[\"'+paymentReq+'\"]')
-        
+
         payoutBlockHash = slow_gen(self.nodes[0], 1)[0]
 
         payoutTxID = self.nodes[0].getblock(payoutBlockHash)["tx"][0]
-        
+
         return payoutTxID
 
 

qa/rpc-tests/cfund-paymentrequest-payout.py

@@ -103,11 +103,10 @@ def run_test(self):
 
         for x in range(self.num_nodes):
             assert(self.nodes[x].getpaymentrequest(paymentReq)["state"] == 1)
-            assert(self.nodes[x].getpaymentrequest(paymentReq)["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000")
 
         wallet_info1 = self.nodes[0].getwalletinfo()
 
-        while self.nodes[0].getpaymentrequest(paymentReq)["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000":
+        while self.nodes[0].getpaymentrequest(paymentReq)["state"] != 6:
             blocks = slow_gen(self.nodes[0], 1)
 
         sync_blocks(self.nodes)
@@ -117,7 +116,7 @@ def run_test(self):
         # check all wallets see the payout
 
         for x in range(self.num_nodes):
-            payoutBlockHash = self.nodes[x].getpaymentrequest(paymentReq)["paidOnBlock"]
+            payoutBlockHash = self.nodes[x].getpaymentrequest(paymentReq)["stateChangedOnBlock"]
             payoutBlock = self.nodes[x].getblock(payoutBlockHash)
             payoutHex = self.nodes[x].getrawtransaction(payoutBlock["tx"][0])
             payoutTx = self.nodes[x].decoderawtransaction(payoutHex)
@@ -143,15 +142,14 @@ def run_test(self):
         for x in range(self.num_nodes):
             assert(self.nodes[x].getbestblockhash() == bestBlockHash)
             assert(self.nodes[x].getpaymentrequest(paymentReq)["state"] == 1)
-            assert(self.nodes[x].getpaymentrequest(paymentReq)["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000")
 
-        while self.nodes[1].getpaymentrequest(paymentReq)["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000":
+        while self.nodes[1].getpaymentrequest(paymentReq)["state"] != 6:
             blocks = slow_gen(self.nodes[1], 1)
 
         sync_blocks(self.nodes)
 
         for x in range(self.num_nodes):
-            payoutBlockHash = self.nodes[x].getpaymentrequest(paymentReq)["paidOnBlock"]
+            payoutBlockHash = self.nodes[x].getpaymentrequest(paymentReq)["stateChangedOnBlock"]
             payoutBlock = self.nodes[x].getblock(payoutBlockHash)
             payoutHex = self.nodes[x].getrawtransaction(payoutBlock["tx"][0])
             payoutTx = self.nodes[x].decoderawtransaction(payoutHex)
@@ -169,36 +167,30 @@ def run_test(self):
         for x in range(self.num_nodes):
             assert(self.nodes[x].getbestblockhash() == bestBlockHash)
             assert(self.nodes[x].getpaymentrequest(paymentReq)["state"] == 1)
-            assert(self.nodes[x].getpaymentrequest(paymentReq)["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000")
+            assert(self.nodes[x].getpaymentrequest(paymentReq)["stateChangedOnBlock"] != "0000000000000000000000000000000000000000000000000000000000000000")
 
         # disconnect the nodes and generate the payout on each node
         url = urllib.parse.urlparse(self.nodes[1].url)
         self.nodes[0].disconnectnode(url.hostname+":"+str(p2p_port(1)))
 
         time.sleep(1)
 
-        while self.nodes[0].getpaymentrequest(paymentReq)["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000":
+        while self.nodes[0].getpaymentrequest(paymentReq)["state"] != 6:
             slow_gen(self.nodes[0], 1)
 
-        while self.nodes[1].getpaymentrequest(paymentReq)["paidOnBlock"] == "0000000000000000000000000000000000000000000000000000000000000000":
+        while self.nodes[1].getpaymentrequest(paymentReq)["state"] != 6:
             slow_gen(self.nodes[1], 1)
 
-        payoutBlockHash = self.nodes[0].getpaymentrequest(paymentReq)["paidOnBlock"]
-
-        # check both think they have paid out
-        for x in range(self.num_nodes):
-            assert(x == 0 or self.nodes[x].getpaymentrequest(paymentReq)["paidOnBlock"] != payoutBlockHash)
-
         slow_gen(self.nodes[1], 1)
         connect_nodes_bi(self.nodes,0,1) #reconnect the node
 
         sync_blocks(self.nodes)
 
-        payoutBlockHash = self.nodes[0].getpaymentrequest(paymentReq)["paidOnBlock"]
+        payoutBlockHash = self.nodes[0].getpaymentrequest(paymentReq)["stateChangedOnBlock"]
 
         # check both agree on the payout block
         for x in range(self.num_nodes):
-            assert(self.nodes[x].getpaymentrequest(paymentReq)["paidOnBlock"] == payoutBlockHash)
+            assert(self.nodes[x].getpaymentrequest(paymentReq)["stateChangedOnBlock"] == payoutBlockHash)
 
 
 if __name__ == '__main__':

qa/rpc-tests/cfund-paymentrequest-state-accept-expired-proposal.py

@@ -17,7 +17,8 @@ def __init__(self):
         self.num_nodes = 1
 
     def setup_network(self, split=False):
-        self.nodes = self.setup_nodes()
+        self.nodes = []
+        self.nodes.append(start_node(0, self.options.tmpdir, ["-debug=dao"]))
         self.is_network_split = split
 
     def run_test(self):
@@ -29,7 +30,7 @@ def run_test(self):
         proposal_amount = 10
 
         # Create a proposal and accept by voting
-        proposalid0 = self.nodes[0].createproposal(self.nodes[0].getnewaddress(), proposal_amount, proposal_duration, "test")["hash"]        
+        proposalid0 = self.nodes[0].createproposal(self.nodes[0].getnewaddress(), proposal_amount, proposal_duration, "test")["hash"]
         locked_before = self.nodes[0].cfundstats()["funds"]["locked"]
         end_cycle(self.nodes[0])
 
@@ -150,7 +151,7 @@ def run_test(self):
         self.nodes[0].paymentrequestvote(paymentrequestid0, "remove")
 
         assert(self.nodes[0].getpaymentrequest(paymentrequestid0)["state"] == 0)
-        assert(self.nodes[0].getpaymentrequest(paymentrequestid0)["status"] == "accepted waiting for end of voting period")
+        assert_equal(self.nodes[0].getpaymentrequest(paymentrequestid0)["status"], "accepted waiting for end of voting period")
         assert(self.nodes[0].cfundstats()["funds"]["locked"] == locked_accepted)
 
         time.sleep(0.2)
@@ -185,7 +186,6 @@ def run_test(self):
         end_cycle(self.nodes[0])
 
         # Locked amount should be 0, as this was the only payment request and the proposal was expired
-        print(self.nodes[0].cfundstats())
         assert(self.nodes[0].cfundstats()["funds"]["locked"] == 0)
         assert(self.nodes[0].getproposal(proposalid0)["status"] == "expired")
 

qa/rpc-tests/cfund-paymentrequest-state-accept.py

@@ -26,7 +26,7 @@ def run_test(self):
         self.nodes[0].donatefund(100)
 
         # Create a proposal and accept by voting
-        proposalid0 = self.nodes[0].createproposal(self.nodes[0].getnewaddress(), 10, 3600, "test")["hash"]        
+        proposalid0 = self.nodes[0].createproposal(self.nodes[0].getnewaddress(), 10, 3600, "test")["hash"]
         locked_before = self.nodes[0].cfundstats()["funds"]["locked"]
         end_cycle(self.nodes[0])
 
@@ -151,15 +151,15 @@ def run_test(self):
         assert(self.nodes[0].getpaymentrequest(paymentrequestid0)["status"] == "accepted")
         assert(self.nodes[0].cfundstats()["funds"]["locked"] == locked_after_payment)
 
-        # Check that paymentrequest remains in accepted state after the max number of cycles
+        # Check that paymentrequest moves to paid state
 
         cycles_to_expire = self.nodes[0].cfundstats()["consensus"]["maxCountVotingCyclePaymentRequests"]
 
         for idx in range(cycles_to_expire):
             end_cycle(self.nodes[0])
 
-        assert(self.nodes[0].getpaymentrequest(paymentrequestid0)["state"] == 1)
-        assert(self.nodes[0].getpaymentrequest(paymentrequestid0)["status"] == "accepted")
+        assert(self.nodes[0].getpaymentrequest(paymentrequestid0)["state"] == 6)
+        assert(self.nodes[0].getpaymentrequest(paymentrequestid0)["status"] == "paid")
 
         # Create multiple payment requests
 

qa/rpc-tests/cfund-rawtx-paymentrequest-create.py

@@ -105,7 +105,7 @@ def run_test(self):
         slow_gen(self.nodes[0], 10)
 
         # Check the payment request is paid out
-        assert (float(self.nodes[0].getproposal(self.goodProposalHash)["notPaidYet"]) == 9)
+        assert_equal(float(self.nodes[0].getproposal(self.goodProposalHash)["notPaidYet"]), 9)
         assert (float(self.nodes[0].cfundstats()["funds"]["locked"]) == 9)
 
 
@@ -203,7 +203,7 @@ def check_good_paymentrequest(self, paymentRequest):
         assert (paymentRequest['votingCycle'] == 0)
         assert (paymentRequest['status'] == 'pending')
         assert (paymentRequest['state'] == 0)
-        assert (paymentRequest['stateChangedOnBlock'] == '0000000000000000000000000000000000000000000000000000000000000000')
+        assert('stateChangedOnBlock' not in paymentRequest.keys())
 
     def send_raw_paymentrequest(self, amount, address, proposal_hash, description):
         amount = amount * 100000000