From 3602d313953b1ea6f393eeadac68cb61e97cda81 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Wed, 12 Jul 2017 10:29:02 -0400 Subject: [PATCH 01/26] [tests] remove direct testing on JSONRPCException from individual test cases --- test/functional/import-rescan.py | 30 +++++++++++--------------- test/functional/importmulti.py | 4 ++-- test/functional/signrawtransactions.py | 2 +- test/functional/test_framework/util.py | 10 ++++++++- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/test/functional/import-rescan.py b/test/functional/import-rescan.py index 6296a1a473f73..7922ed64963a6 100755 --- a/test/functional/import-rescan.py +++ b/test/functional/import-rescan.py @@ -19,9 +19,8 @@ happened previously. """ -from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (connect_nodes, sync_blocks, assert_equal, set_node_times) +from test_framework.util import (assert_raises_jsonrpc, connect_nodes, sync_blocks, assert_equal, set_node_times) import collections import enum @@ -36,21 +35,26 @@ class Variant(collections.namedtuple("Variant", "call data rescan prune")): """Helper for importing one key and verifying scanned transactions.""" + def try_rpc(self, func, *args, **kwargs): + if self.expect_disabled: + assert_raises_jsonrpc(-4, "Rescan is disabled in pruned mode", func, *args, **kwargs) + else: + return func(*args, **kwargs) + def do_import(self, timestamp): """Call one key import RPC.""" if self.call == Call.single: if self.data == Data.address: - response, error = try_rpc(self.node.importaddress, self.address["address"], self.label, - self.rescan == Rescan.yes) + response = self.try_rpc(self.node.importaddress, self.address["address"], self.label, + self.rescan == Rescan.yes) elif self.data == Data.pub: - response, error = try_rpc(self.node.importpubkey, self.address["pubkey"], self.label, - self.rescan == Rescan.yes) + response = self.try_rpc(self.node.importpubkey, self.address["pubkey"], self.label, + self.rescan == Rescan.yes) elif self.data == Data.priv: - response, error = try_rpc(self.node.importprivkey, self.key, self.label, self.rescan == Rescan.yes) + response = self.try_rpc(self.node.importprivkey, self.key, self.label, self.rescan == Rescan.yes) assert_equal(response, None) - assert_equal(error, {'message': 'Rescan is disabled in pruned mode', - 'code': -4} if self.expect_disabled else None) + elif self.call == Call.multi: response = self.node.importmulti([{ "scriptPubKey": { @@ -182,13 +186,5 @@ def run_test(self): else: variant.check() - -def try_rpc(func, *args, **kwargs): - try: - return func(*args, **kwargs), None - except JSONRPCException as e: - return None, e.error - - if __name__ == "__main__": ImportRescanTest().main() diff --git a/test/functional/importmulti.py b/test/functional/importmulti.py index 6907d64070524..f16c9ab97aad7 100755 --- a/test/functional/importmulti.py +++ b/test/functional/importmulti.py @@ -448,11 +448,11 @@ def run_test (self): # Bad or missing timestamps self.log.info("Should throw on invalid or missing timestamp values") - assert_raises_message(JSONRPCException, 'Missing required timestamp field for key', + assert_raises_jsonrpc(-3, 'Missing required timestamp field for key', self.nodes[1].importmulti, [{ "scriptPubKey": address['scriptPubKey'], }]) - assert_raises_message(JSONRPCException, 'Expected number or "now" timestamp value for key. got type string', + assert_raises_jsonrpc(-3, 'Expected number or "now" timestamp value for key. got type string', self.nodes[1].importmulti, [{ "scriptPubKey": address['scriptPubKey'], "timestamp": "", diff --git a/test/functional/signrawtransactions.py b/test/functional/signrawtransactions.py index 17d9ab793429f..bcab2d7b76efe 100755 --- a/test/functional/signrawtransactions.py +++ b/test/functional/signrawtransactions.py @@ -82,7 +82,7 @@ def script_verification_error_test(self): assert_equal(decodedRawTx["vin"][i]["vout"], inp["vout"]) # Make sure decoderawtransaction throws if there is extra data - assert_raises(JSONRPCException, self.nodes[0].decoderawtransaction, rawTx + "00") + assert_raises_jsonrpc(-22, "TX decode failed", self.nodes[0].decoderawtransaction, rawTx + "00") rawTxSigned = self.nodes[0].signrawtransaction(rawTx, scripts, privKeys) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 104973e14f808..e1b77d4768b30 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -101,6 +101,13 @@ def assert_raises_jsonrpc(code, message, fun, *args, **kwds): args*: positional arguments for the function. kwds**: named arguments for the function. """ + assert try_rpc(code, message, fun, *args, **kwds), "No exception raised" + +def try_rpc(code, message, fun, *args, **kwds): + """Tries to run an rpc command. + + Test against error code and message if the rpc fails. + Returns whether a JSONRPCException was raised.""" try: fun(*args, **kwds) except JSONRPCException as e: @@ -109,10 +116,11 @@ def assert_raises_jsonrpc(code, message, fun, *args, **kwds): raise AssertionError("Unexpected JSONRPC error code %i" % e.error["code"]) if (message is not None) and (message not in e.error['message']): raise AssertionError("Expected substring not found:" + e.error['message']) + return True except Exception as e: raise AssertionError("Unexpected exception raised: " + type(e).__name__) else: - raise AssertionError("No exception raised") + return False def assert_is_hex_string(string): try: From 58a946c6bc7cd93e6227aafa3c4598379e9c2404 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Wed, 12 Jul 2017 10:29:21 -0400 Subject: [PATCH 02/26] [tests] do not allow assert_raises_message to be called with JSONRPCException --- test/functional/test_framework/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index e1b77d4768b30..3acc1ae3eacca 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -53,6 +53,8 @@ def assert_raises(exc, fun, *args, **kwds): def assert_raises_message(exc, message, fun, *args, **kwds): try: fun(*args, **kwds) + except JSONRPCException: + raise AssertionError("Use assert_raises_jsonrpc() to test RPC failures") except exc as e: if message is not None and message not in e.error['message']: raise AssertionError("Expected substring not found:" + e.error['message']) From fa57eafd000d264e450ffe1cceb3f531dcf62f2f Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 25 Sep 2019 11:34:51 +0200 Subject: [PATCH 03/26] scripted-diff: rename assert_raises_jsonrpc to assert_raises_rpc error -BEGIN VERIFY SCRIPT- sed -i 's/assert_raises_jsonrpc/assert_raises_rpc_error/g' test/functional/*py test/functional/test_framework/*py -END VERIFY SCRIPT- --- test/functional/bip65-cltv-p2p.py | 2 +- test/functional/bip68-sequence.py | 10 +++++----- test/functional/bipdersig-p2p.py | 2 +- test/functional/blockchain.py | 4 ++-- test/functional/disablewallet.py | 4 ++-- test/functional/disconnect_ban.py | 12 ++++++------ test/functional/fundrawtransaction.py | 12 ++++++------ test/functional/import-rescan.py | 4 ++-- test/functional/importmulti.py | 4 ++-- test/functional/importprunedfunds.py | 4 ++-- test/functional/keypool.py | 6 +++--- test/functional/llmq-is-cl-conflicts.py | 6 +++--- test/functional/mempool_packages.py | 4 ++-- test/functional/mempool_reorg.py | 4 ++-- test/functional/mempool_spendcoinbase.py | 2 +- test/functional/merkle_blocks.py | 8 ++++---- test/functional/mining.py | 8 ++++---- test/functional/multiwallet.py | 6 +++--- test/functional/net.py | 4 ++-- test/functional/nulldummy.py | 4 ++-- test/functional/p2p-acceptblock.py | 2 +- test/functional/p2p-instantsend.py | 4 ++-- test/functional/prioritise_transaction.py | 2 +- test/functional/pruning.py | 8 ++++---- test/functional/rawtransactions.py | 12 ++++++------ test/functional/resendwallettransactions.py | 4 ++-- test/functional/rpc_getblockstats.py | 14 +++++++------- test/functional/rpcbind_test.py | 2 +- test/functional/rpcnamedargs.py | 4 ++-- test/functional/signrawtransactions.py | 2 +- test/functional/test_framework/util.py | 4 ++-- test/functional/wallet-encryption.py | 12 ++++++------ test/functional/wallet.py | 8 ++++---- test/functional/zapwallettxes.py | 4 ++-- 34 files changed, 96 insertions(+), 96 deletions(-) diff --git a/test/functional/bip65-cltv-p2p.py b/test/functional/bip65-cltv-p2p.py index 95a88fcffb3c3..37bab3729066b 100755 --- a/test/functional/bip65-cltv-p2p.py +++ b/test/functional/bip65-cltv-p2p.py @@ -124,7 +124,7 @@ def run_test(self): # First we show that this tx is valid except for CLTV by getting it # rejected from the mempool for exactly that reason. - assert_raises_jsonrpc(-26, '64: non-mandatory-script-verify-flag (Negative locktime)', self.nodes[0].sendrawtransaction, bytes_to_hex_str(spendtx.serialize()), True) + assert_raises_rpc_error(-26, '64: non-mandatory-script-verify-flag (Negative locktime)', self.nodes[0].sendrawtransaction, bytes_to_hex_str(spendtx.serialize()), True) # Now we verify that a block with this transaction is also invalid. block.vtx.append(spendtx) diff --git a/test/functional/bip68-sequence.py b/test/functional/bip68-sequence.py index 9da5ad998b3f2..a193b057ff12c 100755 --- a/test/functional/bip68-sequence.py +++ b/test/functional/bip68-sequence.py @@ -83,7 +83,7 @@ def test_disable_flag(self): tx2.vout = [CTxOut(int(value-self.relayfee*COIN), CScript([b'a']))] tx2.rehash() - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx2)) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx2)) # Setting the version back down to 1 should disable the sequence lock, # so this should be accepted. @@ -180,7 +180,7 @@ def test_sequence_lock_confirmed_inputs(self): if (using_sequence_locks and not should_pass): # This transaction should be rejected - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, rawtx) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, rawtx) else: # This raw transaction should be accepted self.nodes[0].sendrawtransaction(rawtx) @@ -227,7 +227,7 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): if (orig_tx.hash in node.getrawmempool()): # sendrawtransaction should fail if the tx is in the mempool - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, node.sendrawtransaction, ToHex(tx)) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, node.sendrawtransaction, ToHex(tx)) else: # sendrawtransaction should succeed if the tx is not in the mempool node.sendrawtransaction(ToHex(tx)) @@ -280,7 +280,7 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): tx5.vout[0].nValue += int(utxos[0]["amount"]*COIN) raw_tx5 = self.nodes[0].signrawtransaction(ToHex(tx5))["hex"] - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, raw_tx5) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, raw_tx5) # Test mempool-BIP68 consistency after reorg # @@ -353,7 +353,7 @@ def test_bip68_not_consensus(self): tx3.vout = [CTxOut(int(tx2.vout[0].nValue - self.relayfee*COIN), CScript([b'a']))] tx3.rehash() - assert_raises_jsonrpc(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx3)) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, ToHex(tx3)) # make a block that violates bip68; ensure that the tip updates tip = int(self.nodes[0].getbestblockhash(), 16) diff --git a/test/functional/bipdersig-p2p.py b/test/functional/bipdersig-p2p.py index e8067b49b07c5..52ab9714eb8ec 100755 --- a/test/functional/bipdersig-p2p.py +++ b/test/functional/bipdersig-p2p.py @@ -114,7 +114,7 @@ def run_test(self): # First we show that this tx is valid except for DERSIG by getting it # rejected from the mempool for exactly that reason. - assert_raises_jsonrpc(-26, '64: non-mandatory-script-verify-flag (Non-canonical DER signature)', self.nodes[0].sendrawtransaction, bytes_to_hex_str(spendtx.serialize()), True) + assert_raises_rpc_error(-26, '64: non-mandatory-script-verify-flag (Non-canonical DER signature)', self.nodes[0].sendrawtransaction, bytes_to_hex_str(spendtx.serialize()), True) # Now we verify that a block with this transaction is also invalid. block.vtx.append(spendtx) diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py index 0d44f5cb928e2..6d10fb1711067 100755 --- a/test/functional/blockchain.py +++ b/test/functional/blockchain.py @@ -25,7 +25,7 @@ from test_framework.util import ( assert_equal, assert_raises, - assert_raises_jsonrpc, + assert_raises_rpc_error, assert_is_hex_string, assert_is_hash_string, ) @@ -95,7 +95,7 @@ def _test_gettxoutsetinfo(self): def _test_getblockheader(self): node = self.nodes[0] - assert_raises_jsonrpc(-5, "Block not found", + assert_raises_rpc_error(-5, "Block not found", node.getblockheader, "nonsense") besthash = node.getbestblockhash() diff --git a/test/functional/disablewallet.py b/test/functional/disablewallet.py index 58a751b49d69c..8591652fc08e4 100755 --- a/test/functional/disablewallet.py +++ b/test/functional/disablewallet.py @@ -19,7 +19,7 @@ def set_test_params(self): def run_test (self): # Make sure wallet is really disabled - assert_raises_jsonrpc(-32601, 'Method not found', self.nodes[0].getwalletinfo) + assert_raises_rpc_error(-32601, 'Method not found', self.nodes[0].getwalletinfo) x = self.nodes[0].validateaddress('7TSBtVu959hGEGPKyHjJz9k55RpWrPffXz') assert(x['isvalid'] == False) x = self.nodes[0].validateaddress('ycwedq2f3sz2Yf9JqZsBCQPxp18WU3Hp4J') @@ -28,7 +28,7 @@ def run_test (self): # Checking mining to an address without a wallet. Generating to a valid address should succeed # but generating to an invalid address will fail. self.nodes[0].generatetoaddress(1, 'ycwedq2f3sz2Yf9JqZsBCQPxp18WU3Hp4J') - assert_raises_jsonrpc(-5, "Invalid address", self.nodes[0].generatetoaddress, 1, '7TSBtVu959hGEGPKyHjJz9k55RpWrPffXz') + assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].generatetoaddress, 1, '7TSBtVu959hGEGPKyHjJz9k55RpWrPffXz') if __name__ == '__main__': DisableWalletTest ().main () diff --git a/test/functional/disconnect_ban.py b/test/functional/disconnect_ban.py index e957cf3b3399f..7fe540679a928 100755 --- a/test/functional/disconnect_ban.py +++ b/test/functional/disconnect_ban.py @@ -7,7 +7,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, connect_nodes_bi, wait_until, set_node_times, @@ -34,14 +34,14 @@ def run_test(self): self.log.info("setban: fail to ban an already banned subnet") assert_equal(len(self.nodes[1].listbanned()), 1) - assert_raises_jsonrpc(-23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") + assert_raises_rpc_error(-23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") self.log.info("setban: fail to ban an invalid subnet") - assert_raises_jsonrpc(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") + assert_raises_rpc_error(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") assert_equal(len(self.nodes[1].listbanned()), 1) # still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 self.log.info("setban remove: fail to unban a non-banned subnet") - assert_raises_jsonrpc(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") + assert_raises_rpc_error(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") assert_equal(len(self.nodes[1].listbanned()), 1) self.log.info("setban remove: successfully unban subnet") @@ -78,10 +78,10 @@ def run_test(self): self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") address1 = self.nodes[0].getpeerinfo()[0]['addr'] node1 = self.nodes[0].getpeerinfo()[0]['addr'] - assert_raises_jsonrpc(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) + assert_raises_rpc_error(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) self.log.info("disconnectnode: fail to disconnect when calling with junk address") - assert_raises_jsonrpc(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") + assert_raises_rpc_error(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") self.log.info("disconnectnode: successfully disconnect node by address") address1 = self.nodes[0].getpeerinfo()[0]['addr'] diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index d2156a1137651..e5e86327b95e2 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -180,7 +180,7 @@ def run_test(self): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - assert_raises_jsonrpc(-3, "Unexpected key foo", self.nodes[2].fundrawtransaction, rawtx, {'foo':'bar'}) + assert_raises_rpc_error(-3, "Unexpected key foo", self.nodes[2].fundrawtransaction, rawtx, {'foo':'bar'}) ############################################################ # test a fundrawtransaction with an invalid change address # @@ -193,7 +193,7 @@ def run_test(self): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - assert_raises_jsonrpc(-5, "changeAddress must be a valid dash address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) + assert_raises_rpc_error(-5, "changeAddress must be a valid dash address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) ############################################################ # test a fundrawtransaction with a provided change address # @@ -207,7 +207,7 @@ def run_test(self): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) change = self.nodes[2].getnewaddress() - assert_raises_jsonrpc(-8, "changePosition out of bounds", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':change, 'changePosition':2}) + assert_raises_rpc_error(-8, "changePosition out of bounds", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':change, 'changePosition':2}) rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': change, 'changePosition': 0}) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) out = dec_tx['vout'][0] @@ -316,7 +316,7 @@ def run_test(self): rawtx = self.nodes[2].createrawtransaction(inputs, outputs) dec_tx = self.nodes[2].decoderawtransaction(rawtx) - assert_raises_jsonrpc(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx) + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx) ############################################################ #compare fee of a standard pubkeyhash transaction @@ -470,13 +470,13 @@ def run_test(self): rawtx = self.nodes[1].createrawtransaction(inputs, outputs) # fund a transaction that requires a new key for the change output # creating the key must be impossible because the wallet is locked - assert_raises_jsonrpc(-4, "Keypool ran out, please call keypoolrefill first", self.nodes[1].fundrawtransaction, rawtx) + assert_raises_rpc_error(-4, "Keypool ran out, please call keypoolrefill first", self.nodes[1].fundrawtransaction, rawtx) #refill the keypool self.nodes[1].walletpassphrase("test", 100) self.nodes[1].walletlock() - assert_raises_jsonrpc(-13, "walletpassphrase", self.nodes[1].sendtoaddress, self.nodes[0].getnewaddress(), 12) + assert_raises_rpc_error(-13, "walletpassphrase", self.nodes[1].sendtoaddress, self.nodes[0].getnewaddress(), 12) oldBalance = self.nodes[0].getbalance() diff --git a/test/functional/import-rescan.py b/test/functional/import-rescan.py index 7922ed64963a6..f78370c33ae29 100755 --- a/test/functional/import-rescan.py +++ b/test/functional/import-rescan.py @@ -20,7 +20,7 @@ """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (assert_raises_jsonrpc, connect_nodes, sync_blocks, assert_equal, set_node_times) +from test_framework.util import (assert_raises_rpc_error, connect_nodes, sync_blocks, assert_equal, set_node_times) import collections import enum @@ -37,7 +37,7 @@ class Variant(collections.namedtuple("Variant", "call data rescan prune")): def try_rpc(self, func, *args, **kwargs): if self.expect_disabled: - assert_raises_jsonrpc(-4, "Rescan is disabled in pruned mode", func, *args, **kwargs) + assert_raises_rpc_error(-4, "Rescan is disabled in pruned mode", func, *args, **kwargs) else: return func(*args, **kwargs) diff --git a/test/functional/importmulti.py b/test/functional/importmulti.py index f16c9ab97aad7..324f6458e3059 100755 --- a/test/functional/importmulti.py +++ b/test/functional/importmulti.py @@ -448,11 +448,11 @@ def run_test (self): # Bad or missing timestamps self.log.info("Should throw on invalid or missing timestamp values") - assert_raises_jsonrpc(-3, 'Missing required timestamp field for key', + assert_raises_rpc_error(-3, 'Missing required timestamp field for key', self.nodes[1].importmulti, [{ "scriptPubKey": address['scriptPubKey'], }]) - assert_raises_jsonrpc(-3, 'Expected number or "now" timestamp value for key. got type string', + assert_raises_rpc_error(-3, 'Expected number or "now" timestamp value for key. got type string', self.nodes[1].importmulti, [{ "scriptPubKey": address['scriptPubKey'], "timestamp": "", diff --git a/test/functional/importprunedfunds.py b/test/functional/importprunedfunds.py index 3815440a3844d..b1a74ba89215e 100755 --- a/test/functional/importprunedfunds.py +++ b/test/functional/importprunedfunds.py @@ -67,7 +67,7 @@ def run_test(self): self.sync_all() #Import with no affiliated address - assert_raises_jsonrpc(-5, "No addresses", self.nodes[1].importprunedfunds, rawtxn1, proof1) + assert_raises_rpc_error(-5, "No addresses", self.nodes[1].importprunedfunds, rawtxn1, proof1) balance1 = self.nodes[1].getbalance("", 0, False, True) assert_equal(balance1, Decimal(0)) @@ -98,7 +98,7 @@ def run_test(self): assert_equal(address_info['ismine'], True) #Remove transactions - assert_raises_jsonrpc(-8, "Transaction does not exist in wallet.", self.nodes[1].removeprunedfunds, txnid1) + assert_raises_rpc_error(-8, "Transaction does not exist in wallet.", self.nodes[1].removeprunedfunds, txnid1) balance1 = self.nodes[1].getbalance("*", 0, False, True) assert_equal(balance1, Decimal('0.075')) diff --git a/test/functional/keypool.py b/test/functional/keypool.py index df4249c44ac2d..fd71f96536c38 100755 --- a/test/functional/keypool.py +++ b/test/functional/keypool.py @@ -22,7 +22,7 @@ def run_test(self): # Keep creating keys addr = nodes[0].getnewaddress() - assert_raises_jsonrpc(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) + assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) # put three new keys in the keypool nodes[0].walletpassphrase('test', 12000) @@ -37,7 +37,7 @@ def run_test(self): # assert that three unique addresses were returned assert(len(addr) == 3) # the next one should fail - assert_raises_jsonrpc(-12, "Keypool ran out", nodes[0].getrawchangeaddress) + assert_raises_rpc_error(-12, "Keypool ran out", nodes[0].getrawchangeaddress) # refill keypool with three new addresses nodes[0].walletpassphrase('test', 1) @@ -50,7 +50,7 @@ def run_test(self): nodes[0].generate(1) nodes[0].generate(1) nodes[0].generate(1) - assert_raises_jsonrpc(-12, "Keypool ran out", nodes[0].generate, 1) + assert_raises_rpc_error(-12, "Keypool ran out", nodes[0].generate, 1) if __name__ == '__main__': KeyPoolTest().main() diff --git a/test/functional/llmq-is-cl-conflicts.py b/test/functional/llmq-is-cl-conflicts.py index 99bd98f3e3a98..fcade3de242f8 100755 --- a/test/functional/llmq-is-cl-conflicts.py +++ b/test/functional/llmq-is-cl-conflicts.py @@ -148,8 +148,8 @@ def test_chainlock_overrides_islock(self, test_block_conflict): # Lets verify that the ISLOCKs got pruned for node in self.nodes: - assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx1_txid, True) - assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx4_txid, True) + assert_raises_rpc_error(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx1_txid, True) + assert_raises_rpc_error(-5, "No such mempool or blockchain transaction", node.getrawtransaction, rawtx4_txid, True) rawtx = node.getrawtransaction(rawtx2_txid, True) assert(rawtx['chainlock']) assert(rawtx['instantlock']) @@ -186,7 +186,7 @@ def test_islock_overrides_nonchainlock(self): # Assert that the conflicting tx got mined and the locked TX is not valid assert(self.nodes[0].getrawtransaction(rawtx1_txid, True)['confirmations'] > 0) - assert_raises_jsonrpc(-25, "Missing inputs", self.nodes[0].sendrawtransaction, rawtx2) + assert_raises_rpc_error(-25, "Missing inputs", self.nodes[0].sendrawtransaction, rawtx2) # Send the ISLOCK, which should result in the last 2 blocks to be invalidated, even though the nodes don't know # the locked transaction yet diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index e5d6ce31cc7fb..8e18f311de6be 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -115,7 +115,7 @@ def run_test(self): assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 1000) # Adding one more transaction on to the chain should fail. - assert_raises_jsonrpc(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], txid, vout, value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], txid, vout, value, fee, 1) # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. @@ -167,7 +167,7 @@ def run_test(self): # Sending one more chained transaction will fail utxo = transaction_package.pop(0) - assert_raises_jsonrpc(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) + assert_raises_rpc_error(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) # TODO: check that node1's mempool is as expected diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index f75c507def261..fb9b7d77ef5f4 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -50,14 +50,14 @@ def run_test(self): timelock_tx = timelock_tx[:-8] + hex(self.nodes[0].getblockcount() + 2)[2:] + "000000" timelock_tx = self.nodes[0].signrawtransaction(timelock_tx)["hex"] # This will raise an exception because the timelock transaction is too immature to spend - assert_raises_jsonrpc(-26, "non-final", self.nodes[0].sendrawtransaction, timelock_tx) + assert_raises_rpc_error(-26, "non-final", self.nodes[0].sendrawtransaction, timelock_tx) # Broadcast and mine spend_102 and 103: spend_102_id = self.nodes[0].sendrawtransaction(spend_102_raw) spend_103_id = self.nodes[0].sendrawtransaction(spend_103_raw) self.nodes[0].generate(1) # Time-locked transaction is still too immature to spend - assert_raises_jsonrpc(-26,'non-final', self.nodes[0].sendrawtransaction, timelock_tx) + assert_raises_rpc_error(-26,'non-final', self.nodes[0].sendrawtransaction, timelock_tx) # Create 102_1 and 103_1: spend_102_1_raw = create_tx(self.nodes[0], spend_102_id, node1_address, 499.8) diff --git a/test/functional/mempool_spendcoinbase.py b/test/functional/mempool_spendcoinbase.py index 358bf392eaaad..ea297f150a89a 100755 --- a/test/functional/mempool_spendcoinbase.py +++ b/test/functional/mempool_spendcoinbase.py @@ -36,7 +36,7 @@ def run_test(self): spend_101_id = self.nodes[0].sendrawtransaction(spends_raw[0]) # coinbase at height 102 should be too immature to spend - assert_raises_jsonrpc(-26,"bad-txns-premature-spend-of-coinbase", self.nodes[0].sendrawtransaction, spends_raw[1]) + assert_raises_rpc_error(-26,"bad-txns-premature-spend-of-coinbase", self.nodes[0].sendrawtransaction, spends_raw[1]) # mempool should have just spend_101: assert_equal(self.nodes[0].getrawmempool(), [ spend_101_id ]) diff --git a/test/functional/merkle_blocks.py b/test/functional/merkle_blocks.py index 7c58654ef18ab..99158bf9433e4 100755 --- a/test/functional/merkle_blocks.py +++ b/test/functional/merkle_blocks.py @@ -39,7 +39,7 @@ def run_test(self): tx2 = self.nodes[0].createrawtransaction([node0utxos.pop()], {self.nodes[1].getnewaddress(): 500 - tx_fee}) txid2 = self.nodes[0].sendrawtransaction(self.nodes[0].signrawtransaction(tx2)["hex"]) # This will raise an exception because the transaction is not yet in a block - assert_raises_jsonrpc(-5, "Transaction not yet in block", self.nodes[0].gettxoutproof, [txid1]) + assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[0].gettxoutproof, [txid1]) self.nodes[0].generate(1) blockhash = self.nodes[0].getblockhash(chain_height + 1) @@ -65,11 +65,11 @@ def run_test(self): # We can't find the block from a fully-spent tx # Doesn't apply to Dash Core - we have txindex always on - # assert_raises_jsonrpc(-5, "Transaction not yet in block", self.nodes[2].gettxoutproof, [txid_spent]) + # assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[2].gettxoutproof, [txid_spent]) # We can get the proof if we specify the block assert_equal(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid_spent], blockhash)), [txid_spent]) # We can't get the proof if we specify a non-existent block - assert_raises_jsonrpc(-5, "Block not found", self.nodes[2].gettxoutproof, [txid_spent], "00000000000000000000000000000000") + assert_raises_rpc_error(-5, "Block not found", self.nodes[2].gettxoutproof, [txid_spent], "00000000000000000000000000000000") # We can get the proof if the transaction is unspent assert_equal(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid_unspent])), [txid_unspent]) # We can get the proof if we provide a list of transactions and one of them is unspent. The ordering of the list should not matter. @@ -78,7 +78,7 @@ def run_test(self): # We can always get a proof if we have a -txindex assert_equal(self.nodes[2].verifytxoutproof(self.nodes[3].gettxoutproof([txid_spent])), [txid_spent]) # We can't get a proof if we specify transactions from different blocks - assert_raises_jsonrpc(-5, "Not all transactions found in specified or retrieved block", self.nodes[2].gettxoutproof, [txid1, txid3]) + assert_raises_rpc_error(-5, "Not all transactions found in specified or retrieved block", self.nodes[2].gettxoutproof, [txid1, txid3]) if __name__ == '__main__': diff --git a/test/functional/mining.py b/test/functional/mining.py index 45fbaa145caf6..239f964e8eff0 100755 --- a/test/functional/mining.py +++ b/test/functional/mining.py @@ -68,7 +68,7 @@ def run_test(self): assert_template(node, block, None) self.log.info("submitblock: Test block decode failure") - assert_raises_jsonrpc(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) + assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) self.log.info("getblocktemplate: Test bad input hash for coinbase transaction") bad_block = copy.deepcopy(block) @@ -77,10 +77,10 @@ def run_test(self): assert_template(node, bad_block, 'bad-cb-missing') self.log.info("submitblock: Test invalid coinbase transaction") - assert_raises_jsonrpc(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) + assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) self.log.info("getblocktemplate: Test truncated final transaction") - assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'}) self.log.info("getblocktemplate: Test duplicate transaction") bad_block = copy.deepcopy(block) @@ -107,7 +107,7 @@ def run_test(self): bad_block_sn = bytearray(block.serialize()) assert_equal(bad_block_sn[TX_COUNT_OFFSET], 1) bad_block_sn[TX_COUNT_OFFSET] += 1 - assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'}) self.log.info("getblocktemplate: Test bad bits") bad_block = copy.deepcopy(block) diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py index b37f7a892c631..7ffa402100931 100755 --- a/test/functional/multiwallet.py +++ b/test/functional/multiwallet.py @@ -9,7 +9,7 @@ import os from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_jsonrpc +from test_framework.util import assert_equal, assert_raises_rpc_error class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): @@ -43,10 +43,10 @@ def run_test(self): w1.generate(1) # accessing invalid wallet fails - assert_raises_jsonrpc(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) + assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) # accessing wallet RPC without using wallet endpoint fails - assert_raises_jsonrpc(-19, "Wallet file not specified", self.nodes[0].getwalletinfo) + assert_raises_rpc_error(-19, "Wallet file not specified", self.nodes[0].getwalletinfo) # check w1 wallet balance w1_info = w1.getwalletinfo() diff --git a/test/functional/net.py b/test/functional/net.py index dedb2fe7cbef3..a8d8d2061386f 100755 --- a/test/functional/net.py +++ b/test/functional/net.py @@ -12,7 +12,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, connect_nodes_bi, p2p_port ) @@ -84,7 +84,7 @@ def _test_getaddednodeinfo(self): assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) # check that a non-existant node returns an error - assert_raises_jsonrpc(-24, "Node has not been added", + assert_raises_rpc_error(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') def _test_getpeerinfo(self): diff --git a/test/functional/nulldummy.py b/test/functional/nulldummy.py index 21a115a62ddd9..6053f21453c73 100755 --- a/test/functional/nulldummy.py +++ b/test/functional/nulldummy.py @@ -66,7 +66,7 @@ def run_test(self): self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") test2tx = self.create_transaction(self.nodes[0], txid2, self.ms_address, 47) trueDummy(test2tx) - assert_raises_jsonrpc(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test2tx.serialize()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test2tx.serialize()), True) self.log.info("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") self.block_submit(self.nodes[0], [test2tx], True) @@ -75,7 +75,7 @@ def run_test(self): test4tx = self.create_transaction(self.nodes[0], test2tx.hash, self.address, 46) test6txs=[CTransaction(test4tx)] trueDummy(test4tx) - assert_raises_jsonrpc(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize()), True) self.block_submit(self.nodes[0], [test4tx]) self.log.info("Test 6: NULLDUMMY compliant transactions should be accepted to mempool and in block after activation [432]") diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index 489b6328b7dbf..3fe38882cc9dd 100755 --- a/test/functional/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -172,7 +172,7 @@ def run_test(self): # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead for x in all_blocks[:-1]: self.nodes[0].getblock(x.hash) - assert_raises_jsonrpc(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) headers_message.headers.pop() # Ensure the last block is unrequested white_node.send_message(headers_message) # Send headers leading to tip diff --git a/test/functional/p2p-instantsend.py b/test/functional/p2p-instantsend.py index c22533ece09d9..b00452e849ded 100755 --- a/test/functional/p2p-instantsend.py +++ b/test/functional/p2p-instantsend.py @@ -104,14 +104,14 @@ def test_mempool_doublespend(self): reconnect_isolated_node(isolated, 0) for node in self.nodes: if node is not isolated: - assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", node.getrawtransaction, dblspnd_txid) + assert_raises_rpc_error(-5, "No such mempool or blockchain transaction", node.getrawtransaction, dblspnd_txid) # instantsend to receiver. The previously isolated node should prune the doublespend TX and request the correct # TX from other nodes. receiver_addr = receiver.getnewaddress() is_id = sender.sendtoaddress(receiver_addr, 0.9) for node in self.nodes: self.wait_for_instantlock(is_id, node) - assert_raises_jsonrpc(-5, "No such mempool or blockchain transaction", isolated.getrawtransaction, dblspnd_txid) + assert_raises_rpc_error(-5, "No such mempool or blockchain transaction", isolated.getrawtransaction, dblspnd_txid) # send coins back to the controller node without waiting for confirmations receiver.sendtoaddress(self.nodes[0].getnewaddress(), 0.9, "", "", True) assert_equal(receiver.getwalletinfo()["balance"], 0) diff --git a/test/functional/prioritise_transaction.py b/test/functional/prioritise_transaction.py index e3975f169dc02..677372f5530e1 100755 --- a/test/functional/prioritise_transaction.py +++ b/test/functional/prioritise_transaction.py @@ -101,7 +101,7 @@ def run_test(self): tx_id = self.nodes[0].decoderawtransaction(tx_hex)["txid"] # This will raise an exception due to min relay fee not being met - assert_raises_jsonrpc(-26, "66: min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) + assert_raises_rpc_error(-26, "66: min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) assert(tx_id not in self.nodes[0].getrawmempool()) # This is a less than 1000-byte transaction, so just set the fee diff --git a/test/functional/pruning.py b/test/functional/pruning.py index 8c37577936b7e..cc2b7b5d1673e 100755 --- a/test/functional/pruning.py +++ b/test/functional/pruning.py @@ -186,7 +186,7 @@ def reorg_test(self): def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away - assert_raises_jsonrpc(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) + assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) self.log.info("Will need to redownload block %d" % self.forkheight) # Verify that we have enough history to reorg back to the fork point @@ -233,7 +233,7 @@ def manual_test(self, node_number, use_timestamp): self.start_node(node_number, extra_args=["-litemode", "-txindex=0"]) node = self.nodes[node_number] assert_equal(node.getblockcount(), 995) - assert_raises_jsonrpc(-1, "not in prune mode", node.pruneblockchain, 500) + assert_raises_rpc_error(-1, "not in prune mode", node.pruneblockchain, 500) # now re-start in manual pruning mode self.stop_node(node_number) @@ -266,14 +266,14 @@ def has_block(index): return os.path.isfile(self.options.tmpdir + "/node{}/regtest/blocks/blk{:05}.dat".format(node_number, index)) # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000) - assert_raises_jsonrpc(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) + assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight) node.generate(6) assert_equal(node.getblockchaininfo()["blocks"], 1001) # negative heights should raise an exception - assert_raises_jsonrpc(-8, "Negative", node.pruneblockchain, -10) + assert_raises_rpc_error(-8, "Negative", node.pruneblockchain, -10) # height=100 too low to prune first block file so this is a no-op prune(100) diff --git a/test/functional/rawtransactions.py b/test/functional/rawtransactions.py index 09488eed5d7e2..e3aa33aa30133 100755 --- a/test/functional/rawtransactions.py +++ b/test/functional/rawtransactions.py @@ -48,7 +48,7 @@ def run_test(self): rawtx = self.nodes[2].signrawtransaction(rawtx) # This will raise an exception since there are missing inputs - assert_raises_jsonrpc(-25, "Missing inputs", self.nodes[2].sendrawtransaction, rawtx['hex']) + assert_raises_rpc_error(-25, "Missing inputs", self.nodes[2].sendrawtransaction, rawtx['hex']) ######################### # RAW TX MULTISIG TESTS # @@ -190,13 +190,13 @@ def run_test(self): assert_equal(self.nodes[0].getrawtransaction(txHash, True)["hex"], rawTxSigned['hex']) # 6. invalid parameters - supply txid and string "Flase" - assert_raises_jsonrpc(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, "Flase") + assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, "Flase") # 7. invalid parameters - supply txid and empty array - assert_raises_jsonrpc(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, []) + assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, []) # 8. invalid parameters - supply txid and empty dict - assert_raises_jsonrpc(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, {}) + assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, {}) inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 1000}] outputs = { self.nodes[0].getnewaddress() : 1 } @@ -207,12 +207,12 @@ def run_test(self): # 9. invalid parameters - sequence number out of range inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : -1}] outputs = { self.nodes[0].getnewaddress() : 1 } - assert_raises_jsonrpc(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) + assert_raises_rpc_error(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) # 10. invalid parameters - sequence number out of range inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967296}] outputs = { self.nodes[0].getnewaddress() : 1 } - assert_raises_jsonrpc(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) + assert_raises_rpc_error(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967294}] outputs = { self.nodes[0].getnewaddress() : 1 } diff --git a/test/functional/resendwallettransactions.py b/test/functional/resendwallettransactions.py index d6ba5913912d3..d959bb4c38364 100755 --- a/test/functional/resendwallettransactions.py +++ b/test/functional/resendwallettransactions.py @@ -5,7 +5,7 @@ """Test resendwallettransactions RPC.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_jsonrpc +from test_framework.util import assert_equal, assert_raises_rpc_error class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): @@ -14,7 +14,7 @@ def set_test_params(self): def run_test(self): # Should raise RPC_WALLET_ERROR (-4) if walletbroadcast is disabled. - assert_raises_jsonrpc(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions) + assert_raises_rpc_error(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions) # Should return an empty array if there aren't unconfirmed wallet transactions. self.stop_node(0) diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index ff9c82baf3964..bbbf7a4b31732 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -9,7 +9,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, ) import json import os @@ -148,9 +148,9 @@ def run_test(self): # Test invalid parameters raise the proper json exceptions tip = self.start_height + self.max_stat_pos - assert_raises_jsonrpc(-8, 'Target block height %d after current tip %d' % (tip+1, tip), + assert_raises_rpc_error(-8, 'Target block height %d after current tip %d' % (tip+1, tip), self.nodes[0].getblockstats, hash_or_height=tip+1) - assert_raises_jsonrpc(-8, 'Target block height %d is negative' % (-1), + assert_raises_rpc_error(-8, 'Target block height %d is negative' % (-1), self.nodes[0].getblockstats, hash_or_height=-1) # Make sure not valid stats aren't allowed @@ -162,18 +162,18 @@ def run_test(self): ['minfee', inv_sel_stat, 'maxfee'], ] for inv_stat in inv_stats: - assert_raises_jsonrpc(-8, 'Invalid selected statistic %s' % inv_sel_stat, + assert_raises_rpc_error(-8, 'Invalid selected statistic %s' % inv_sel_stat, self.nodes[0].getblockstats, hash_or_height=1, stats=inv_stat) # Make sure we aren't always returning inv_sel_stat as the culprit stat - assert_raises_jsonrpc(-8, 'Invalid selected statistic aaa%s' % inv_sel_stat, + assert_raises_rpc_error(-8, 'Invalid selected statistic aaa%s' % inv_sel_stat, self.nodes[0].getblockstats, hash_or_height=1, stats=['minfee' , 'aaa%s' % inv_sel_stat]) - assert_raises_jsonrpc(-8, 'One or more of the selected stats requires -txindex enabled', + assert_raises_rpc_error(-8, 'One or more of the selected stats requires -txindex enabled', self.nodes[1].getblockstats, hash_or_height=self.start_height + self.max_stat_pos) # Mainchain's genesis block shouldn't be found on regtest - assert_raises_jsonrpc(-5, 'Block not found', self.nodes[0].getblockstats, + assert_raises_rpc_error(-5, 'Block not found', self.nodes[0].getblockstats, hash_or_height='000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f') if __name__ == '__main__': diff --git a/test/functional/rpcbind_test.py b/test/functional/rpcbind_test.py index 0cf64beebd408..0e8c3fa209ee8 100755 --- a/test/functional/rpcbind_test.py +++ b/test/functional/rpcbind_test.py @@ -101,7 +101,7 @@ def run_test(self): # Check that with invalid rpcallowip, we are denied self.run_allowip_test([non_loopback_ip], non_loopback_ip, defaultport) - assert_raises_jsonrpc(-342, "non-JSON HTTP response with '403 Forbidden' from server", self.run_allowip_test, ['1.1.1.1'], non_loopback_ip, defaultport) + assert_raises_rpc_error(-342, "non-JSON HTTP response with '403 Forbidden' from server", self.run_allowip_test, ['1.1.1.1'], non_loopback_ip, defaultport) if __name__ == '__main__': RPCBindTest().main() diff --git a/test/functional/rpcnamedargs.py b/test/functional/rpcnamedargs.py index da61cc66e6854..0fb0a62afd02b 100755 --- a/test/functional/rpcnamedargs.py +++ b/test/functional/rpcnamedargs.py @@ -7,7 +7,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, ) class NamedArgumentTest(BitcoinTestFramework): @@ -19,7 +19,7 @@ def run_test(self): h = node.help(command='getinfo') assert(h.startswith('getinfo\n')) - assert_raises_jsonrpc(-8, 'Unknown named parameter', node.help, random='getinfo') + assert_raises_rpc_error(-8, 'Unknown named parameter', node.help, random='getinfo') h = node.getblockhash(height=0) node.getblock(blockhash=h) diff --git a/test/functional/signrawtransactions.py b/test/functional/signrawtransactions.py index bcab2d7b76efe..96a3b6382198e 100755 --- a/test/functional/signrawtransactions.py +++ b/test/functional/signrawtransactions.py @@ -82,7 +82,7 @@ def script_verification_error_test(self): assert_equal(decodedRawTx["vin"][i]["vout"], inp["vout"]) # Make sure decoderawtransaction throws if there is extra data - assert_raises_jsonrpc(-22, "TX decode failed", self.nodes[0].decoderawtransaction, rawTx + "00") + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, rawTx + "00") rawTxSigned = self.nodes[0].signrawtransaction(rawTx, scripts, privKeys) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 3acc1ae3eacca..f724348d556cb 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -54,7 +54,7 @@ def assert_raises_message(exc, message, fun, *args, **kwds): try: fun(*args, **kwds) except JSONRPCException: - raise AssertionError("Use assert_raises_jsonrpc() to test RPC failures") + raise AssertionError("Use assert_raises_rpc_error() to test RPC failures") except exc as e: if message is not None and message not in e.error['message']: raise AssertionError("Expected substring not found:" + e.error['message']) @@ -87,7 +87,7 @@ def assert_raises_process_error(returncode, output, fun, *args, **kwds): else: raise AssertionError("No exception raised") -def assert_raises_jsonrpc(code, message, fun, *args, **kwds): +def assert_raises_rpc_error(code, message, fun, *args, **kwds): """Run an RPC and verify that a specific JSONRPC exception code and message is raised. Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException diff --git a/test/functional/wallet-encryption.py b/test/functional/wallet-encryption.py index ce1e7744e903f..db62e1e30f489 100755 --- a/test/functional/wallet-encryption.py +++ b/test/functional/wallet-encryption.py @@ -9,7 +9,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, ) class WalletEncryptionTest(BitcoinTestFramework): @@ -32,7 +32,7 @@ def run_test(self): self.start_node(0) # Test that the wallet is encrypted - assert_raises_jsonrpc(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) # Check that walletpassphrase works self.nodes[0].walletpassphrase(passphrase, 2) @@ -40,20 +40,20 @@ def run_test(self): # Check that the timeout is right time.sleep(2) - assert_raises_jsonrpc(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) # Test wrong passphrase - assert_raises_jsonrpc(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase + "wrong", 10) + assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase + "wrong", 10) # Test walletlock self.nodes[0].walletpassphrase(passphrase, 84600) assert_equal(privkey, self.nodes[0].dumpprivkey(address)) self.nodes[0].walletlock() - assert_raises_jsonrpc(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) # Test passphrase changes self.nodes[0].walletpassphrasechange(passphrase, passphrase2) - assert_raises_jsonrpc(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase, 10) + assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase, 10) self.nodes[0].walletpassphrase(passphrase2, 10) assert_equal(privkey, self.nodes[0].dumpprivkey(address)) diff --git a/test/functional/wallet.py b/test/functional/wallet.py index 4e9737af4b1a5..d4d7aad7f3afc 100755 --- a/test/functional/wallet.py +++ b/test/functional/wallet.py @@ -101,7 +101,7 @@ def run_test(self): unspent_0 = self.nodes[2].listunspent()[0] unspent_0 = {"txid": unspent_0["txid"], "vout": unspent_0["vout"]} self.nodes[2].lockunspent(False, [unspent_0]) - assert_raises_jsonrpc(-4, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 200) + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 200) assert_equal([unspent_0], self.nodes[2].listlockunspent()) self.nodes[2].lockunspent(True, [unspent_0]) assert_equal(len(self.nodes[2].listlockunspent()), 0) @@ -289,10 +289,10 @@ def run_test(self): assert_equal(txObj['amount'], Decimal('-0.0001')) # This will raise an exception because the amount type is wrong - assert_raises_jsonrpc(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") + assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") # This will raise an exception since generate does not accept a string - assert_raises_jsonrpc(-1, "not an integer", self.nodes[0].generate, "2") + assert_raises_rpc_error(-1, "not an integer", self.nodes[0].generate, "2") # Import address and private key to check correct behavior of spendable unspents # 1. Send some coins to generate new UTXO @@ -425,7 +425,7 @@ def run_test(self): node0_balance = self.nodes[0].getbalance() # With walletrejectlongchains we will not create the tx and store it in our wallet. - assert_raises_jsonrpc(-4, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) + assert_raises_rpc_error(-4, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) # Verify nothing new in wallet assert_equal(total_txs, len(self.nodes[0].listtransactions("*",99999))) diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index 1233c82fc5713..666925ac39c0a 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -17,7 +17,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_raises_jsonrpc, + assert_raises_rpc_error, ) from test_framework.mininode import wait_until @@ -72,7 +72,7 @@ def run_test(self): assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) # This will raise an exception because the unconfirmed transaction has been zapped - assert_raises_jsonrpc(-5, 'Invalid or non-wallet transaction id', self.nodes[0].gettransaction, txid2) + assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', self.nodes[0].gettransaction, txid2) if __name__ == '__main__': ZapWalletTXesTest().main() From b6116b20c0ec30c47b192f09618222165f6d88bd Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 1 Nov 2017 12:26:23 -0400 Subject: [PATCH 04/26] Merge #11376: Ensure backupwallet fails when attempting to backup to source file 5d465e396 Ensure backupwallet fails when attempting to backup to source file (Tomas van der Wansem) Pull request description: Previous behaviour was to destroy the wallet (to zero-length) This fixes #11375 Tree-SHA512: bfd1738659b15e3f23b6bbdf55ec12269c62c820bf701daec19500b52bd5845bb5516733c6f76f36197eb155182a8a35dc239ad4de2ef1e59bbb0f124a455759 --- src/wallet/db.cpp | 5 +++++ test/functional/walletbackup.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 8459d6ad47be5..ff5b342ca04a1 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -673,6 +673,11 @@ bool CWalletDBWrapper::Backup(const std::string& strDest) pathDest /= strFile; try { + if (fs::equivalent(pathSrc, pathDest)) { + LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); + return false; + } + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); LogPrintf("copied %s to %s\n", strFile, pathDest.string()); return true; diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py index 3c5ec69b3ada7..430fb47c5a6d4 100755 --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -192,6 +192,16 @@ def run_test(self): assert_equal(self.nodes[1].getbalance(), balance1) assert_equal(self.nodes[2].getbalance(), balance2) + # Backup to source wallet file must fail + sourcePaths = [ + tmpdir + "/node0/regtest/wallet.dat", + tmpdir + "/node0/./regtest/wallet.dat", + tmpdir + "/node0/regtest/", + tmpdir + "/node0/regtest"] + + for sourcePath in sourcePaths: + assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) + if __name__ == '__main__': WalletBackupTest().main() From d05801c8a21661967a2931948a298655ea011eae Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 18 Oct 2017 15:38:07 +0200 Subject: [PATCH 05/26] Merge #11492: [wallet] Fix leak in CDB constructor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7104de8 [wallet] Fix leak in CDB constructor (João Barbosa) Pull request description: First commit fixes a minor leak. Second commit improves the constructor in the failure cases. Tree-SHA512: 5165413d60ed9fc28203c9fe128adbba03a9ea9e9aa3734d9ea2522dafd815ba0fb8b90fd0809dbc06eb3ad360e7764de01dadf653ade3350fe86f6b8f04bc90 --- src/wallet/db.cpp | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index ff5b342ca04a1..4fb6215ed66fc 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -379,45 +379,43 @@ CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb if (!env->Open(GetDataDir())) throw std::runtime_error("CDB: Failed to open database environment."); - strFile = strFilename; - ++env->mapFileUseCount[strFile]; - pdb = env->mapDb[strFile]; + pdb = env->mapDb[strFilename]; if (pdb == nullptr) { int ret; - pdb = new Db(env->dbenv, 0); + std::unique_ptr pdb_temp(new Db(env->dbenv, 0)); bool fMockDb = env->IsMock(); if (fMockDb) { - DbMpoolFile* mpf = pdb->get_mpf(); + DbMpoolFile* mpf = pdb_temp->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); - if (ret != 0) - throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile)); + if (ret != 0) { + throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFilename)); + } } - ret = pdb->open(nullptr, // Txn pointer - fMockDb ? nullptr : strFile.c_str(), // Filename - fMockDb ? strFile.c_str() : "main", // Logical db name - DB_BTREE, // Database type - nFlags, // Flags + ret = pdb_temp->open(nullptr, // Txn pointer + fMockDb ? nullptr : strFilename.c_str(), // Filename + fMockDb ? strFilename.c_str() : "main", // Logical db name + DB_BTREE, // Database type + nFlags, // Flags 0); if (ret != 0) { - delete pdb; - pdb = nullptr; - --env->mapFileUseCount[strFile]; - strFile = ""; throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); } + pdb = pdb_temp.release(); + env->mapDb[strFilename] = pdb; + if (fCreate && !Exists(std::string("version"))) { bool fTmp = fReadOnly; fReadOnly = false; WriteVersion(CLIENT_VERSION); fReadOnly = fTmp; } - - env->mapDb[strFile] = pdb; } + ++env->mapFileUseCount[strFilename]; + strFile = strFilename; } } From b155130224ca7c2ca9f0970b4c0d0d4e464698ba Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 19 Oct 2017 18:12:59 +0200 Subject: [PATCH 06/26] Merge #11476: Avoid opening copied wallet databases simultaneously 478a89c Avoid opening copied wallet databases simultaneously (Russell Yanofsky) Pull request description: Make sure wallet databases have unique fileids. If they don't, throw an error. BDB caches do not work properly when more than one open database has the same fileid, because values written to one database may show up in reads to other databases. Bitcoin will never create different databases with the same fileid, but users can create them by manually copying database files. BDB caching bug was reported by @dooglus in https://github.com/bitcoin/bitcoin/issues/11429 Tree-SHA512: e7635dc81a181801f42324b72fe9e0a2a7dd00b1dcf5abcbf27fa50938eb9a1fc3065c2321326c3456c48c29ae6504353b02f3d46e6eb2f7b09e46d8fe24388d --- src/wallet/db.cpp | 35 ++++++++++++++++++++++++++++++++++ test/functional/multiwallet.py | 6 ++++++ 2 files changed, 41 insertions(+) diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 4fb6215ed66fc..192ae5daebdc9 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -20,6 +20,40 @@ #include +namespace { +//! Make sure database has a unique fileid within the environment. If it +//! doesn't, throw an error. BDB caches do not work properly when more than one +//! open database has the same fileid (values written to one database may show +//! up in reads to other databases). +//! +//! BerkeleyDB generates unique fileids by default +//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), +//! so bitcoin should never create different databases with the same fileid, but +//! this error can be triggered if users manually copy database files. +void CheckUniqueFileid(const CDBEnv& env, const std::string& filename, Db& db) +{ + if (env.IsMock()) return; + + u_int8_t fileid[DB_FILE_ID_LEN]; + int ret = db.get_mpf()->get_fileid(fileid); + if (ret != 0) { + throw std::runtime_error(strprintf("CDB: Can't open database %s (get_fileid failed with %d)", filename, ret)); + } + + for (const auto& item : env.mapDb) { + u_int8_t item_fileid[DB_FILE_ID_LEN]; + if (item.second && item.second->get_mpf()->get_fileid(item_fileid) == 0 && + memcmp(fileid, item_fileid, sizeof(fileid)) == 0) { + const char* item_filename = nullptr; + item.second->get_dbname(&item_filename, nullptr); + throw std::runtime_error(strprintf("CDB: Can't open database %s (duplicates fileid %s from %s)", filename, + HexStr(std::begin(item_fileid), std::end(item_fileid)), + item_filename ? item_filename : "(unknown database)")); + } + } +} +} // namespace + // // CDB // @@ -403,6 +437,7 @@ CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb if (ret != 0) { throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); } + CheckUniqueFileid(*env, strFilename, *pdb_temp); pdb = pdb_temp.release(); env->mapDb[strFilename] = pdb; diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py index 7ffa402100931..04c912fee66dc 100755 --- a/test/functional/multiwallet.py +++ b/test/functional/multiwallet.py @@ -7,6 +7,7 @@ Verify that a bitcoind node can load multiple wallet files """ import os +import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error @@ -29,6 +30,11 @@ def run_test(self): os.mkdir(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w11')) self.assert_start_raises_init_error(0, ['-wallet=w11'], 'Error loading wallet w11. -wallet filename must be a regular file.') + # should not initialize if one wallet is a copy of another + shutil.copyfile(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w2'), + os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w22')) + self.assert_start_raises_init_error(0, ['-wallet=w2', '-wallet=w22'], 'duplicates fileid') + # should not initialize if wallet file is a symlink os.symlink(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w1'), os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w12')) self.assert_start_raises_init_error(0, ['-wallet=w12'], 'Error loading wallet w12. -wallet filename must be a regular file.') From f18fa576b98f29885e265f7ab83bbc0041e555ae Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 18 Oct 2017 16:52:44 +0200 Subject: [PATCH 07/26] Merge #11472: qa: Make tmpdir option an absolute path, misc cleanup fafa003 qa: Remove never used return value of sync_with_ping (MarcoFalke) fa9de37 qa: Make tmpdir option an absolute path (MarcoFalke) Pull request description: This should fix issues with the multiwallet test and its symlinks when the tmpdir is a relative path. Rather than fixing os.symlink to work with paths relative to a directory descriptor, which does not work on Windows, normalize the path instead. Tree-SHA512: 189690f3d065ea2f0f48e06775c86d513d0916c7c86312432e8e16df160e65539e288c2bd53d49a4180735fa940f6fcd52b506ccd7d9815651a9b1a69850dda6 --- test/functional/p2p-acceptblock.py | 9 ++++++--- test/functional/test_framework/mininode.py | 1 - test/functional/test_framework/test_framework.py | 3 +++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index 3fe38882cc9dd..a007ddb4feeb6 100755 --- a/test/functional/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -103,7 +103,8 @@ def run_test(self): test_node.send_message(msg_block(blocks_h2[0])) white_node.send_message(msg_block(blocks_h2[1])) - [ x.sync_with_ping() for x in [test_node, white_node] ] + for x in [test_node, white_node]: + x.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) assert_equal(self.nodes[1].getblockcount(), 2) self.log.info("First height 2 block accepted by both nodes") @@ -116,7 +117,8 @@ def run_test(self): test_node.send_message(msg_block(blocks_h2f[0])) white_node.send_message(msg_block(blocks_h2f[1])) - [ x.sync_with_ping() for x in [test_node, white_node] ] + for x in [test_node, white_node]: + x.sync_with_ping() for x in self.nodes[0].getchaintips(): if x['hash'] == blocks_h2f[0].hash: assert_equal(x['status'], "headers-only") @@ -135,7 +137,8 @@ def run_test(self): test_node.send_message(msg_block(blocks_h3[0])) white_node.send_message(msg_block(blocks_h3[1])) - [ x.sync_with_ping() for x in [test_node, white_node] ] + for x in [test_node, white_node]: + x.sync_with_ping() # Since the earlier block was not processed by node0, the new block # can't be fully validated. for x in self.nodes[0].getchaintips(): diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 8d9f2fdfe4772..ec3334f04093e 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1656,7 +1656,6 @@ def sync_with_ping(self, timeout=60): test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter wait_until(test_function, timeout=timeout, lock=mininode_lock) self.ping_counter += 1 - return True # The actual NodeConn class # This class provides an interface for a p2p connection to a specified node diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 457ff763e66a1..c59b6277d2234 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -110,8 +110,11 @@ def main(self): check_json_precision() + self.options.cachedir = os.path.abspath(self.options.cachedir) + # Set up temp directory and start logging if self.options.tmpdir: + self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: self.options.tmpdir = tempfile.mkdtemp(prefix="test") From 9938dd83d43a218f88b0eacd958405a1d4930963 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 6 Sep 2017 18:49:43 +0200 Subject: [PATCH 08/26] Merge #10357: Allow setting nMinimumChainWork on command line eac64bb7a [qa] Test nMinimumChainWork (Suhas Daftuar) 0311836f6 Allow setting nMinimumChainWork on command line (Suhas Daftuar) Pull request description: As discussed briefly here: https://botbot.me/freenode/bitcoin-core-dev/2017-02-28/?msg=81712308&page=4 This adds a hidden command line option for setting `nMinimumChainWork`, which allows us to test this parameter in our functional tests, as well as allowing for niche use cases like syncing nodes that are otherwise disconnected from the network. See also #10345, which proposes a new use of `nMinimumChainWork`. Tree-SHA512: fe4d8f4f289697615c98d8760f1cc74c076110310ea0b5b875fcab78c127da9195b4eb84148aebacc7606c246e5773d3f13bd5d9559d0a8bffac20a3a28c62df --- src/init.cpp | 17 +++++++ src/net_processing.cpp | 2 +- src/test/util_tests.cpp | 25 ++++++++++ src/utilstrencodings.cpp | 13 ++++++ src/utilstrencodings.h | 6 +++ src/validation.cpp | 5 +- src/validation.h | 3 ++ test/functional/minchainwork.py | 81 +++++++++++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 9 files changed, 150 insertions(+), 3 deletions(-) create mode 100755 test/functional/minchainwork.py diff --git a/src/init.cpp b/src/init.cpp index 57048ba491bab..16a2bb80f545d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -462,6 +462,9 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-maxorphantx=", strprintf(_("Keep at most unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=", strprintf(_("Keep the transaction memory pool below megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=", strprintf(_("Do not keep transactions in the mempool longer than hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); + if (showDebug) { + strUsage += HelpMessageOpt("-minimumchainwork=", strprintf("Minimum work assumed to exist on a valid chain in hex (default: %s, testnet: %s)", defaultChainParams->GetConsensus().nMinimumChainWork.GetHex(), testnetChainParams->GetConsensus().nMinimumChainWork.GetHex())); + } strUsage += HelpMessageOpt("-persistmempool", strprintf(_("Whether to save the mempool on shutdown and load on restart (default: %u)"), DEFAULT_PERSIST_MEMPOOL)); strUsage += HelpMessageOpt("-blockreconstructionextratxn=", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); strUsage += HelpMessageOpt("-par=", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), @@ -1190,6 +1193,20 @@ bool AppInitParameterInteraction() else LogPrintf("Validating signatures for all blocks.\n"); + if (gArgs.IsArgSet("-minimumchainwork")) { + const std::string minChainWorkStr = gArgs.GetArg("-minimumchainwork", ""); + if (!IsHexNumber(minChainWorkStr)) { + return InitError(strprintf("Invalid non-hex (%s) minimum chain work value specified", minChainWorkStr)); + } + nMinimumChainWork = UintToArith256(uint256S(minChainWorkStr)); + } else { + nMinimumChainWork = UintToArith256(chainparams.GetConsensus().nMinimumChainWork); + } + LogPrintf("Setting nMinimumChainWork=%s\n", nMinimumChainWork.GetHex()); + if (nMinimumChainWork < UintToArith256(chainparams.GetConsensus().nMinimumChainWork)) { + LogPrintf("Warning: nMinimumChainWork set below default value of %s\n", chainparams.GetConsensus().nMinimumChainWork.GetHex()); + } + // mempool limits int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t nMempoolSizeMin = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2d3dc4e8d06f8..cf832afac7ba0 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -501,7 +501,7 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vectorpindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < UintToArith256(consensusParams.nMinimumChainWork)) { + if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { // This peer has nothing interesting. return; } diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index e1de704f4eeb2..9cf9899c212ff 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -253,6 +253,31 @@ BOOST_AUTO_TEST_CASE(util_IsHex) BOOST_CHECK(!IsHex("0x0000")); } +BOOST_AUTO_TEST_CASE(util_IsHexNumber) +{ + BOOST_CHECK(IsHexNumber("0x0")); + BOOST_CHECK(IsHexNumber("0")); + BOOST_CHECK(IsHexNumber("0x10")); + BOOST_CHECK(IsHexNumber("10")); + BOOST_CHECK(IsHexNumber("0xff")); + BOOST_CHECK(IsHexNumber("ff")); + BOOST_CHECK(IsHexNumber("0xFfa")); + BOOST_CHECK(IsHexNumber("Ffa")); + BOOST_CHECK(IsHexNumber("0x00112233445566778899aabbccddeeffAABBCCDDEEFF")); + BOOST_CHECK(IsHexNumber("00112233445566778899aabbccddeeffAABBCCDDEEFF")); + + BOOST_CHECK(!IsHexNumber("")); // empty string not allowed + BOOST_CHECK(!IsHexNumber("0x")); // empty string after prefix not allowed + BOOST_CHECK(!IsHexNumber("0x0 ")); // no spaces at end, + BOOST_CHECK(!IsHexNumber(" 0x0")); // or beginning, + BOOST_CHECK(!IsHexNumber("0x 0")); // or middle, + BOOST_CHECK(!IsHexNumber(" ")); // etc. + BOOST_CHECK(!IsHexNumber("0x0ga")); // invalid character + BOOST_CHECK(!IsHexNumber("x0")); // broken prefix + BOOST_CHECK(!IsHexNumber("0x0x00")); // two prefixes not allowed + +} + BOOST_AUTO_TEST_CASE(util_seed_insecure_rand) { SeedInsecureRand(true); diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 5dbf4fd7e0343..4ac0e025e55cc 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -65,6 +65,19 @@ bool IsHex(const std::string& str) return (str.size() > 0) && (str.size()%2 == 0); } +bool IsHexNumber(const std::string& str) +{ + size_t starting_location = 0; + if (str.size() > 2 && *str.begin() == '0' && *(str.begin()+1) == 'x') { + starting_location = 2; + } + for (auto c : str.substr(starting_location)) { + if (HexDigit(c) < 0) return false; + } + // Return false for empty string or "0x". + return (str.size() > starting_location); +} + std::vector ParseHex(const char* psz) { // convert hex dump to vector diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index a1475f3be9318..c5b9f985fa2b4 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -38,7 +38,13 @@ std::string SanitizeString(const std::string& str, int rule = SAFE_CHARS_DEFAULT std::vector ParseHex(const char* psz); std::vector ParseHex(const std::string& str); signed char HexDigit(char c); +/* Returns true if each character in str is a hex character, and has an even + * number of hex digits.*/ bool IsHex(const std::string& str); +/** +* Return true if the string is a hex number, optionally prefixed with "0x" +*/ +bool IsHexNumber(const std::string& str); std::vector DecodeBase64(const char* p, bool* pfInvalid = nullptr); std::string DecodeBase64(const std::string& str); std::string EncodeBase64(const unsigned char* pch, size_t len); diff --git a/src/validation.cpp b/src/validation.cpp index e55c4df742ef0..11edb19094095 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -98,6 +98,7 @@ std::atomic fDIP0001ActiveAtTip{false}; std::atomic fDIP0003ActiveAtTip{false}; uint256 hashAssumeValid; +arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; @@ -1075,7 +1076,7 @@ bool IsInitialBlockDownload() const CChainParams& chainParams = Params(); if (chainActive.Tip() == nullptr) return true; - if (chainActive.Tip()->nChainWork < UintToArith256(chainParams.GetConsensus().nMinimumChainWork)) + if (chainActive.Tip()->nChainWork < nMinimumChainWork) return true; if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; @@ -1838,7 +1839,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd if (it != mapBlockIndex.end()) { if (it->second->GetAncestor(pindex->nHeight) == pindex && pindexBestHeader->GetAncestor(pindex->nHeight) == pindex && - pindexBestHeader->nChainWork >= UintToArith256(chainparams.GetConsensus().nMinimumChainWork)) { + pindexBestHeader->nChainWork >= nMinimumChainWork) { // This block is a member of the assumed verified chain and an ancestor of the best header. // The equivalent time check discourages hash power from extorting the network via DOS attack // into accepting an invalid block through telling users they must manually set assumevalid. diff --git a/src/validation.h b/src/validation.h index 580b709b26a53..8e6ad606aa903 100644 --- a/src/validation.h +++ b/src/validation.h @@ -192,6 +192,9 @@ extern std::atomic fDIP0001ActiveAtTip; /** Block hash whose ancestors we will assume to have valid scripts without checking them. */ extern uint256 hashAssumeValid; +/** Minimum work we will assume exists on some valid chain. */ +extern arith_uint256 nMinimumChainWork; + /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; diff --git a/test/functional/minchainwork.py b/test/functional/minchainwork.py new file mode 100755 index 0000000000000..c7579d2548e46 --- /dev/null +++ b/test/functional/minchainwork.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test logic for setting nMinimumChainWork on command line. + +Nodes don't consider themselves out of "initial block download" until +their active chain has more work than nMinimumChainWork. + +Nodes don't download blocks from a peer unless the peer's best known block +has more work than nMinimumChainWork. + +While in initial block download, nodes won't relay blocks to their peers, so +test that this parameter functions as intended by verifying that block relay +only succeeds past a given node once its nMinimumChainWork has been exceeded. +""" + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import sync_blocks, connect_nodes, assert_equal + +# 2 hashes required per regtest block (with no difficulty adjustment) +REGTEST_WORK_PER_BLOCK = 2 + +class MinimumChainWorkTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 3 + self.extra_args = [[], ["-minimumchainwork=0x65"], ["-minimumchainwork=0x65"]] + self.node_min_work = [0, 101, 101] + + def setup_network(self): + # This test relies on the chain setup being: + # node0 <- node1 <- node2 + # Before leaving IBD, nodes prefer to download blocks from outbound + # peers, so ensure that we're mining on an outbound peer and testing + # block relay to inbound peers. + self.setup_nodes() + for i in range(self.num_nodes-1): + connect_nodes(self.nodes[i+1], i) + + def run_test(self): + # Start building a chain on node0. node2 shouldn't be able to sync until node1's + # minchainwork is exceeded + starting_chain_work = REGTEST_WORK_PER_BLOCK # Genesis block's work + self.log.info("Testing relay across node %d (minChainWork = %d)", 1, self.node_min_work[1]) + + starting_blockcount = self.nodes[2].getblockcount() + + num_blocks_to_generate = int((self.node_min_work[1] - starting_chain_work) / REGTEST_WORK_PER_BLOCK) + self.log.info("Generating %d blocks on node0", num_blocks_to_generate) + hashes = self.nodes[0].generate(num_blocks_to_generate) + + self.log.info("Node0 current chain work: %s", self.nodes[0].getblockheader(hashes[-1])['chainwork']) + + # Sleep a few seconds and verify that node2 didn't get any new blocks + # or headers. We sleep, rather than sync_blocks(node0, node1) because + # it's reasonable either way for node1 to get the blocks, or not get + # them (since they're below node1's minchainwork). + time.sleep(3) + + self.log.info("Verifying node 2 has no more blocks than before") + self.log.info("Blockcounts: %s", [n.getblockcount() for n in self.nodes]) + # Node2 shouldn't have any new headers yet, because node1 should not + # have relayed anything. + assert_equal(len(self.nodes[2].getchaintips()), 1) + assert_equal(self.nodes[2].getchaintips()[0]['height'], 0) + + assert self.nodes[1].getbestblockhash() != self.nodes[0].getbestblockhash() + assert_equal(self.nodes[2].getblockcount(), starting_blockcount) + + self.log.info("Generating one more block") + self.nodes[0].generate(1) + + self.log.info("Verifying nodes are all synced") + self.sync_all() + self.log.info("Blockcounts: %s", [n.getblockcount() for n in self.nodes]) + +if __name__ == '__main__': + MinimumChainWorkTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index f0d34f4051f39..f523410cc9527 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -135,6 +135,7 @@ 'bip65-cltv-p2p.py', 'uptime.py', 'resendwallettransactions.py', + 'minchainwork.py', ] EXTENDED_SCRIPTS = [ From 35d60de2b5dfc19af189728d4dad5c1faac411c2 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Sat, 21 Oct 2017 11:13:46 +0200 Subject: [PATCH 09/26] Merge #11458: Don't process unrequested, low-work blocks 01b52ce Add comment explaining forced processing of compact blocks (Suhas Daftuar) 08fd822 qa: add test for minchainwork use in acceptblock (Suhas Daftuar) ce8cd7a Don't process unrequested, low-work blocks (Suhas Daftuar) Pull request description: A peer could try to waste our resources by sending us unrequested blocks with low work (eg to fill up our disk). Since e265200 we no longer request blocks until we know we're on a chain with more than nMinimumChainWork (our anti-DoS threshold), but we would still process unrequested blocks that had more work than our tip (which generally has low-work during IBD), even though we may not yet have found a headers chain with sufficient work. Fix this and add a test. Tree-SHA512: 1a4fb0bbd78054b84683f995c8c3194dd44fa914dc351ae4379c7c1a6f83224f609f8b9c2d9dde28741426c6af008ffffea836d21aa31a5ebaa00f8e0f81229e --- src/net_processing.cpp | 17 ++++++++++-- src/validation.cpp | 6 ++++ test/functional/p2p-acceptblock.py | 44 ++++++++++++++++++++++-------- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index cf832afac7ba0..20c8064bb7315 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2449,7 +2449,16 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mapBlockSource.emplace(pblock->GetHash(), std::make_pair(pfrom->GetId(), false)); } bool fNewBlock = false; - ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + // Setting fForceProcessing to true means that we bypass some of + // our anti-DoS protections in AcceptBlock, which filters + // unrequested blocks that might be trying to waste our resources + // (eg disk space). Because we only try to reconstruct blocks when + // we're close to caught up (via the CanDirectFetch() requirement + // above, combined with the behavior of not requesting blocks until + // we have a chain with at least nMinimumChainWork), and we ignore + // compact blocks with less work than our tip, it is safe to treat + // reconstructed compact blocks as having been requested. + ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -2529,7 +2538,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool fNewBlock = false; // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) - ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + // This bypasses some anti-DoS logic in AcceptBlock (eg to prevent + // disk-space attacks), but this should be safe due to the + // protections in the compact block handler -- see related comment + // in compact block optimistic reconstruction handling. + ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { diff --git a/src/validation.cpp b/src/validation.cpp index 11edb19094095..49c14bbb7933f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3464,6 +3464,12 @@ static bool AcceptBlock(const std::shared_ptr& pblock, CValidation if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned if (!fHasMoreWork) return true; // Don't process less-work chains if (fTooFarAhead) return true; // Block height is too high + + // Protect against DoS attacks from low-work chains. + // If our tip is behind, a peer could try to send us + // low-work blocks on a fake chain that we would never + // request; don't process these. + if (pindex->nChainWork < nMinimumChainWork) return true; } if (fNewBlock) *fNewBlock = true; diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index a007ddb4feeb6..38052d103f0c1 100755 --- a/test/functional/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -8,17 +8,22 @@ versus non-whitelisted peers, this tests the behavior of both (effectively two separate tests running in parallel). -Setup: two nodes, node0 and node1, not connected to each other. Node0 does not +Setup: three nodes, node0+node1+node2, not connected to each other. Node0 does not whitelist localhost, but node1 does. They will each be on their own chain for -this test. +this test. Node2 will have nMinimumChainWork set to 0x10, so it won't process +low-work unrequested blocks. -We have one NodeConn connection to each, test_node and white_node respectively. +We have one NodeConn connection to each, test_node, white_node, and min_work_node, +respectively. The test: 1. Generate one block on each node, to leave IBD. 2. Mine a new block on each tip, and deliver to each node from node's peer. - The tip should advance. + The tip should advance for node0 and node1, but node2 should skip processing + due to nMinimumChainWork. + +Node2 is unused in tests 3-7: 3. Mine a block that forks the previous block, and deliver to each node from corresponding peer. @@ -46,6 +51,10 @@ 7. Send Node0 the missing block again. Node0 should process and the tip should advance. + +8. Test Node2 is able to sync when connected to node0 (which should have sufficient +work on its chain). + """ from test_framework.mininode import * @@ -62,52 +71,60 @@ def add_options(self, parser): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 2 - self.extra_args = [[], ["-whitelist=127.0.0.1"]] + self.num_nodes = 3 + self.extra_args = [[], ["-whitelist=127.0.0.1"], ["-minimumchainwork=0x10"]] def setup_network(self): # Node0 will be used to test behavior of processing unrequested blocks # from peers which are not whitelisted, while Node1 will be used for # the whitelisted case. + # Node2 will be used for non-whitelisted peers to test the interaction + # with nMinimumChainWork. self.setup_nodes() def run_test(self): # Setup the p2p connections and start up the network thread. test_node = NodeConnCB() # connects to node0 (not whitelisted) white_node = NodeConnCB() # connects to node1 (whitelisted) + min_work_node = NodeConnCB() # connects to node2 (not whitelisted) connections = [] connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node)) connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], white_node)) + connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], min_work_node)) test_node.add_connection(connections[0]) white_node.add_connection(connections[1]) + min_work_node.add_connection(connections[2]) NetworkThread().start() # Start up network handling in another thread # Test logic begins here test_node.wait_for_verack() white_node.wait_for_verack() + min_work_node.wait_for_verack() - # 1. Have both nodes mine a block (leave IBD) + # 1. Have nodes mine a block (nodes1/2 leave IBD) [ n.generate(1) for n in self.nodes ] tips = [ int("0x" + n.getbestblockhash(), 0) for n in self.nodes ] # 2. Send one block that builds on each tip. - # This should be accepted. + # This should be accepted by nodes 1/2 blocks_h2 = [] # the height 2 blocks on each node's chain block_time = self.mocktime + 1 - for i in range(2): + for i in range(3): blocks_h2.append(create_block(tips[i], create_coinbase(2), block_time + 1)) blocks_h2[i].solve() block_time += 1 test_node.send_message(msg_block(blocks_h2[0])) white_node.send_message(msg_block(blocks_h2[1])) + min_work_node.send_message(msg_block(blocks_h2[2])) - for x in [test_node, white_node]: + for x in [test_node, white_node, min_work_node]: x.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) assert_equal(self.nodes[1].getblockcount(), 2) - self.log.info("First height 2 block accepted by both nodes") + assert_equal(self.nodes[2].getblockcount(), 1) + self.log.info("First height 2 block accepted by node0/node1; correctly rejected by node2") # 3. Send another block that builds on the original tip. blocks_h2f = [] # Blocks at height 2 that fork off the main chain @@ -221,6 +238,11 @@ def run_test(self): assert_equal(self.nodes[0].getblockcount(), 290) self.log.info("Successfully reorged to longer chain from non-whitelisted peer") + # 8. Connect node2 to node0 and ensure it is able to sync + connect_nodes(self.nodes[0], 2) + sync_blocks([self.nodes[0], self.nodes[2]]) + self.log.info("Successfully synced nodes 2 and 0") + [ c.disconnect_node() for c in connections ] if __name__ == '__main__': From f8c310a974e627592d2e1164d6f20ad14390cbc2 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 13 Oct 2017 15:25:16 -0700 Subject: [PATCH 10/26] Merge #11456: Replace relevant services logic with a function suite. 15f5d3b17 Switch DNSSeed-needed metric to any-automatic-nodes, not services (Matt Corallo) 5ee88b4bd Clarify docs for requirements/handling of addnode/connect nodes (Matt Corallo) 57edc0b0c Rename fAddnode to a more-descriptive "manual_connection" (Matt Corallo) 44407100f Replace relevant services logic with a function suite. (Matt Corallo) Pull request description: This was mostly written as a way to clean things up so that the NETWORK_LIMITED PR (#10387) can be simplified a ton, but its also a nice standalone cleanup that will also require a bit of review because it tweaks a lot of stuff across net. The new functions are fine in protocol.h right now since they're straight-forward, but after NETWORK_LIMITED will really want to move elsewhere after @theuni moves the nServices-based selection to addrman from connman. Adds HasAllRelevantServices and GetRelevantServices, which check for NETWORK|WITNESS. This changes the following: * Removes nRelevantServices from CConnman, disconnecting it a bit more from protocol-level logic. * Replaces our sometimes-connect-to-!WITNESS-nodes logic with simply always requiring WITNESS|NETWORK for outbound non-feeler connections (feelers still only require NETWORK). * This has the added benefit of removing nServicesExpected from CNode - instead letting net_processing's VERSION message handling simply check HasAllRelevantServices. * This implies we believe WITNESS nodes to continue to be a significant majority of nodes on the network, but also because we cannot sync properly from !WITNESS nodes, it is strange to continue using our valuable outbound slots on them. * In order to prevent this change from preventing connection to -connect= nodes which have !WITNESS, -connect nodes are now given the "addnode" flag. This also allows outbound connections to !NODE_NETWORK nodes for -connect nodes (which was already true of addnodes). * Has the (somewhat unintended) consequence of changing one of the eviction metrics from the same sometimes-connect-to-!WITNESS-nodes metric to requiring HasRelevantServices. This should make NODE_NETWORK_LIMITED much simpler to implement. Tree-SHA512: 90606896c86cc5da14c77843b16674a6a012065e7b583d76d1c47a18215358abefcbab44ff4fab3fadcd39aa9a42d4740c6dc8874a58033bdfc8ad3fb5c649fc --- src/init.cpp | 6 ++--- src/net.cpp | 52 +++++++++++++----------------------------- src/net.h | 14 +++--------- src/net_processing.cpp | 15 +++++++----- src/protocol.h | 37 ++++++++++++++++++++++++++++++ src/rpc/net.cpp | 8 ++++--- 6 files changed, 72 insertions(+), 60 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 16a2bb80f545d..91941ba9ca9d0 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -487,12 +487,12 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-spentindex", strprintf(_("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)"), DEFAULT_SPENTINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); - strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); + strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info)")); strUsage += HelpMessageOpt("-allowprivatenet", strprintf(_("Allow RFC1918 addresses to be relayed and connected to (default: %u)"), DEFAULT_ALLOWPRIVATENET)); strUsage += HelpMessageOpt("-banscore=", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt("-bantime=", strprintf(_("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt("-bind=", _("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-connect=", _("Connect only to the specified node(s); -connect=0 disables automatic connections")); + strUsage += HelpMessageOpt("-connect=", _("Connect only to the specified node(s); -connect=0 disables automatic connections (the rules for this peer are the same as for -addnode)")); strUsage += HelpMessageOpt("-discover", _("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); strUsage += HelpMessageOpt("-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + strprintf(_("(default: %u)"), DEFAULT_NAME_LOOKUP)); strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect used)")); @@ -1016,7 +1016,6 @@ void InitLogging() namespace { // Variables internal to initialization process only -ServiceFlags nRelevantServices = NODE_NETWORK; int nMaxConnections; int nUserMaxConnections; int nFD; @@ -2122,7 +2121,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; - connOptions.nRelevantServices = nRelevantServices; connOptions.nMaxConnections = nMaxConnections; connOptions.nMaxOutbound = std::min(MAX_OUTBOUND_CONNECTIONS, connOptions.nMaxConnections); connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; diff --git a/src/net.cpp b/src/net.cpp index 1160f278d769c..009a60f068b22 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -436,7 +436,6 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); CAddress addr_bind = GetBindAddress(hSocket); CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false); - pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); pnode->AddRef(); @@ -688,7 +687,7 @@ void CNode::copyStats(CNodeStats &stats) X(cleanSubVer); } X(fInbound); - X(fAddnode); + X(m_manual_connection); X(nStartingHeight); { LOCK(cs_vSend); @@ -1026,7 +1025,7 @@ bool CConnman::AttemptToEvictConnection() NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, node->nLastBlockTime, node->nLastTXTime, - (node->nServices & nRelevantServices) == nRelevantServices, + HasAllDesirableServiceFlags(node->nServices), node->fRelayTxes, node->pfilter != nullptr, node->nKeyedNetGroup}; vEvictionCandidates.push_back(candidate); } @@ -1717,7 +1716,7 @@ void CConnman::ThreadDNSAddressSeed() LOCK(cs_vNodes); int nRelevant = 0; for (auto pnode : vNodes) { - nRelevant += pnode->fSuccessfullyConnected && ((pnode->nServices & nRelevantServices) == nRelevantServices); + nRelevant += pnode->fSuccessfullyConnected && !pnode->fFeeler && !pnode->fOneShot && !pnode->m_manual_connection && !pnode->fInbound; } if (nRelevant >= 2) { LogPrintf("P2P peers available. Skipped DNS seeding.\n"); @@ -1739,7 +1738,7 @@ void CConnman::ThreadDNSAddressSeed() } else { std::vector vIPs; std::vector vAdd; - ServiceFlags requiredServiceBits = nRelevantServices; + ServiceFlags requiredServiceBits = GetDesirableServiceFlags(NODE_NONE); std::string host = GetDNSHost(seed, &requiredServiceBits); CNetAddr resolveSource; if (!resolveSource.SetInternal(host)) { @@ -1820,7 +1819,7 @@ void CConnman::ThreadOpenConnections() for (const std::string& strAddr : gArgs.GetArgs("-connect")) { CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, nullptr, strAddr.c_str()); + OpenNetworkConnection(addr, false, nullptr, strAddr.c_str(), false, false, true); for (int i = 0; i < 10 && i < nLoop; i++) { if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) @@ -1869,17 +1868,11 @@ void CConnman::ThreadOpenConnections() // Do this here so we don't have to critsect vNodes inside mapAddresses critsect. // This is only done for mainnet and testnet int nOutbound = 0; - int nOutboundRelevant = 0; std::set > setConnected; if (!Params().AllowMultipleAddressesFromGroup()) { LOCK(cs_vNodes); for (CNode* pnode : vNodes) { - if (!pnode->fInbound && !pnode->fAddnode && !pnode->fMasternode) { - - // Count the peers that have all relevant services - if (pnode->fSuccessfullyConnected && !pnode->fFeeler && ((pnode->nServices & nRelevantServices) == nRelevantServices)) { - nOutboundRelevant++; - } + if (!pnode->fInbound && !pnode->fMasternode && !pnode->m_manual_connection) { // Netgroups for inbound and addnode peers are not excluded because our goal here // is to not use multiple of our limited outbound slots on a single netgroup // but inbound and addnode peers do not use our outbound slots. Inbound peers @@ -1943,21 +1936,16 @@ void CConnman::ThreadOpenConnections() if (IsLimited(addr)) continue; - // only connect to full nodes - if (!isMasternode && (addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) - continue; - // only consider very recently tried nodes after 30 failed attempts if (nANow - addr.nLastTry < 600 && nTries < 30) continue; - // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. - ServiceFlags nRequiredServices = nRelevantServices; - if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { - nRequiredServices = REQUIRED_SERVICES; - } - - if (!isMasternode && (addr.nServices & nRequiredServices) != nRequiredServices) { + // for non-feelers, require all the services we'll want, + // for feelers, only require they be a full node (only because most + // SPV clients don't have a good address DB available) + if (!isMasternode && !fFeeler && !HasAllDesirableServiceFlags(addr.nServices)) { + continue; + } else if (!isMasternode && fFeeler && !MayHaveUsefulAddressDB(addr.nServices)) { continue; } @@ -1966,13 +1954,6 @@ void CConnman::ThreadOpenConnections() continue; addrConnect = addr; - - // regardless of the services assumed to be available, only require the minimum if half or more outbound have relevant services - if (nOutboundRelevant >= (nMaxOutbound >> 1)) { - addrConnect.nServices = REQUIRED_SERVICES; - } else { - addrConnect.nServices = nRequiredServices; - } break; } @@ -2165,7 +2146,7 @@ void CConnman::ThreadOpenMasternodeConnections() } // if successful, this moves the passed grant to the constructed node -bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool fAddnode, bool fConnectToMasternode) +bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool manual_connection, bool fConnectToMasternode) { // // Initiate outbound network connection @@ -2201,8 +2182,8 @@ bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai pnode->fOneShot = true; if (fFeeler) pnode->fFeeler = true; - if (fAddnode) - pnode->fAddnode = true; + if (manual_connection) + pnode->m_manual_connection = true; if (fConnectToMasternode) pnode->fMasternode = true; @@ -3138,7 +3119,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn nSendVersion(0) { nServices = NODE_NONE; - nServicesExpected = NODE_NONE; hSocket = hSocketIn; nRecvVersion = INIT_PROTO_VERSION; nLastSend = 0; @@ -3153,7 +3133,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn strSubVer = ""; fWhitelisted = false; fOneShot = false; - fAddnode = false; + m_manual_connection = false; fClient = false; // set by version message fFeeler = false; fSuccessfullyConnected = false; diff --git a/src/net.h b/src/net.h index b3fd255a7e4c0..5f11017cdb3e5 100644 --- a/src/net.h +++ b/src/net.h @@ -111,8 +111,6 @@ static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; -static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; - // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban @@ -157,7 +155,6 @@ class CConnman struct Options { ServiceFlags nLocalServices = NODE_NONE; - ServiceFlags nRelevantServices = NODE_NONE; int nMaxConnections = 0; int nMaxOutbound = 0; int nMaxAddnode = 0; @@ -175,7 +172,6 @@ class CConnman void Init(const Options& connOptions) { nLocalServices = connOptions.nLocalServices; - nRelevantServices = connOptions.nRelevantServices; nMaxConnections = connOptions.nMaxConnections; nMaxOutbound = std::min(connOptions.nMaxOutbound, connOptions.nMaxConnections); nMaxAddnode = connOptions.nMaxAddnode; @@ -196,7 +192,7 @@ class CConnman void Interrupt(); bool GetNetworkActive() const { return fNetworkActive; }; void SetNetworkActive(bool active); - bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool fAddnode = false, bool fConnectToMasternode = false); + bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool manual_connection = false, bool fConnectToMasternode = false); bool OpenMasternodeConnection(const CAddress& addrConnect); bool CheckIncomingNonce(uint64_t nonce); @@ -532,9 +528,6 @@ class CConnman /** Services this instance offers */ ServiceFlags nLocalServices; - /** Services this instance cares about */ - ServiceFlags nRelevantServices; - CSemaphore *semOutbound; CSemaphore *semAddnode; CSemaphore *semMasternodeOutbound; @@ -663,7 +656,7 @@ class CNodeStats int nVersion; std::string cleanSubVer; bool fInbound; - bool fAddnode; + bool m_manual_connection; int nStartingHeight; uint64_t nSendBytes; mapMsgCmdSize mapSendBytesPerMsgCmd; @@ -735,7 +728,6 @@ class CNode public: // socket std::atomic nServices; - ServiceFlags nServicesExpected; SOCKET hSocket; size_t nSendSize; // total size of all vSendMsg entries size_t nSendOffset; // offset inside the first vSendMsg already sent @@ -777,7 +769,7 @@ class CNode bool fWhitelisted; // This peer can bypass DoS banning. bool fFeeler; // If true this node is being used as a short lived feeler. bool fOneShot; - bool fAddnode; + bool m_manual_connection; bool fClient; const bool fInbound; std::atomic_bool fSuccessfullyConnected; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 20c8064bb7315..6b43e0648e076 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1456,11 +1456,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { connman.SetServices(pfrom->addr, nServices); } - if (pfrom->nServicesExpected & ~nServices) + if (!pfrom->fInbound && !pfrom->fFeeler && !pfrom->m_manual_connection && !HasAllDesirableServiceFlags(nServices)) { - LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, pfrom->nServicesExpected); + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, GetDesirableServiceFlags(nServices)); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, - strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); + strprintf("Expected to offer services %08x", GetDesirableServiceFlags(nServices)))); pfrom->fDisconnect = true; return false; } @@ -1697,7 +1697,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (interruptMsgProc) return true; - if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) + // We only bother storing full nodes, though this may include + // things which we would not make an outbound connection to, in + // part because we may make feeler connections to them. + if (!MayHaveUsefulAddressDB(addr.nServices)) continue; if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) @@ -3001,8 +3004,8 @@ static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman& connman) state.fShouldBan = false; if (pnode->fWhitelisted) LogPrintf("Warning: not punishing whitelisted peer %s!\n", pnode->GetLogString()); - else if (pnode->fAddnode) - LogPrintf("Warning: not punishing addnoded peer %s!\n", pnode->GetLogString()); + else if (pnode->m_manual_connection) + LogPrintf("Warning: not punishing manually-connected peer %s!\n", pnode->GetLogString()); else { pnode->fDisconnect = true; if (pnode->addr.IsLocal()) diff --git a/src/protocol.h b/src/protocol.h index e7ee254f0a180..b27c7d8285ab2 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -306,6 +306,43 @@ enum ServiceFlags : uint64_t { // BIP process. }; +/** + * Gets the set of service flags which are "desirable" for a given peer. + * + * These are the flags which are required for a peer to support for them + * to be "interesting" to us, ie for us to wish to use one of our few + * outbound connection slots for or for us to wish to prioritize keeping + * their connection around. + * + * Relevant service flags may be peer- and state-specific in that the + * version of the peer may determine which flags are required (eg in the + * case of NODE_NETWORK_LIMITED where we seek out NODE_NETWORK peers + * unless they set NODE_NETWORK_LIMITED and we are out of IBD, in which + * case NODE_NETWORK_LIMITED suffices). + * + * Thus, generally, avoid calling with peerServices == NODE_NONE. + */ +static ServiceFlags GetDesirableServiceFlags(ServiceFlags services) { + return ServiceFlags(NODE_NETWORK | NODE_WITNESS); +} + +/** + * A shortcut for (services & GetDesirableServiceFlags(services)) + * == GetDesirableServiceFlags(services), ie determines whether the given + * set of service flags are sufficient for a peer to be "relevant". + */ +static inline bool HasAllDesirableServiceFlags(ServiceFlags services) { + return !(GetDesirableServiceFlags(services) & (~services)); +} + +/** + * Checks if a peer with the given service flags may be capable of having a + * robust address-storage DB. Currently an alias for checking NODE_NETWORK. + */ +static inline bool MayHaveUsefulAddressDB(ServiceFlags services) { + return services & NODE_NETWORK; +} + /** A CService with information about it as peer */ class CAddress : public CService { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index f16c2d1163536..d90ddf2829227 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -93,7 +93,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) " \"version\": v, (numeric) The peer version, such as 7001\n" " \"subver\": \"/Dash Core:x.x.x/\", (string) The string version\n" " \"inbound\": true|false, (boolean) Inbound (true) or Outbound (false)\n" - " \"addnode\": true|false, (boolean) Whether connection was due to addnode and is using an addnode slot\n" + " \"addnode\": true|false, (boolean) Whether connection was due to addnode/-connect or if it was an automatic/inbound connection\n" " \"startingheight\": n, (numeric) The starting height (block) of the peer\n" " \"banscore\": n, (numeric) The ban score\n" " \"synced_headers\": n, (numeric) The last header we have in common with this peer\n" @@ -157,7 +157,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) // their ver message. obj.push_back(Pair("subver", stats.cleanSubVer)); obj.push_back(Pair("inbound", stats.fInbound)); - obj.push_back(Pair("addnode", stats.fAddnode)); + obj.push_back(Pair("addnode", stats.m_manual_connection)); obj.push_back(Pair("startingheight", stats.nStartingHeight)); if (fStateStats) { obj.push_back(Pair("banscore", statestats.nMisbehavior)); @@ -202,6 +202,8 @@ UniValue addnode(const JSONRPCRequest& request) "addnode \"node\" \"add|remove|onetry\"\n" "\nAttempts to add or remove a node from the addnode list.\n" "Or try a connection to a node once.\n" + "Nodes added using addnode (or -connect) are protected from DoS disconnection and are not required to be\n" + "full nodes/support SegWit as other outbound peers are (though such peers will not be synced from).\n" "\nArguments:\n" "1. \"node\" (string, required) The node (see getpeerinfo for nodes)\n" "2. \"command\" (string, required) 'add' to add a node to the list, 'remove' to remove a node from the list, 'onetry' to try a connection to the node once\n" @@ -218,7 +220,7 @@ UniValue addnode(const JSONRPCRequest& request) if (strCommand == "onetry") { CAddress addr; - g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str()); + g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); return NullUniValue; } From 2edc29ee32854cc930bb71d0656fc40c669c0aed Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 8 Sep 2017 01:00:49 +0200 Subject: [PATCH 11/26] Merge #10756: net processing: swap out signals for an interface class 2525b972a net: stop both net/net_processing before destroying them (Cory Fields) 80e2e9d0c net: drop unused connman param (Cory Fields) 8ad663c1f net: use an interface class rather than signals for message processing (Cory Fields) 28f11e940 net: pass CConnman via pointer rather than reference (Cory Fields) Pull request description: See individual commits. Benefits: - Allows us to begin moving stuff out of CNode and into CNodeState (after #10652 and follow-ups) - Drops boost dependency and overhead - Drops global signal registration - Friendlier backtraces Tree-SHA512: af2038c959dbec25f0c90c74c88dc6a630e6b9e984adf52aceadd6954aa463b6aadfccf979c2459a9f3354326b5077ee02048128eda2a649236fadb595b66ee3 --- src/init.cpp | 9 +- src/net.cpp | 20 ++-- src/net.h | 25 ++-- src/net_processing.cpp | 262 +++++++++++++++++++---------------------- src/net_processing.h | 35 +++--- src/test/DoS_tests.cpp | 22 ++-- src/test/test_dash.cpp | 6 +- src/test/test_dash.h | 2 + 8 files changed, 180 insertions(+), 201 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 91941ba9ca9d0..bca3b7d882742 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -243,12 +243,14 @@ void PrepareShutdown() } #endif MapPort(false); + + // Because these depend on each-other, we make sure that neither can be + // using the other before destroying them. UnregisterValidationInterface(peerLogic.get()); - peerLogic.reset(); if (g_connman) { - // make sure to stop all threads before g_connman is reset to nullptr as these threads might still be accessing it g_connman->Stop(); } + peerLogic.reset(); g_connman.reset(); if (!fLiteMode && !fRPCInWarmup) { @@ -263,7 +265,6 @@ void PrepareShutdown() flatdb6.Dump(sporkManager); } - UnregisterNodeSignals(GetNodeSignals()); if (fDumpMempoolLater && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); } @@ -1588,7 +1589,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) peerLogic.reset(new PeerLogicValidation(&connman)); RegisterValidationInterface(peerLogic.get()); - RegisterNodeSignals(GetNodeSignals()); // sanitize comments per BIP-0014, format user agent and check total size std::vector uacomments; @@ -2127,6 +2127,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) connOptions.nMaxFeeler = 1; connOptions.nBestHeight = chainActive.Height(); connOptions.uiInterface = &uiInterface; + connOptions.m_msgproc = peerLogic.get(); connOptions.nSendBufferMaxSize = 1000*gArgs.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); connOptions.nReceiveFloodSize = 1000*gArgs.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); diff --git a/src/net.cpp b/src/net.cpp index 009a60f068b22..df7d032a933a6 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -99,10 +99,6 @@ std::string strSubVersion; unordered_limitedmap mapAlreadyAskedFor(MAX_INV_SZ, MAX_INV_SZ * 2); -// Signals for message handling -static CNodeSignals g_signals; -CNodeSignals& GetNodeSignals() { return g_signals; } - void CConnman::AddOneShot(const std::string& strDest) { LOCK(cs_vOneShots); @@ -1201,7 +1197,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", true); pnode->AddRef(); pnode->fWhitelisted = whitelisted; - GetNodeSignals().InitializeNode(pnode, *this); + m_msgproc->InitializeNode(pnode); if (fLogIPs) { LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); @@ -2187,7 +2183,7 @@ bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai if (fConnectToMasternode) pnode->fMasternode = true; - GetNodeSignals().InitializeNode(pnode, *this); + m_msgproc->InitializeNode(pnode); { LOCK(cs_vNodes); vNodes.push_back(pnode); @@ -2214,16 +2210,16 @@ void CConnman::ThreadMessageHandler() continue; // Receive messages - bool fMoreNodeWork = GetNodeSignals().ProcessMessages(pnode, *this, flagInterruptMsgProc); + bool fMoreNodeWork = m_msgproc->ProcessMessages(pnode, flagInterruptMsgProc); fMoreWork |= (fMoreNodeWork && !pnode->fPauseSend); if (flagInterruptMsgProc) return; - // Send messages { LOCK(pnode->cs_sendProcessing); - GetNodeSignals().SendMessages(pnode, *this, flagInterruptMsgProc); + m_msgproc->SendMessages(pnode, flagInterruptMsgProc); } + if (flagInterruptMsgProc) return; } @@ -2546,6 +2542,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) // // Start threads // + assert(m_msgproc); InterruptSocks5(false); interruptNet.reset(); flagInterruptMsgProc = false; @@ -2709,9 +2706,10 @@ void CConnman::DeleteNode(CNode* pnode) { assert(pnode); bool fUpdateConnectionTime = false; - GetNodeSignals().FinalizeNode(pnode->GetId(), fUpdateConnectionTime); - if(fUpdateConnectionTime) + m_msgproc->FinalizeNode(pnode->GetId(), fUpdateConnectionTime); + if(fUpdateConnectionTime) { addrman.Connected(pnode->addr); + } delete pnode; } diff --git a/src/net.h b/src/net.h index 5f11017cdb3e5..63aea36460d35 100644 --- a/src/net.h +++ b/src/net.h @@ -37,7 +37,6 @@ #include #endif -#include // "Optimistic send" was introduced in the beginning of the Bitcoin project. I assume this was done because it was // thought that "send" would be very cheap when the send buffer is empty. This is not true, as shown by profiling. @@ -140,7 +139,7 @@ struct CSerializedNetMsg std::string command; }; - +class NetEventsInterface; class CConnman { public: @@ -161,6 +160,7 @@ class CConnman int nMaxFeeler = 0; int nBestHeight = 0; CClientUIInterface* uiInterface = nullptr; + NetEventsInterface* m_msgproc = nullptr; unsigned int nSendBufferMaxSize = 0; unsigned int nReceiveFloodSize = 0; uint64_t nMaxOutboundTimeframe = 0; @@ -178,6 +178,7 @@ class CConnman nMaxFeeler = connOptions.nMaxFeeler; nBestHeight = connOptions.nBestHeight; clientInterface = connOptions.uiInterface; + m_msgproc = connOptions.m_msgproc; nSendBufferMaxSize = connOptions.nSendBufferMaxSize; nReceiveFloodSize = connOptions.nReceiveFloodSize; nMaxOutboundTimeframe = connOptions.nMaxOutboundTimeframe; @@ -537,6 +538,7 @@ class CConnman int nMaxFeeler; std::atomic nBestHeight; CClientUIInterface* clientInterface; + NetEventsInterface* m_msgproc; /** SipHasher seeds for deterministic randomness */ const uint64_t nSeed0, nSeed1; @@ -584,19 +586,18 @@ struct CombinerAll } }; -// Signals for message handling -struct CNodeSignals +/** + * Interface for message handling + */ +class NetEventsInterface { - boost::signals2::signal&), CombinerAll> ProcessMessages; - boost::signals2::signal&), CombinerAll> SendMessages; - boost::signals2::signal InitializeNode; - boost::signals2::signal FinalizeNode; +public: + virtual bool ProcessMessages(CNode* pnode, std::atomic& interrupt) = 0; + virtual bool SendMessages(CNode* pnode, std::atomic& interrupt) = 0; + virtual void InitializeNode(CNode* pnode) = 0; + virtual void FinalizeNode(NodeId id, bool& update_connection_time) = 0; }; - -CNodeSignals& GetNodeSignals(); - - enum { LOCAL_NONE, // unknown diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6b43e0648e076..af06bc108c671 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -161,11 +161,6 @@ namespace { std::deque> vRelayExpiration; } // namespace -////////////////////////////////////////////////////////////////////////////// -// -// Registration of network node signals. -// - namespace { struct CBlockReject { @@ -272,7 +267,7 @@ void UpdatePreferredDownload(CNode* node, CNodeState* state) nPreferredDownload += state->fPreferredDownload; } -void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) +void PushNodeVersion(CNode *pnode, CConnman* connman, int64_t nTime) { ServiceFlags nLocalNodeServices = pnode->GetLocalServices(); uint64_t nonce = pnode->GetLocalNonce(); @@ -290,7 +285,7 @@ void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) pnode->sentMNAuthChallenge = mnauthChallenge; } - connman.PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + connman->PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, nonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes, mnauthChallenge)); if (fLogIPs) { @@ -300,49 +295,6 @@ void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) } } -void InitializeNode(CNode *pnode, CConnman& connman) { - CAddress addr = pnode->addr; - std::string addrName = pnode->GetAddrName(); - NodeId nodeid = pnode->GetId(); - { - LOCK(cs_main); - mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName))); - } - if(!pnode->fInbound) - PushNodeVersion(pnode, connman, GetTime()); -} - -void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { - fUpdateConnectionTime = false; - LOCK(cs_main); - CNodeState *state = State(nodeid); - - if (state->fSyncStarted) - nSyncStarted--; - - if (state->nMisbehavior == 0 && state->fCurrentlyConnected) { - fUpdateConnectionTime = true; - } - - for (const QueuedBlock& entry : state->vBlocksInFlight) { - mapBlocksInFlight.erase(entry.hash); - } - EraseOrphansFor(nodeid); - nPreferredDownload -= state->fPreferredDownload; - nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); - assert(nPeersWithValidatedDownloads >= 0); - - mapNodeState.erase(nodeid); - - if (mapNodeState.empty()) { - // Do a consistency check after the last peer is removed. - assert(mapBlocksInFlight.empty()); - assert(nPreferredDownload == 0); - assert(nPeersWithValidatedDownloads == 0); - } - LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); -} - // Requires cs_main. // Returns a bool indicating whether we requested this block. // Also used if a block was /not/ received and timed out or started with another peer @@ -437,7 +389,7 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { } } -void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connman) { +void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman* connman) { AssertLockHeld(cs_main); CNodeState* nodestate = State(nodeid); if (!nodestate || !nodestate->fSupportsDesiredCmpctVersion) { @@ -452,20 +404,20 @@ void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connman) { return; } } - connman.ForNode(nodeid, [&connman](CNode* pfrom){ + connman->ForNode(nodeid, [connman](CNode* pfrom){ bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 1; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ - connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->ForNode(lNodesAnnouncingHeaderAndIDs.front(), [connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman->PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); return true; }); lNodesAnnouncingHeaderAndIDs.pop_front(); } fAnnounceUsingCMPCTBLOCK = true; - connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); return true; }); @@ -574,6 +526,50 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vectoraddr; + std::string addrName = pnode->GetAddrName(); + NodeId nodeid = pnode->GetId(); + { + LOCK(cs_main); + mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName))); + } + if(!pnode->fInbound) + PushNodeVersion(pnode, connman, GetTime()); +} + +void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { + fUpdateConnectionTime = false; + LOCK(cs_main); + CNodeState *state = State(nodeid); + assert(state != nullptr); + + if (state->fSyncStarted) + nSyncStarted--; + + if (state->nMisbehavior == 0 && state->fCurrentlyConnected) { + fUpdateConnectionTime = true; + } + + for (const QueuedBlock& entry : state->vBlocksInFlight) { + mapBlocksInFlight.erase(entry.hash); + } + EraseOrphansFor(nodeid); + nPreferredDownload -= state->fPreferredDownload; + nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); + assert(nPeersWithValidatedDownloads >= 0); + + mapNodeState.erase(nodeid); + + if (mapNodeState.empty()) { + // Do a consistency check after the last peer is removed. + assert(mapBlocksInFlight.empty()); + assert(nPreferredDownload == 0); + assert(nPeersWithValidatedDownloads == 0); + } + LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); +} + bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { LOCK(cs_main); CNodeState *state = State(nodeid); @@ -589,22 +585,6 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { return true; } -void RegisterNodeSignals(CNodeSignals& nodeSignals) -{ - nodeSignals.ProcessMessages.connect(&ProcessMessages); - nodeSignals.SendMessages.connect(&SendMessages); - nodeSignals.InitializeNode.connect(&InitializeNode); - nodeSignals.FinalizeNode.connect(&FinalizeNode); -} - -void UnregisterNodeSignals(CNodeSignals& nodeSignals) -{ - nodeSignals.ProcessMessages.disconnect(&ProcessMessages); - nodeSignals.SendMessages.disconnect(&SendMessages); - nodeSignals.InitializeNode.disconnect(&InitializeNode); - nodeSignals.FinalizeNode.disconnect(&FinalizeNode); -} - ////////////////////////////////////////////////////////////////////////////// // // mapOrphanTransactions @@ -919,7 +899,7 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta !IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { - MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, *connman); + MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, connman); } } if (it != mapBlockSource.end()) @@ -1007,7 +987,7 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) return true; } -static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connman) +static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connman) { unsigned int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) @@ -1015,7 +995,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connma // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the addrKnowns of the chosen nodes prevent repeats uint64_t hashAddr = addr.GetHash(); - const CSipHasher hasher = connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); + const CSipHasher hasher = connman->GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); FastRandomContext insecure_rand; std::array,2> best{{{0, nullptr}, {0, nullptr}}}; @@ -1040,10 +1020,10 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connma } }; - connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); + connman->ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } -void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensusParams, const CInv& inv, CConnman& connman, const std::atomic& interruptMsgProc) +void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensusParams, const CInv& inv, CConnman* connman, const std::atomic& interruptMsgProc) { bool send = false; std::shared_ptr a_recent_block; @@ -1087,7 +1067,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); // disconnect node in case we have reached the outbound limit for serving historical blocks // never disconnect whitelisted nodes - if (send && connman.OutboundTargetReached(true) && ( ((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) + if (send && connman->OutboundTargetReached(true) && ( ((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) { LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); @@ -1110,7 +1090,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus pblock = pblockRead; } if (inv.type == MSG_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -1123,7 +1103,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus } } if (sendMerkleBlock) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip // Note that there is currently no way for a node to request any single transactions we didn't send here - @@ -1132,7 +1112,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus // however we MUST always provide at least what the remote peer needs typedef std::pair PairType; for (PairType& pair : merkleBlock.vMatchedTxn) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *pblock->vtx[pair.first])); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *pblock->vtx[pair.first])); } // else // no response @@ -1145,13 +1125,13 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus // instead we respond with the full, non-compact block. if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { if (a_recent_compact_block && a_recent_compact_block->header.GetHash() == mi->second->GetBlockHash()) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); } else { CBlockHeaderAndShortTxIDs cmpctblock(*pblock); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); } } else { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); } } @@ -1163,7 +1143,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus // wait for other stuff first. std::vector vInv; vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); pfrom->hashContinue.SetNull(); } } @@ -1197,14 +1177,14 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (inv.type == MSG_TX) { auto mi = mapRelay.find(inv.hash); if (mi != mapRelay.end()) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *mi->second)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *mi->second)); push = true; } else if (pfrom->timeLastMempoolReq) { auto txinfo = mempool.info(inv.hash); // To protect privacy, do not answer getdata using the mempool when // that TX couldn't have been INVed in reply to a MEMPOOL request. if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *txinfo.tx)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::TX, *txinfo.tx)); push = true; } } @@ -1349,11 +1329,11 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // do that because they want to know about (and store and rebroadcast and // risk analyze) the dependencies of transactions relevant to them, without // having to download the entire memory pool. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); } } -inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode* pfrom, CConnman& connman) { +inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode* pfrom, CConnman* connman) { BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { if (req.indexes[i] >= block.vtx.size()) { @@ -1366,10 +1346,10 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac } LOCK(cs_main); CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); } -bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic& interruptMsgProc) +bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) @@ -1430,7 +1410,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Each connection can only send one version message if (pfrom->nVersion != 0) { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, std::string("Duplicate version message"))); + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, std::string("Duplicate version message"))); LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; @@ -1454,12 +1434,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr nServices = ServiceFlags(nServiceInt); if (!pfrom->fInbound) { - connman.SetServices(pfrom->addr, nServices); + connman->SetServices(pfrom->addr, nServices); } if (!pfrom->fInbound && !pfrom->fFeeler && !pfrom->m_manual_connection && !HasAllDesirableServiceFlags(nServices)) { LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, GetDesirableServiceFlags(nServices)); - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, strprintf("Expected to offer services %08x", GetDesirableServiceFlags(nServices)))); pfrom->fDisconnect = true; return false; @@ -1469,7 +1449,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { // disconnect from peers older than this proto version LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); pfrom->fDisconnect = true; return false; @@ -1493,7 +1473,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr vRecv >> pfrom->receivedMNAuthChallenge; } // Disconnect if we connected to ourself - if (pfrom->fInbound && !connman.CheckIncomingNonce(nNonce)) + if (pfrom->fInbound && !connman->CheckIncomingNonce(nNonce)) { LogPrintf("connected to self at %s, disconnecting\n", pfrom->addr.ToString()); pfrom->fDisconnect = true; @@ -1522,7 +1502,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); pfrom->nServices = nServices; pfrom->SetAddrLocal(addrMe); @@ -1567,12 +1547,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } // Get recent addresses - if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || connman.GetAddressCount() < 1000) + if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || connman->GetAddressCount() < 1000) { - connman.PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); + connman->PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); pfrom->fGetAddr = true; } - connman.MarkAddressGood(pfrom->addr); + connman->MarkAddressGood(pfrom->addr); } std::string remoteAddr; @@ -1627,7 +1607,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { @@ -1638,12 +1618,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // they may wish to request compact blocks from us bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 1; - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); } if (pfrom->nVersion >= SENDDSQUEUE_PROTO_VERSION) { // Tell our peer that he should send us PrivateSend queue messages - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDDSQUEUE, true)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDDSQUEUE, true)); } else { // older nodes do not support SENDDSQUEUE and expect us to always send PrivateSend queue messages // TODO we can remove this compatibility code in 0.15.0 @@ -1679,7 +1659,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr vRecv >> vAddr; // Don't want addr from older versions unless seeding - if (pfrom->nVersion < CADDR_TIME_VERSION && connman.GetAddressCount() > 1000) + if (pfrom->nVersion < CADDR_TIME_VERSION && connman->GetAddressCount() > 1000) return true; if (vAddr.size() > 1000) { @@ -1715,7 +1695,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fReachable) vAddrOk.push_back(addr); } - connman.AddNewAddresses(vAddrOk, pfrom->addr, 2 * 60 * 60); + connman->AddNewAddresses(vAddrOk, pfrom->addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom->fGetAddr = false; if (pfrom->fOneShot) @@ -1815,7 +1795,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // fell back to inv we probably have a reorg which we should get the headers for first, // we now only provide a getheaders response here. When we receive the headers, we will // then ask for the blocks we need. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId()); } } @@ -2047,7 +2027,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : chainActive.Tip(); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); } @@ -2270,7 +2250,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->GetId(), FormatStateMessage(state)); if (state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); if (nDoS > 0) { Misbehaving(pfrom->GetId(), nDoS); @@ -2289,7 +2269,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!IsInitialBlockDownload()) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); return true; } } @@ -2344,7 +2324,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // so we just grab the block via normal getdata std::vector vInv(1); vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return true; } @@ -2382,7 +2362,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Duplicate txindexes, the block is now in-flight, so just request it std::vector vInv(1); vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } @@ -2399,7 +2379,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr fProcessBLOCKTXN = true; } else { req.blockhash = pindex->GetBlockHash(); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); } } else { // This block is either already in flight from a different @@ -2425,7 +2405,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // mempool will probably be useless - request the block normally std::vector vInv(1); vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message @@ -2508,7 +2488,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Might have collided, fall back to getdata now :( std::vector invs; invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -2593,7 +2573,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // nUnconnectingHeaders gets reset back to 0. if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), @@ -2648,7 +2628,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); @@ -2696,7 +2676,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); } } } @@ -2757,7 +2737,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->fSentAddr = true; pfrom->vAddrToSend.clear(); - std::vector vAddr = connman.GetAddresses(); + std::vector vAddr = connman->GetAddresses(); FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) pfrom->PushAddress(addr, insecure_rand); @@ -2773,7 +2753,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - if (connman.OutboundTargetReached(false) && !pfrom->fWhitelisted) + if (connman->OutboundTargetReached(false) && !pfrom->fWhitelisted) { LogPrint(BCLog::NET, "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); pfrom->fDisconnect = true; @@ -2802,7 +2782,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); } } @@ -2990,13 +2970,13 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } -static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman& connman) +static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman* connman) { AssertLockHeld(cs_main); CNodeState &state = *State(pnode->GetId()); for (const CBlockReject& reject : state.rejects) { - connman.PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, (std::string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock)); + connman->PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, (std::string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock)); } state.rejects.clear(); @@ -3012,7 +2992,7 @@ static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman& connman) LogPrintf("Warning: not banning local peer %s!\n", pnode->GetLogString()); else { - connman.Ban(pnode->addr, BanReasonNodeMisbehaving); + connman->Ban(pnode->addr, BanReasonNodeMisbehaving); } } return true; @@ -3020,7 +3000,7 @@ static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman& connman) return false; } -bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic& interruptMsgProc) +bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic& interruptMsgProc) { const CChainParams& chainparams = Params(); // @@ -3054,7 +3034,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic& i // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; - pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); + pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman->GetReceiveFloodSize(); fMoreWork = !pfrom->vProcessMsg.empty(); } CNetMessage& msg(msgs.front()); @@ -3103,7 +3083,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic& i } catch (const std::ios_base::failure& e) { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); + connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv @@ -3154,7 +3134,7 @@ class CompareInvMempoolOrder } }; -bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interruptMsgProc) +bool PeerLogicValidation::SendMessages(CNode* pto, std::atomic& interruptMsgProc) { const Consensus::Params& consensusParams = Params().GetConsensus(); { @@ -3186,11 +3166,11 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr pto->nPingUsecStart = GetTimeMicros(); if (pto->nVersion > BIP0031_VERSION) { pto->nPingNonceSent = nonce; - connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); } else { // Peer is too old to support ping command with nonce, pong will never arrive. pto->nPingNonceSent = 0; - connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING)); } } @@ -3225,14 +3205,14 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); vAddr.clear(); } } } pto->vAddrToSend.clear(); if (!vAddr.empty()) - connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); // we only send the big addr message once if (pto->vAddrToSend.capacity() > 40) pto->vAddrToSend.shrink_to_fit(); @@ -3259,7 +3239,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr if (pindexStart->pprev) pindexStart = pindexStart->pprev; LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); } } @@ -3268,7 +3248,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr // transactions become unconfirmed and spams other nodes. if (!fReindex && !fImporting && !IsInitialBlockDownload()) { - GetMainSignals().Broadcast(nTimeBestReceived, &connman); + GetMainSignals().Broadcast(nTimeBestReceived, connman); } // @@ -3358,7 +3338,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr { LOCK(cs_most_recent_block); if (most_recent_block_hash == pBestIndex->GetBlockHash()) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); fGotBlockFromCache = true; } } @@ -3367,7 +3347,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); assert(ret); CBlockHeaderAndShortTxIDs cmpctblock(block); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); } state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { @@ -3380,7 +3360,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->GetId()); } - connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); state.pindexBestHeaderSent = pBestIndex; } else fRevertToInv = true; @@ -3426,7 +3406,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr for (const uint256& hash : pto->vInventoryBlockToSend) { vInv.push_back(CInv(MSG_BLOCK, hash)); if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -3469,7 +3449,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { LogPrint(BCLog::NET, "SendMessages -- pushing inv's: count=%d peer=%d\n", vInv.size(), pto->GetId()); - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -3527,7 +3507,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr } } if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } pto->filterInventoryKnown.insert(hash); @@ -3549,7 +3529,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr pto->vInventoryOtherToSend.clear(); } if (!vInv.empty()) - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling nNow = GetTimeMicros(); @@ -3645,7 +3625,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr vGetData.push_back(inv); if (vGetData.size() >= 1000) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); LogPrint(BCLog::NET, "SendMessages -- GETDATA -- pushed size = %lu peer=%d\n", vGetData.size(), pto->GetId()); vGetData.clear(); } @@ -3658,7 +3638,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interr } pto->vecAskFor.erase(pto->vecAskFor.begin(), it); if (!vGetData.empty()) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); LogPrint(BCLog::NET, "SendMessages -- GETDATA -- pushed size = %lu peer=%d\n", vGetData.size(), pto->GetId()); } diff --git a/src/net_processing.h b/src/net_processing.h index b6c0e8ef64365..ddcb47afff581 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -24,22 +24,31 @@ static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1000; // 1ms/head /** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; -/** Register with a network node to receive its signals */ -void RegisterNodeSignals(CNodeSignals& nodeSignals); -/** Unregister a network node */ -void UnregisterNodeSignals(CNodeSignals& nodeSignals); - -class PeerLogicValidation : public CValidationInterface { +class PeerLogicValidation : public CValidationInterface, public NetEventsInterface { private: CConnman* connman; public: - PeerLogicValidation(CConnman* connmanIn); + explicit PeerLogicValidation(CConnman* connmanIn); void BlockConnected(const std::shared_ptr& pblock, const CBlockIndex* pindexConnected, const std::vector& vtxConflicted) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; void BlockChecked(const CBlock& block, const CValidationState& state) override; void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr& pblock) override; + + + void InitializeNode(CNode* pnode) override; + void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) override; + /** Process protocol messages received from a given node */ + bool ProcessMessages(CNode* pfrom, std::atomic& interrupt) override; + /** + * Send queued protocol messages to be sent to a give node. + * + * @param[in] pto The node which we are sending messages to. + * @param[in] interrupt Interrupt condition for processing threads + * @return True if there is more work to be done + */ + bool SendMessages(CNode* pto, std::atomic& interrupt) override; }; struct CNodeStateStats { @@ -55,16 +64,4 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); void Misbehaving(NodeId nodeid, int howmuch); bool IsBanned(NodeId nodeid); -/** Process protocol messages received from a given node */ -bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic& interrupt); -/** - * Send queued protocol messages to be sent to a give node. - * - * @param[in] pto The node which we are sending messages to. - * @param[in] connman The connection manager for that node. - * @param[in] interrupt Interrupt condition for processing threads - * @return True if there is more work to be done - */ -bool SendMessages(CNode* pto, CConnman& connman, const std::atomic& interrupt); - #endif // BITCOIN_NET_PROCESSING_H diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 961b18c45425e..ac898466440c9 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -50,26 +50,26 @@ BOOST_AUTO_TEST_CASE(DoS_banning) CAddress addr1(ip(0xa0b0c001), NODE_NONE); CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); - GetNodeSignals().InitializeNode(&dummyNode1, *connman); + peerLogic->InitializeNode(&dummyNode1); dummyNode1.nVersion = 1; dummyNode1.fSuccessfullyConnected = true; Misbehaving(dummyNode1.GetId(), 100); // Should get banned - SendMessages(&dummyNode1, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(connman->IsBanned(addr1)); BOOST_CHECK(!connman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned CAddress addr2(ip(0xa0b0c002), NODE_NONE); CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", true); dummyNode2.SetSendVersion(PROTOCOL_VERSION); - GetNodeSignals().InitializeNode(&dummyNode2, *connman); + peerLogic->InitializeNode(&dummyNode2); dummyNode2.nVersion = 1; dummyNode2.fSuccessfullyConnected = true; Misbehaving(dummyNode2.GetId(), 50); - SendMessages(&dummyNode2, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode2, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr2)); // 2 not banned yet... BOOST_CHECK(connman->IsBanned(addr1)); // ... but 1 still should be Misbehaving(dummyNode2.GetId(), 50); - SendMessages(&dummyNode2, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode2, interruptDummy); BOOST_CHECK(connman->IsBanned(addr2)); } @@ -82,17 +82,17 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) CAddress addr1(ip(0xa0b0c001), NODE_NONE); CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); - GetNodeSignals().InitializeNode(&dummyNode1, *connman); + peerLogic->InitializeNode(&dummyNode1); dummyNode1.nVersion = 1; dummyNode1.fSuccessfullyConnected = true; Misbehaving(dummyNode1.GetId(), 100); - SendMessages(&dummyNode1, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr1)); Misbehaving(dummyNode1.GetId(), 10); - SendMessages(&dummyNode1, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr1)); Misbehaving(dummyNode1.GetId(), 1); - SendMessages(&dummyNode1, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(connman->IsBanned(addr1)); gArgs.ForceSetArg("-banscore", std::to_string(DEFAULT_BANSCORE_THRESHOLD)); } @@ -108,12 +108,12 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) CAddress addr(ip(0xa0b0c001), NODE_NONE); CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, CAddress(), "", true); dummyNode.SetSendVersion(PROTOCOL_VERSION); - GetNodeSignals().InitializeNode(&dummyNode, *connman); + peerLogic->InitializeNode(&dummyNode); dummyNode.nVersion = 1; dummyNode.fSuccessfullyConnected = true; Misbehaving(dummyNode.GetId(), 100); - SendMessages(&dummyNode, *connman, interruptDummy); + peerLogic->SendMessages(&dummyNode, interruptDummy); BOOST_CHECK(connman->IsBanned(addr)); SetMockTime(nStartTime+60*60); diff --git a/src/test/test_dash.cpp b/src/test/test_dash.cpp index e7054324a7b96..7d002341226df 100644 --- a/src/test/test_dash.cpp +++ b/src/test/test_dash.cpp @@ -59,7 +59,6 @@ BasicTestingSetup::~BasicTestingSetup() delete evoDb; ECC_Stop(); - g_connman.reset(); } TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) @@ -96,17 +95,18 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha nScriptCheckThreads = 3; for (int i=0; i < nScriptCheckThreads-1; i++) threadGroup.create_thread(&ThreadScriptCheck); - RegisterNodeSignals(GetNodeSignals()); + peerLogic.reset(new PeerLogicValidation(connman)); } TestingSetup::~TestingSetup() { - UnregisterNodeSignals(GetNodeSignals()); llmq::InterruptLLMQSystem(); threadGroup.interrupt_all(); threadGroup.join_all(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); + g_connman.reset(); + peerLogic.reset(); UnloadBlockIndex(); delete pcoinsTip; llmq::DestroyLLMQSystem(); diff --git a/src/test/test_dash.h b/src/test/test_dash.h index 98a631d790a16..c911666f4fb0c 100644 --- a/src/test/test_dash.h +++ b/src/test/test_dash.h @@ -50,12 +50,14 @@ struct BasicTestingSetup { * Included are data directory, coins database, script check threads setup. */ class CConnman; +class PeerLogicValidation; struct TestingSetup: public BasicTestingSetup { CCoinsViewDB *pcoinsdbview; fs::path pathTemp; boost::thread_group threadGroup; CConnman* connman; CScheduler scheduler; + std::unique_ptr peerLogic; TestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~TestingSetup(); From bf7485213b9f38f0f7ac1580141d872681be2844 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 25 Sep 2019 12:19:52 +0200 Subject: [PATCH 12/26] More "connman." to "connman->" changes --- src/net_processing.cpp | 66 +++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index af06bc108c671..06e5fdf116f2e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1149,7 +1149,7 @@ void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensus } } -void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParams, CConnman& connman, const std::atomic& interruptMsgProc) +void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParams, CConnman* connman, const std::atomic& interruptMsgProc) { AssertLockNotHeld(cs_main); @@ -1193,7 +1193,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && inv.type == MSG_SPORK) { CSporkMessage spork; if(sporkManager.GetSporkByHash(inv.hash, spork)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SPORK, spork)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SPORK, spork)); push = true; } } @@ -1201,7 +1201,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && inv.type == MSG_DSTX) { CPrivateSendBroadcastTx dstx = CPrivateSend::GetDSTX(inv.hash); if(dstx) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::DSTX, dstx)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::DSTX, dstx)); push = true; } } @@ -1220,7 +1220,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } LogPrint(BCLog::NET, "ProcessGetData -- MSG_GOVERNANCE_OBJECT: topush = %d, inv = %s\n", topush, inv.ToString()); if(topush) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MNGOVERNANCEOBJECT, ss)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MNGOVERNANCEOBJECT, ss)); push = true; } } @@ -1238,7 +1238,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } if(topush) { LogPrint(BCLog::NET, "ProcessGetData -- pushing: inv = %s\n", inv.ToString()); - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MNGOVERNANCEOBJECTVOTE, ss)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MNGOVERNANCEOBJECTVOTE, ss)); push = true; } } @@ -1246,7 +1246,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && (inv.type == MSG_QUORUM_FINAL_COMMITMENT)) { llmq::CFinalCommitment o; if (llmq::quorumBlockProcessor->GetMinableCommitmentByHash(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QFCOMMITMENT, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QFCOMMITMENT, o)); push = true; } } @@ -1254,35 +1254,35 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && (inv.type == MSG_QUORUM_CONTRIB)) { llmq::CDKGContribution o; if (llmq::quorumDKGSessionManager->GetContribution(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QCONTRIB, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QCONTRIB, o)); push = true; } } if (!push && (inv.type == MSG_QUORUM_COMPLAINT)) { llmq::CDKGComplaint o; if (llmq::quorumDKGSessionManager->GetComplaint(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QCOMPLAINT, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QCOMPLAINT, o)); push = true; } } if (!push && (inv.type == MSG_QUORUM_JUSTIFICATION)) { llmq::CDKGJustification o; if (llmq::quorumDKGSessionManager->GetJustification(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QJUSTIFICATION, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QJUSTIFICATION, o)); push = true; } } if (!push && (inv.type == MSG_QUORUM_PREMATURE_COMMITMENT)) { llmq::CDKGPrematureCommitment o; if (llmq::quorumDKGSessionManager->GetPrematureCommitment(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QPCOMMITMENT, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QPCOMMITMENT, o)); push = true; } } if (!push && (inv.type == MSG_QUORUM_RECOVERED_SIG)) { llmq::CRecoveredSig o; if (llmq::quorumSigningManager->GetRecoveredSigForGetData(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QSIGREC, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QSIGREC, o)); push = true; } } @@ -1290,7 +1290,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && (inv.type == MSG_CLSIG)) { llmq::CChainLockSig o; if (llmq::chainLocksHandler->GetChainLockByHash(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::CLSIG, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::CLSIG, o)); push = true; } } @@ -1298,7 +1298,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!push && (inv.type == MSG_ISLOCK)) { llmq::CInstantSendLock o; if (llmq::quorumInstantSendManager->GetInstantSendLockByHash(inv.hash, o)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::ISLOCK, o)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::ISLOCK, o)); push = true; } } @@ -1599,7 +1599,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (pfrom->nVersion >= LLMQS_PROTO_VERSION) { - CMNAuth::PushMNAUTH(pfrom, connman); + CMNAuth::PushMNAUTH(pfrom, *connman); } if (pfrom->nVersion >= SENDHEADERS_VERSION) { @@ -1635,11 +1635,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Otherwise the peer would only announce/send messages resulting from QRECSIG, // e.g. InstantSend locks or ChainLocks. SPV nodes should not send this message // as they are usually only interested in the higher level messages - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QSENDRECSIGS, true)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QSENDRECSIGS, true)); } if (gArgs.GetBoolArg("-watchquorums", llmq::DEFAULT_WATCH_QUORUMS)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::QWATCH)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::QWATCH)); } pfrom->fSuccessfullyConnected = true; @@ -2064,7 +2064,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->AddInventoryKnown(inv); { LOCK(cs_main); - connman.RemoveAskFor(inv.hash); + connman->RemoveAskFor(inv.hash); } // Process custom logic, no matter if tx will be accepted to mempool later or not @@ -2116,7 +2116,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } mempool.check(pcoinsTip); - connman.RelayTransaction(tx); + connman->RelayTransaction(tx); for (unsigned int i = 0; i < tx.vout.size(); i++) { vWorkQueue.emplace_back(inv.hash, i); } @@ -2154,7 +2154,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr continue; if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2)) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - connman.RelayTransaction(orphanTx); + connman->RelayTransaction(orphanTx); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { vWorkQueue.emplace_back(orphanHash, i); } @@ -2236,7 +2236,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr int nDoS = 0; if (!state.IsInvalid(nDoS) || nDoS == 0) { LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->GetId()); - connman.RelayTransaction(tx); + connman->RelayTransaction(tx); } else { LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); } @@ -2911,7 +2911,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr CSimplifiedMNListDiff mnListDiff; std::string strError; if (BuildSimplifiedMNListDiff(cmd.baseBlockHash, cmd.blockHash, mnListDiff, strError)) { - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MNLISTDIFF, mnListDiff)); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MNLISTDIFF, mnListDiff)); } else { LogPrint(BCLog::NET, "getmnlistdiff failed for baseBlockHash=%s, blockHash=%s. error=%s\n", cmd.baseBlockHash.ToString(), cmd.blockHash.ToString(), strError); Misbehaving(pfrom->GetId(), 1); @@ -2946,19 +2946,19 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { //probably one the extensions #ifdef ENABLE_WALLET - privateSendClient.ProcessMessage(pfrom, strCommand, vRecv, connman); + privateSendClient.ProcessMessage(pfrom, strCommand, vRecv, *connman); #endif // ENABLE_WALLET - privateSendServer.ProcessMessage(pfrom, strCommand, vRecv, connman); - sporkManager.ProcessSpork(pfrom, strCommand, vRecv, connman); + privateSendServer.ProcessMessage(pfrom, strCommand, vRecv, *connman); + sporkManager.ProcessSpork(pfrom, strCommand, vRecv, *connman); masternodeSync.ProcessMessage(pfrom, strCommand, vRecv); - governance.ProcessMessage(pfrom, strCommand, vRecv, connman); - CMNAuth::ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumBlockProcessor->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumDKGSessionManager->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumSigSharesManager->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumSigningManager->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::chainLocksHandler->ProcessMessage(pfrom, strCommand, vRecv, connman); - llmq::quorumInstantSendManager->ProcessMessage(pfrom, strCommand, vRecv, connman); + governance.ProcessMessage(pfrom, strCommand, vRecv, *connman); + CMNAuth::ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumBlockProcessor->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumDKGSessionManager->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumSigSharesManager->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumSigningManager->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::chainLocksHandler->ProcessMessage(pfrom, strCommand, vRecv, *connman); + llmq::quorumInstantSendManager->ProcessMessage(pfrom, strCommand, vRecv, *connman); } else { @@ -3522,7 +3522,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto, std::atomic& interruptM vInv.push_back(inv); pto->filterInventoryKnown.insert(inv.hash); if (vInv.size() == MAX_INV_SZ) { - connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } From 38a45a53d261ce15245eea4ef1836e797db189ae Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 25 Sep 2019 12:28:11 +0200 Subject: [PATCH 13/26] Make CDBEnv::IsMock() const --- src/wallet/db.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/db.h b/src/wallet/db.h index 8d28d2dcec735..8b1105cccba7a 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -45,7 +45,7 @@ class CDBEnv void Reset(); void MakeMock(); - bool IsMock() { return fMockDb; } + bool IsMock() const { return fMockDb; } /** * Verify that database file strFile is OK. If it is not, From c35aa663bcf9075e591b032b6116a235ed06bc2f Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Thu, 14 Sep 2017 16:45:09 +0200 Subject: [PATCH 14/26] Merge #11326: Fix crash on shutdown with invalid wallet 77939f27f Fix uninitialized g_connman crash in Shutdown() (MeshCollider) Pull request description: Fixes https://github.com/bitcoin/bitcoin/issues/11312 As @dooglus pointed out, `g_connman` is uninitialized when an invalid wallet path is passed on start up, but then dereferenced in `Shutdown()`, so this tiny PR just fixes that. Tree-SHA512: 2557133422a6e393017081450a7e6c100fe7d9ce36e628e5f5f479bc07617a7bd9a9ad4d44c0d8abadf2e3eb62a11ce9743abc27b4ae8c20f709e72df4f25a7f --- src/init.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index bca3b7d882742..965cc31442652 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -247,9 +247,7 @@ void PrepareShutdown() // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. UnregisterValidationInterface(peerLogic.get()); - if (g_connman) { - g_connman->Stop(); - } + if(g_connman) g_connman->Stop(); peerLogic.reset(); g_connman.reset(); From b451094e2e58685d8eb10bfa1048dccc132253b5 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 26 Oct 2017 21:53:19 +0200 Subject: [PATCH 15/26] Merge #11490: Disconnect from outbound peers with bad headers chains e065249 Add unit test for outbound peer eviction (Suhas Daftuar) 5a6d00c Permit disconnection of outbound peers on bad/slow chains (Suhas Daftuar) c60fd71 Disconnecting from bad outbound peers in IBD (Suhas Daftuar) Pull request description: The first commit will disconnect an outbound peer that serves us a headers chain with insufficient work while we're in IBD. The second commit introduces a way to disconnect outbound peers whose chains fall out of sync with ours: For a given outbound peer, we check whether their best known block (which is known from the blocks they announce to us) has at least as much work as our tip. If it doesn't, we set a 20 minute timeout, and if we still haven't heard about a block with as much work as our tip had when we set the timeout, then we send a single getheaders message, and wait 2 more minutes. If after two minutes their best known block has insufficient work, we disconnect that peer. We protect 4 of our outbound peers (who provide some "good" headers chains, ie a chain with at least as much work as our tip at some point) from being subject to this logic, to prevent excessive network topology changes as a result of this algorithm, while still ensuring that we have a reasonable number of nodes not known to be on bogus chains. We also don't require our peers to be on the same chain as us, to prevent accidental partitioning of the network in the event of a chain split. Note that if our peers are ever on a more work chain than our tip, then we will download and validate it, and then either reorg to it, or learn of a consensus incompatibility with that peer and disconnect. This PR is designed to protect against peers that are on a less work chain which we may never try to download and validate. Tree-SHA512: 2e0169a1dd8a7fb95980573ac4a201924bffdd724c19afcab5efcef076fdbe1f2cec7dc5f5d7e0a6327216f56d3828884f73642e00c8534b56ec2bb4c854a656 --- src/net_processing.cpp | 129 ++++++++++++++++++++++++++++++++ src/net_processing.h | 8 ++ src/test/DoS_tests.cpp | 55 ++++++++++++++ test/functional/minchainwork.py | 8 ++ 4 files changed, 200 insertions(+) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 06e5fdf116f2e..f3867f9082ba5 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -147,6 +147,9 @@ namespace { /** Number of peers from which we're downloading blocks. */ int nPeersWithValidatedDownloads = 0; + /** Number of outbound peers with m_chain_sync.m_protect. */ + int g_outbound_peers_with_protect_from_disconnect = 0; + /* This comment is here to force a merge conflict when bitcoin#11560 is backported * It introduces this member as int64_t while bitcoin#11824 changes it to atomic. * bitcoin#11824 is partially backported already, which means you'll have to take the atomic @@ -223,6 +226,33 @@ struct CNodeState { */ bool fSupportsDesiredCmpctVersion; + /** State used to enforce CHAIN_SYNC_TIMEOUT + * Only in effect for outbound, non-manual connections, with + * m_protect == false + * Algorithm: if a peer's best known block has less work than our tip, + * set a timeout CHAIN_SYNC_TIMEOUT seconds in the future: + * - If at timeout their best known block now has more work than our tip + * when the timeout was set, then either reset the timeout or clear it + * (after comparing against our current tip's work) + * - If at timeout their best known block still has less work than our + * tip did when the timeout was set, then send a getheaders message, + * and set a shorter timeout, HEADERS_RESPONSE_TIME seconds in future. + * If their best known block is still behind when that new timeout is + * reached, disconnect. + */ + struct ChainSyncTimeoutState { + //! A timeout used for checking whether our peer has sufficiently synced + int64_t m_timeout; + //! A header with the work we require on our peer's chain + const CBlockIndex * m_work_header; + //! After timeout is reached, set to true after sending getheaders + bool m_sent_getheaders; + //! Whether this peer is protected from disconnection due to a bad/slow chain + bool m_protect; + }; + + ChainSyncTimeoutState m_chain_sync; + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; nMisbehavior = 0; @@ -243,6 +273,7 @@ struct CNodeState { fPreferHeaderAndIDs = false; fProvidesHeaderAndIDs = false; fSupportsDesiredCmpctVersion = false; + m_chain_sync = { 0, nullptr, false, false }; } }; @@ -526,6 +557,13 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vectorfInbound || node->m_manual_connection || node->fFeeler || node->fOneShot); +} + void PeerLogicValidation::InitializeNode(CNode *pnode) { CAddress addr = pnode->addr; std::string addrName = pnode->GetAddrName(); @@ -558,6 +596,8 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim nPreferredDownload -= state->fPreferredDownload; nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); assert(nPeersWithValidatedDownloads >= 0); + g_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; + assert(g_outbound_peers_with_protect_from_disconnect >= 0); mapNodeState.erase(nodeid); @@ -566,6 +606,7 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim assert(mapBlocksInFlight.empty()); assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); + assert(g_outbound_peers_with_protect_from_disconnect == 0); } LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } @@ -2623,6 +2664,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr assert(pindexLast); UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + // From here, pindexBestKnownBlock should be guaranteed to be non-null, + // because it is set in UpdateBlockAvailability. Some nullptr checks + // are still present, however, as belt-and-suspenders. + if (nCount == MAX_HEADERS_RESULTS) { // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue @@ -2680,6 +2725,35 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } } + // If we're in IBD, we want outbound peers that will serve us a useful + // chain. Disconnect peers that are on chains with insufficient work. + if (IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { + // When nCount < MAX_HEADERS_RESULTS, we know we have no more + // headers to fetch from this peer. + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + // This peer has too little work on their headers chain to help + // us sync -- disconnect if using an outbound slot (unless + // whitelisted or addnode). + // Note: We compare their tip to nMinimumChainWork (rather than + // chainActive.Tip()) because we won't start block download + // until we have a headers chain that has at least + // nMinimumChainWork, even if a peer has a chain past our tip, + // as an anti-DoS measure. + if (IsOutboundDisconnectionCandidate(pfrom)) { + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom->GetId()); + pfrom->fDisconnect = true; + } + } + } + + if (!pfrom->fDisconnect && IsOutboundDisconnectionCandidate(pfrom) && nodestate->pindexBestKnownBlock != nullptr) { + // If this is an outbound peer, check to see if we should protect + // it from the bad/lagging chain logic. + if (g_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + nodestate->m_chain_sync.m_protect = true; + ++g_outbound_peers_with_protect_from_disconnect; + } + } } } @@ -3117,6 +3191,58 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic& inter return fMoreWork; } +void PeerLogicValidation::ConsiderEviction(CNode *pto, int64_t time_in_seconds) +{ + AssertLockHeld(cs_main); + + CNodeState &state = *State(pto->GetId()); + const CNetMsgMaker msgMaker(pto->GetSendVersion()); + + if (!state.m_chain_sync.m_protect && IsOutboundDisconnectionCandidate(pto) && state.fSyncStarted) { + // This is an outbound peer subject to disconnection if they don't + // announce a block with as much work as the current tip within + // CHAIN_SYNC_TIMEOUT + HEADERS_RESPONSE_TIME seconds (note: if + // their chain has more work than ours, we should sync to it, + // unless it's invalid, in which case we should find that out and + // disconnect from them elsewhere). + if (state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork) { + if (state.m_chain_sync.m_timeout != 0) { + state.m_chain_sync.m_timeout = 0; + state.m_chain_sync.m_work_header = nullptr; + state.m_chain_sync.m_sent_getheaders = false; + } + } else if (state.m_chain_sync.m_timeout == 0 || (state.m_chain_sync.m_work_header != nullptr && state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= state.m_chain_sync.m_work_header->nChainWork)) { + // Our best block known by this peer is behind our tip, and we're either noticing + // that for the first time, OR this peer was able to catch up to some earlier point + // where we checked against our tip. + // Either way, set a new timeout based on current tip. + state.m_chain_sync.m_timeout = time_in_seconds + CHAIN_SYNC_TIMEOUT; + state.m_chain_sync.m_work_header = chainActive.Tip(); + state.m_chain_sync.m_sent_getheaders = false; + } else if (state.m_chain_sync.m_timeout > 0 && time_in_seconds > state.m_chain_sync.m_timeout) { + // No evidence yet that our peer has synced to a chain with work equal to that + // of our tip, when we first detected it was behind. Send a single getheaders + // message to give the peer a chance to update us. + if (state.m_chain_sync.m_sent_getheaders) { + // They've run out of time to catch up! + LogPrintf("Disconnecting outbound peer %d for old chain, best known block = %s\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : ""); + pto->fDisconnect = true; + } else { + LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); + state.m_chain_sync.m_sent_getheaders = true; + constexpr int64_t HEADERS_RESPONSE_TIME = 120; // 2 minutes + // Bump the timeout to allow a response, which could clear the timeout + // (if the response shows the peer has synced), reset the timeout (if + // the peer syncs to the required work but not to our tip), or result + // in disconnect (if we advance to the timeout and pindexBestKnownBlock + // has not sufficiently progressed) + state.m_chain_sync.m_timeout = time_in_seconds + HEADERS_RESPONSE_TIME; + } + } + } +} + class CompareInvMempoolOrder { CTxMemPool *mp; @@ -3588,6 +3714,9 @@ bool PeerLogicValidation::SendMessages(CNode* pto, std::atomic& interruptM } } + // Check that outbound peers have reasonable chains + // GetTime() is used by this anti-DoS logic so we can test this using mocktime + ConsiderEviction(pto, GetTime()); // // Message: getdata (blocks) diff --git a/src/net_processing.h b/src/net_processing.h index ddcb47afff581..8253938a27f3f 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -20,6 +20,12 @@ static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; * Timeout = base + per_header * (expected number of headers) */ static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1000; // 1ms/header +/** Protect at least this many outbound peers from disconnection due to slow/ + * behind headers chain. + */ +static constexpr int32_t MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT = 4; +/** Timeout for (unprotected) outbound peers to sync to our chainwork, in seconds */ +static constexpr int64_t CHAIN_SYNC_TIMEOUT = 20 * 60; // 20 minutes /** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; @@ -49,6 +55,8 @@ class PeerLogicValidation : public CValidationInterface, public NetEventsInterfa * @return True if there is more work to be done */ bool SendMessages(CNode* pto, std::atomic& interrupt) override; + + void ConsiderEviction(CNode *pto, int64_t time_in_seconds); }; struct CNodeStateStats { diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index ac898466440c9..b5f28ee9559e2 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -42,6 +42,51 @@ static NodeId id = 0; BOOST_FIXTURE_TEST_SUITE(DoS_tests, TestingSetup) +// Test eviction of an outbound peer whose chain never advances +// Mock a node connection, and use mocktime to simulate a peer +// which never sends any headers messages. PeerLogic should +// decide to evict that outbound peer, after the appropriate timeouts. +// Note that we protect 4 outbound nodes from being subject to +// this logic; this test takes advantage of that protection only +// being applied to nodes which send headers with sufficient +// work. +BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) +{ + std::atomic interruptDummy(false); + + // Mock an outbound peer + CAddress addr1(ip(0xa0b0c001), NODE_NONE); + CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", /*fInboundIn=*/ false); + dummyNode1.SetSendVersion(PROTOCOL_VERSION); + + peerLogic->InitializeNode(&dummyNode1); + dummyNode1.nVersion = 1; + dummyNode1.fSuccessfullyConnected = true; + + // This test requires that we have a chain with non-zero work. + BOOST_CHECK(chainActive.Tip() != nullptr); + BOOST_CHECK(chainActive.Tip()->nChainWork > 0); + + // Test starts here + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in getheaders + BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); + dummyNode1.vSendMsg.clear(); + + int64_t nStartTime = GetTime(); + // Wait 21 minutes + SetMockTime(nStartTime+21*60); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in getheaders + BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); + // Wait 3 more minutes + SetMockTime(nStartTime+24*60); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in disconnect + BOOST_CHECK(dummyNode1.fDisconnect == true); + SetMockTime(0); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); +} + BOOST_AUTO_TEST_CASE(DoS_banning) { std::atomic interruptDummy(false); @@ -71,6 +116,10 @@ BOOST_AUTO_TEST_CASE(DoS_banning) Misbehaving(dummyNode2.GetId(), 50); peerLogic->SendMessages(&dummyNode2, interruptDummy); BOOST_CHECK(connman->IsBanned(addr2)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); + peerLogic->FinalizeNode(dummyNode2.GetId(), dummy); } BOOST_AUTO_TEST_CASE(DoS_banscore) @@ -95,6 +144,9 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(connman->IsBanned(addr1)); gArgs.ForceSetArg("-banscore", std::to_string(DEFAULT_BANSCORE_THRESHOLD)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); } BOOST_AUTO_TEST_CASE(DoS_bantime) @@ -121,6 +173,9 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime+60*60*24+1); BOOST_CHECK(!connman->IsBanned(addr)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode.GetId(), dummy); } CTransactionRef RandomOrphan() diff --git a/test/functional/minchainwork.py b/test/functional/minchainwork.py index c7579d2548e46..35cd7ad141807 100755 --- a/test/functional/minchainwork.py +++ b/test/functional/minchainwork.py @@ -27,6 +27,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [[], ["-minimumchainwork=0x65"], ["-minimumchainwork=0x65"]] self.node_min_work = [0, 101, 101] @@ -74,6 +75,13 @@ def run_test(self): self.nodes[0].generate(1) self.log.info("Verifying nodes are all synced") + + # Because nodes in regtest are all manual connections (eg using + # addnode), node1 should not have disconnected node0. If not for that, + # we'd expect node1 to have disconnected node0 for serving an + # insufficient work chain, in which case we'd need to reconnect them to + # continue the test. + self.sync_all() self.log.info("Blockcounts: %s", [n.getblockcount() for n in self.nodes]) From 2e980018cfca9c1f299f9341ba9705c76b66ee2e Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 28 Oct 2017 11:11:23 -0700 Subject: [PATCH 16/26] Merge #11568: Disconnect outbound peers on invalid chains 37886d5e2 Disconnect outbound peers relaying invalid headers (Suhas Daftuar) 4637f1852 moveonly: factor out headers processing into separate function (Suhas Daftuar) Pull request description: Alternate to #11446. Disconnect outbound (non-manual) peers that serve us block headers that are already known to be invalid, but exempt compact block announcements from such disconnects. We restrict disconnection to outbound peers that are using up an outbound connection slot, because we rely on those peers to give us connectivity to the honest network (our inbound peers are not chosen by us and hence could all be from an attacker/sybil). Maintaining connectivity to peers that serve us invalid headers is sometimes desirable, eg after a soft-fork, to protect unupgraded software from being partitioned off the honest network, so we prefer to only disconnect when necessary. Compact block announcements are exempted from this logic to comply with BIP 152, which explicitly permits nodes to relay compact blocks before fully validating them. Tree-SHA512: 3ea88e4ccc1184f292a85b17f800d401d2c3806fefc7ad5429d05d6872c53acfa5751e3df83ce6b9c0060ab289511ed70ae1323d140ccc5b12e3c8da6de49936 --- src/net_processing.cpp | 387 +++++++++++++++++++++++------------------ src/validation.cpp | 4 +- src/validation.h | 3 +- 3 files changed, 224 insertions(+), 170 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index f3867f9082ba5..cb230af06c002 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1390,6 +1390,211 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); } +bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::vector& headers, const CChainParams& chainparams, bool punish_duplicate_invalid) +{ + const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + size_t nCount = headers.size(); + + if (nCount == 0) { + // Nothing interesting. Stop asking this peers for more headers. + return true; + } + + const CBlockIndex *pindexLast = nullptr; + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + + // If this looks like it could be a block announcement (nCount < + // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that + // don't connect: + // - Send a getheaders message in response to try to connect the chain. + // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + // don't connect before giving DoS points + // - Once a headers message is received that is valid and does connect, + // nUnconnectingHeaders gets reset back to 0. + if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + nodestate->nUnconnectingHeaders++; + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + pindexBestHeader->nHeight, + pfrom->GetId(), nodestate->nUnconnectingHeaders); + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); + + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(pfrom->GetId(), 20); + } + return true; + } + + uint256 hashLastBlock; + for (const CBlockHeader& header : headers) { + if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { + Misbehaving(pfrom->GetId(), 20); + return error("non-continuous headers sequence"); + } + hashLastBlock = header.GetHash(); + } + } + + CValidationState state; + CBlockHeader first_invalid_header; + if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast, &first_invalid_header)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + if (nDoS > 0) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), nDoS); + } + if (punish_duplicate_invalid && mapBlockIndex.find(first_invalid_header.GetHash()) != mapBlockIndex.end()) { + // Goal: don't allow outbound peers to use up our outbound + // connection slots if they are on incompatible chains. + // + // We ask the caller to set punish_invalid appropriately based + // on the peer and the method of header delivery (compact + // blocks are allowed to be invalid in some circumstances, + // under BIP 152). + // Here, we try to detect the narrow situation that we have a + // valid block header (ie it was valid at the time the header + // was received, and hence stored in mapBlockIndex) but know the + // block is invalid, and that a peer has announced that same + // block as being on its active chain. + // Disconnect the peer in such a situation. + // + // Note: if the header that is invalid was not accepted to our + // mapBlockIndex at all, that may also be grounds for + // disconnecting the peer, as the chain they are on is likely + // to be incompatible. However, there is a circumstance where + // that does not hold: if the header's timestamp is more than + // 2 hours ahead of our current time. In that case, the header + // may become valid in the future, and we don't want to + // disconnect a peer merely for serving us one too-far-ahead + // block header, to prevent an attacker from splitting the + // network by mining a block right at the 2 hour boundary. + // + // TODO: update the DoS logic (or, rather, rewrite the + // DoS-interface between validation and net_processing) so that + // the interface is cleaner, and so that we disconnect on all the + // reasons that a peer's headers chain is incompatible + // with ours (eg block->nVersion softforks, MTP violations, + // etc), and not just the duplicate-invalid case. + pfrom->fDisconnect = true; + } + return error("invalid header received"); + } + } + + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + + // From here, pindexBestKnownBlock should be guaranteed to be non-null, + // because it is set in UpdateBlockAvailability. Some nullptr checks + // are still present, however, as belt-and-suspenders. + + if (nCount == MAX_HEADERS_RESULTS) { + // Headers message had its maximum size; the peer may have more headers. + // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue + // from there instead. + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); + } + + bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); + // If this set of headers is valid and ends in a block with at least as + // much work as our tip, download as much as possible. + if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { + std::vector vToFetch; + const CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !mapBlocksInFlight.count(pindexWalk->GetBlockHash())) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!chainActive.Contains(pindexWalk)) { + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + std::vector vGetData; + // Download as much as possible, from earliest to latest. + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom->GetId()); + } + if (vGetData.size() > 1) { + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + } + } + } + // If we're in IBD, we want outbound peers that will serve us a useful + // chain. Disconnect peers that are on chains with insufficient work. + if (IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { + // When nCount < MAX_HEADERS_RESULTS, we know we have no more + // headers to fetch from this peer. + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + // This peer has too little work on their headers chain to help + // us sync -- disconnect if using an outbound slot (unless + // whitelisted or addnode). + // Note: We compare their tip to nMinimumChainWork (rather than + // chainActive.Tip()) because we won't start block download + // until we have a headers chain that has at least + // nMinimumChainWork, even if a peer has a chain past our tip, + // as an anti-DoS measure. + if (IsOutboundDisconnectionCandidate(pfrom)) { + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom->GetId()); + pfrom->fDisconnect = true; + } + } + } + + if (!pfrom->fDisconnect && IsOutboundDisconnectionCandidate(pfrom) && nodestate->pindexBestKnownBlock != nullptr) { + // If this is an outbound peer, check to see if we should protect + // it from the bad/lagging chain logic. + if (g_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + nodestate->m_chain_sync.m_protect = true; + ++g_outbound_peers_with_protect_from_disconnect; + } + } + } + + return true; +} + bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); @@ -2339,7 +2544,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // If we end up treating this as a plain headers message, call that as well // without cs_main. bool fRevertToHeaderProcessing = false; - CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); // Keep a CBlock for "optimistic" compactblock reconstructions (see // below) @@ -2450,10 +2654,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message - // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) - std::vector headers; - headers.push_back(cmpctblock.header); - vHeadersMsg << headers; fRevertToHeaderProcessing = true; } } @@ -2462,8 +2662,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fProcessBLOCKTXN) return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, interruptMsgProc); - if (fRevertToHeaderProcessing) - return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + if (fRevertToHeaderProcessing) { + // Headers received from HB compact block peers are permitted to be + // relayed before full validation (see BIP 152), so we don't want to disconnect + // the peer if the header turns out to be for an invalid block. + // Note that if a peer tries to build on an invalid chain, that + // will be detected and the peer will be banned. + return ProcessHeadersMessage(pfrom, connman, {cmpctblock.header}, chainparams, /*punish_duplicate_invalid=*/false); + } if (fBlockReconstructed) { // If we got here, we were able to optimistically reconstruct a @@ -2594,167 +2800,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - if (nCount == 0) { - // Nothing interesting. Stop asking this peers for more headers. - return true; - } - - const CBlockIndex *pindexLast = nullptr; - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - - // If this looks like it could be a block announcement (nCount < - // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that - // don't connect: - // - Send a getheaders message in response to try to connect the chain. - // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that - // don't connect before giving DoS points - // - Once a headers message is received that is valid and does connect, - // nUnconnectingHeaders gets reset back to 0. - if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { - nodestate->nUnconnectingHeaders++; - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - pindexBestHeader->nHeight, - pfrom->GetId(), nodestate->nUnconnectingHeaders); - // Set hashLastUnknownBlock for this peer, so that if we - // eventually get the headers - even from a different peer - - // we can use this peer to download. - UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); - - if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(pfrom->GetId(), 20); - } - return true; - } - - uint256 hashLastBlock; - for (const CBlockHeader& header : headers) { - if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(pfrom->GetId(), 20); - return error("non-continuous headers sequence"); - } - hashLastBlock = header.GetHash(); - } - } - - CValidationState state; - if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { - int nDoS; - if (state.IsInvalid(nDoS)) { - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS); - } - return error("invalid header received"); - } - } - - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); - } - nodestate->nUnconnectingHeaders = 0; - - assert(pindexLast); - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - - // From here, pindexBestKnownBlock should be guaranteed to be non-null, - // because it is set in UpdateBlockAvailability. Some nullptr checks - // are still present, however, as belt-and-suspenders. - - if (nCount == MAX_HEADERS_RESULTS) { - // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue - // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); - } - - bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); - // If this set of headers is valid and ends in a block with at least as - // much work as our tip, download as much as possible. - if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { - std::vector vToFetch; - const CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. - while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !mapBlocksInFlight.count(pindexWalk->GetBlockHash())) { - // We don't have this block, and it's not yet in flight. - vToFetch.push_back(pindexWalk); - } - pindexWalk = pindexWalk->pprev; - } - // If pindexWalk still isn't on our main chain, we're looking at a - // very large reorg at a time we think we're close to caught up to - // the main chain -- this shouldn't really happen. Bail out on the - // direct fetch and rely on parallel download instead. - if (!chainActive.Contains(pindexWalk)) { - LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); - } else { - std::vector vGetData; - // Download as much as possible, from earliest to latest. - for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - // Can't download any more from this peer - break; - } - vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); - MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); - LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->GetId()); - } - if (vGetData.size() > 1) { - LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); - } - if (vGetData.size() > 0) { - if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // In any case, we want to download using a compact block, not a regular one - vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); - } - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - } - } - } - // If we're in IBD, we want outbound peers that will serve us a useful - // chain. Disconnect peers that are on chains with insufficient work. - if (IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { - // When nCount < MAX_HEADERS_RESULTS, we know we have no more - // headers to fetch from this peer. - if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { - // This peer has too little work on their headers chain to help - // us sync -- disconnect if using an outbound slot (unless - // whitelisted or addnode). - // Note: We compare their tip to nMinimumChainWork (rather than - // chainActive.Tip()) because we won't start block download - // until we have a headers chain that has at least - // nMinimumChainWork, even if a peer has a chain past our tip, - // as an anti-DoS measure. - if (IsOutboundDisconnectionCandidate(pfrom)) { - LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom->GetId()); - pfrom->fDisconnect = true; - } - } - } - - if (!pfrom->fDisconnect && IsOutboundDisconnectionCandidate(pfrom) && nodestate->pindexBestKnownBlock != nullptr) { - // If this is an outbound peer, check to see if we should protect - // it from the bad/lagging chain logic. - if (g_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { - nodestate->m_chain_sync.m_protect = true; - ++g_outbound_peers_with_protect_from_disconnect; - } - } - } + // Headers received via a HEADERS message should be valid, and reflect + // the chain the peer is on. If we receive a known-invalid header, + // disconnect the peer if it is using one of our outbound connection + // slots. + bool should_punish = !pfrom->fInbound && !pfrom->m_manual_connection; + return ProcessHeadersMessage(pfrom, connman, headers, chainparams, should_punish); } else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing diff --git a/src/validation.cpp b/src/validation.cpp index 49c14bbb7933f..72b2d5a4a6dfd 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3408,13 +3408,15 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state } // Exposed wrapper for AcceptBlockHeader -bool ProcessNewBlockHeaders(const std::vector& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) +bool ProcessNewBlockHeaders(const std::vector& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex, CBlockHeader *first_invalid) { + if (first_invalid != nullptr) first_invalid->SetNull(); { LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast if (!AcceptBlockHeader(header, state, chainparams, &pindex)) { + if (first_invalid) *first_invalid = header; return false; } if (ppindex) { diff --git a/src/validation.h b/src/validation.h index 8e6ad606aa903..aecbd25f2fb41 100644 --- a/src/validation.h +++ b/src/validation.h @@ -254,8 +254,9 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr); +bool ProcessNewBlockHeaders(const std::vector& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr, CBlockHeader *first_invalid=nullptr); /** Check whether enough disk space is available for an incoming block */ bool CheckDiskSpace(uint64_t nAdditionalBytes = 0); From c743111c667911372d81afc106d0f2e6f9b0b780 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 31 Oct 2017 13:10:21 +0100 Subject: [PATCH 17/26] Merge #11578: net: Add missing lock in ProcessHeadersMessage(...) 2530bf2 net: Add missing lock in ProcessHeadersMessage(...) (practicalswift) Pull request description: Add missing lock in `ProcessHeadersMessage(...)`. Reading the variable `mapBlockIndex` requires holding the mutex `cs_main`. The new "Disconnect outbound peers relaying invalid headers" code added in commit 37886d5e2f9992678dea4b1bd893f4f10d61d3ad and merged as part of #11568 two days ago did not lock `cs_main` prior to accessing `mapBlockIndex`. Tree-SHA512: b799c234be8043d036183a00bc7867bbf3bd7ffe3baa94c88529da3b3cd0571c31ed11dadfaf29c5b8498341d6d0a3c928029a43b69f3267ef263682c91563a3 --- src/net_processing.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index cb230af06c002..fd50fe1e8a2bc 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1447,8 +1447,8 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast, &first_invalid_header)) { int nDoS; if (state.IsInvalid(nDoS)) { + LOCK(cs_main); if (nDoS > 0) { - LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); } if (punish_duplicate_invalid && mapBlockIndex.find(first_invalid_header.GetHash()) != mapBlockIndex.end()) { From 7d21a78fa1ef7fe5d0b77d680703201266b2e84f Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 1 Nov 2017 14:29:31 +0100 Subject: [PATCH 18/26] Merge #11531: Check that new headers are not a descendant of an invalid block (more effeciently) f3d4adf Make p2p-acceptablock not an extended test (Matt Corallo) 00dcda6 [qa] test that invalid blocks on an invalid chain get a disconnect (Matt Corallo) 015a525 Reject headers building on invalid chains by tracking invalidity (Matt Corallo) 932f118 Accept unrequested blocks with work equal to our tip (Matt Corallo) 3d9c70c Stop always storing blocks from whitelisted peers (Matt Corallo) 3b4ac43 Rewrite p2p-acceptblock in preparation for slight behavior changes (Matt Corallo) Pull request description: @sdaftuar pointed out that the version in #11487 was somewhat DoS-able as someone could feed you a valid chain that forked off the the last checkpoint block and force you to do lots of work just walking backwards across blocks for each new block they gave you. We came up with a few proposals but settled on the one implemented here as likely the simplest without obvious DoS issues. It uses our existing on-load mapBlockIndex walk to make sure everything that descends from an invalid block is marked as such, and then simply caches blocks which we attempted to connect but which were found to be invalid. To avoid DoS issues during IBD, this will need to depend on #11458. Includes tests from #11487. Tree-SHA512: 46aff8332908e122dae72ceb5fe8cd241902c2281a87f58a5fb486bf69d46458d84a096fdcb5f3e8e07fbcf7466232b10c429f4d67855425f11b38ac0bf612e1 --- src/net_processing.cpp | 6 +- src/validation.cpp | 85 ++++++-- test/functional/p2p-acceptblock.py | 314 ++++++++++++++++++----------- test/functional/test_runner.py | 2 +- 4 files changed, 271 insertions(+), 136 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index fd50fe1e8a2bc..6c595595b7a4b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2815,11 +2815,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->GetId()); - // Process all blocks from whitelisted peers, even if not requested, - // unless we're still syncing with the network. - // Such an unrequested block may still be processed, subject to the - // conditions in AcceptBlock(). - bool forceProcessing = pfrom->fWhitelisted && !IsInitialBlockDownload(); + bool forceProcessing = false; const uint256 hash(pblock->GetHash()); { LOCK(cs_main); diff --git a/src/validation.cpp b/src/validation.cpp index 72b2d5a4a6dfd..4c468ddfe2413 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -171,6 +171,26 @@ namespace { /** chainwork for the last block that preciousblock has been applied to. */ arith_uint256 nLastPreciousChainwork = 0; + /** In order to efficiently track invalidity of headers, we keep the set of + * blocks which we tried to connect and found to be invalid here (ie which + * were set to BLOCK_FAILED_VALID since the last restart). We can then + * walk this set and check if a new header is a descendant of something in + * this set, preventing us from having to walk mapBlockIndex when we try + * to connect a bad block and fail. + * + * While this is more complicated than marking everything which descends + * from an invalid block as invalid at the time we discover it to be + * invalid, doing so would require walking all of mapBlockIndex to find all + * descendants. Since this case should be very rare, keeping track of all + * BLOCK_FAILED_VALID blocks in a set should be just fine and work just as + * well. + * + * Because we alreardy walk mapBlockIndex in height-order at startup, we go + * ahead and mark descendants of invalid blocks as FAILED_CHILD at that time, + * instead of putting things in this set. + */ + std::set g_failed_blocks; + /** Dirty block index entries. */ std::set setDirtyBlockIndex; @@ -1205,6 +1225,7 @@ void static InvalidChainFound(CBlockIndex* pindexNew) void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { if (!state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; + g_failed_blocks.insert(pindex); setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); @@ -2891,10 +2912,14 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C { AssertLockHeld(cs_main); - // Mark the block itself as invalid. - pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); - setBlockIndexCandidates.erase(pindex); + // We first disconnect backwards and then mark the blocks as invalid. + // This prevents a case where pruned nodes may fail to invalidateblock + // and be left unable to start as they have no tip candidates (as there + // are no blocks that meet the "have data and are not invalid per + // nStatus" criteria for inclusion in setBlockIndexCandidates). + + bool pindex_was_in_chain = false; + CBlockIndex *invalid_walk_tip = chainActive.Tip(); if (pindex == pindexBestHeader) { pindexBestInvalid = pindexBestHeader; @@ -2903,10 +2928,8 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C DisconnectedBlockTransactions disconnectpool; while (chainActive.Contains(pindex)) { - CBlockIndex *pindexWalk = chainActive.Tip(); - pindexWalk->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(pindexWalk); - setBlockIndexCandidates.erase(pindexWalk); + const CBlockIndex* pindexOldTip = chainActive.Tip(); + pindex_was_in_chain = true; // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. if (!DisconnectTip(state, chainparams, &disconnectpool)) { @@ -2915,12 +2938,27 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C UpdateMempoolForReorg(disconnectpool, false); return false; } - if (pindexWalk == pindexBestHeader) { + if (pindexOldTip == pindexBestHeader) { pindexBestInvalid = pindexBestHeader; pindexBestHeader = pindexBestHeader->pprev; } } + // Now mark the blocks we just disconnected as descendants invalid + // (note this may not be all descendants). + while (pindex_was_in_chain && invalid_walk_tip != pindex) { + invalid_walk_tip->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk_tip); + setBlockIndexCandidates.erase(invalid_walk_tip); + invalid_walk_tip = invalid_walk_tip->pprev; + } + + // Mark the block itself as invalid. + pindex->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(pindex); + setBlockIndexCandidates.erase(pindex); + g_failed_blocks.insert(pindex); + // DisconnectTip will add transactions to disconnectpool; try to add these // back to the mempool. UpdateMempoolForReorg(disconnectpool, true); @@ -2959,6 +2997,7 @@ bool ResetBlockFailureFlags(CBlockIndex *pindex) { // Reset invalid block marker if it was pointing to one of those. pindexBestInvalid = nullptr; } + g_failed_blocks.erase(it->second); } it++; } @@ -3386,6 +3425,21 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); + if (!pindexPrev->IsValid(BLOCK_VALID_SCRIPTS)) { + for (const CBlockIndex* failedit : g_failed_blocks) { + if (pindexPrev->GetAncestor(failedit->nHeight) == failedit) { + assert(failedit->nStatus & BLOCK_FAILED_VALID); + CBlockIndex* invalid_walk = pindexPrev; + while (invalid_walk != failedit) { + invalid_walk->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk); + invalid_walk = invalid_walk->pprev; + } + return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + } + } + } + if (llmq::chainLocksHandler->HasConflictingChainLock(pindexPrev->nHeight + 1, hash)) { if (pindex == nullptr) { AddToBlockIndex(block, BLOCK_CONFLICT_CHAINLOCK); @@ -3446,7 +3500,7 @@ static bool AcceptBlock(const std::shared_ptr& pblock, CValidation // process an unrequested block if it's new and has enough work to // advance our tip, and isn't too many blocks ahead. bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; - bool fHasMoreWork = (chainActive.Tip() ? pindex->nChainWork > chainActive.Tip()->nChainWork : true); + bool fHasMoreOrSameWork = (chainActive.Tip() ? pindex->nChainWork >= chainActive.Tip()->nChainWork : true); // Blocks that are too out-of-order needlessly limit the effectiveness of // pruning, because pruning will not delete block files that contain any // blocks which are too close in height to the tip. Apply this test @@ -3463,9 +3517,9 @@ static bool AcceptBlock(const std::shared_ptr& pblock, CValidation // and unrequested blocks. if (fAlreadyHave) return true; if (!fRequested) { // If we didn't ask for it: - if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned - if (!fHasMoreWork) return true; // Don't process less-work chains - if (fTooFarAhead) return true; // Block height is too high + if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned + if (!fHasMoreOrSameWork) return true; // Don't process less-work chains + if (fTooFarAhead) return true; // Block height is too high // Protect against DoS attacks from low-work chains. // If our tip is behind, a peer could try to send us @@ -3838,6 +3892,10 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) pindex->nChainTx = pindex->nTx; } } + if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { + pindex->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(pindex); + } if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == nullptr)) setBlockIndexCandidates.insert(pindex); if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) @@ -4151,6 +4209,7 @@ void UnloadBlockIndex() nLastBlockFile = 0; nBlockSequenceId = 1; setDirtyBlockIndex.clear(); + g_failed_blocks.clear(); setDirtyFileInfo.clear(); versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index 38052d103f0c1..085c8ecf78023 100755 --- a/test/functional/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -4,42 +4,32 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. -Since behavior differs when receiving unrequested blocks from whitelisted peers -versus non-whitelisted peers, this tests the behavior of both (effectively two -separate tests running in parallel). +Setup: two nodes, node0+node1, not connected to each other. Node1 will have +nMinimumChainWork set to 0x10, so it won't process low-work unrequested blocks. -Setup: three nodes, node0+node1+node2, not connected to each other. Node0 does not -whitelist localhost, but node1 does. They will each be on their own chain for -this test. Node2 will have nMinimumChainWork set to 0x10, so it won't process -low-work unrequested blocks. - -We have one NodeConn connection to each, test_node, white_node, and min_work_node, -respectively. +We have one NodeConn connection to node0 called test_node, and one to node1 +called min_work_node. The test: 1. Generate one block on each node, to leave IBD. 2. Mine a new block on each tip, and deliver to each node from node's peer. - The tip should advance for node0 and node1, but node2 should skip processing - due to nMinimumChainWork. + The tip should advance for node0, but node1 should skip processing due to + nMinimumChainWork. -Node2 is unused in tests 3-7: +Node1 is unused in tests 3-7: -3. Mine a block that forks the previous block, and deliver to each node from - corresponding peer. - Node0 should not process this block (just accept the header), because it is - unrequested and doesn't have more work than the tip. - Node1 should process because this is coming from a whitelisted peer. +3. Mine a block that forks from the genesis block, and deliver to test_node. + Node0 should not process this block (just accept the header), because it + is unrequested and doesn't have more or equal work to the tip. -4. Send another block that builds on the forking block. - Node0 should process this block but be stuck on the shorter chain, because - it's missing an intermediate block. - Node1 should reorg to this longer chain. +4a,b. Send another two blocks that build on the forking block. + Node0 should process the second block but be stuck on the shorter chain, + because it's missing an intermediate block. -4b.Send 288 more blocks on the longer chain. +4c.Send 288 more blocks on the longer chain (the number of blocks ahead + we currently store). Node0 should process all but the last block (too far ahead in height). - Send all headers to Node1, and then send the last block in that chain. - Node1 should accept the block because it's coming from a whitelisted peer. 5. Send a duplicate of the block in #3 to Node0. Node0 should not process the block because it is unrequested, and stay on @@ -52,16 +42,20 @@ 7. Send Node0 the missing block again. Node0 should process and the tip should advance. -8. Test Node2 is able to sync when connected to node0 (which should have sufficient -work on its chain). +8. Create a fork which is invalid at a height longer than the current chain + (ie to which the node will try to reorg) but which has headers built on top + of the invalid block. Check that we get disconnected if we send more headers + on the chain the node now knows to be invalid. +9. Test Node1 is able to sync when connected to node0 (which should have sufficient + work on its chain). """ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * import time -from test_framework.blocktools import create_block, create_coinbase +from test_framework.blocktools import create_block, create_coinbase, create_transaction class AcceptBlockTest(BitcoinTestFramework): def add_options(self, parser): @@ -71,8 +65,8 @@ def add_options(self, parser): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 3 - self.extra_args = [[], ["-whitelist=127.0.0.1"], ["-minimumchainwork=0x10"]] + self.num_nodes = 2 + self.extra_args = [[], ["-minimumchainwork=0x10"]] def setup_network(self): # Node0 will be used to test behavior of processing unrequested blocks @@ -84,133 +78,147 @@ def setup_network(self): def run_test(self): # Setup the p2p connections and start up the network thread. - test_node = NodeConnCB() # connects to node0 (not whitelisted) - white_node = NodeConnCB() # connects to node1 (whitelisted) - min_work_node = NodeConnCB() # connects to node2 (not whitelisted) + test_node = NodeConnCB() # connects to node0 + min_work_node = NodeConnCB() # connects to node1 connections = [] connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node)) - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], white_node)) - connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], min_work_node)) + connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], min_work_node)) test_node.add_connection(connections[0]) - white_node.add_connection(connections[1]) - min_work_node.add_connection(connections[2]) + min_work_node.add_connection(connections[1]) NetworkThread().start() # Start up network handling in another thread # Test logic begins here test_node.wait_for_verack() - white_node.wait_for_verack() min_work_node.wait_for_verack() - # 1. Have nodes mine a block (nodes1/2 leave IBD) + # 1. Have nodes mine a block (leave IBD) [ n.generate(1) for n in self.nodes ] tips = [ int("0x" + n.getbestblockhash(), 0) for n in self.nodes ] # 2. Send one block that builds on each tip. - # This should be accepted by nodes 1/2 + # This should be accepted by node0 blocks_h2 = [] # the height 2 blocks on each node's chain block_time = self.mocktime + 1 - for i in range(3): + for i in range(2): blocks_h2.append(create_block(tips[i], create_coinbase(2), block_time + 1)) blocks_h2[i].solve() block_time += 1 test_node.send_message(msg_block(blocks_h2[0])) - white_node.send_message(msg_block(blocks_h2[1])) - min_work_node.send_message(msg_block(blocks_h2[2])) + min_work_node.send_message(msg_block(blocks_h2[1])) - for x in [test_node, white_node, min_work_node]: + for x in [test_node, min_work_node]: x.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) - assert_equal(self.nodes[1].getblockcount(), 2) - assert_equal(self.nodes[2].getblockcount(), 1) - self.log.info("First height 2 block accepted by node0/node1; correctly rejected by node2") + assert_equal(self.nodes[1].getblockcount(), 1) + self.log.info("First height 2 block accepted by node0; correctly rejected by node1") - # 3. Send another block that builds on the original tip. - blocks_h2f = [] # Blocks at height 2 that fork off the main chain - for i in range(2): - blocks_h2f.append(create_block(tips[i], create_coinbase(2), blocks_h2[i].nTime+1)) - blocks_h2f[i].solve() - test_node.send_message(msg_block(blocks_h2f[0])) - white_node.send_message(msg_block(blocks_h2f[1])) + # 3. Send another block that builds on genesis. + block_h1f = create_block(int("0x" + self.nodes[0].getblockhash(0), 0), create_coinbase(1), block_time) + block_time += 1 + block_h1f.solve() + test_node.send_message(msg_block(block_h1f)) - for x in [test_node, white_node]: - x.sync_with_ping() + test_node.sync_with_ping() + tip_entry_found = False for x in self.nodes[0].getchaintips(): - if x['hash'] == blocks_h2f[0].hash: + if x['hash'] == block_h1f.hash: assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) - for x in self.nodes[1].getchaintips(): - if x['hash'] == blocks_h2f[1].hash: - assert_equal(x['status'], "valid-headers") + # 4. Send another two block that build on the fork. + block_h2f = create_block(block_h1f.sha256, create_coinbase(2), block_time) + block_time += 1 + block_h2f.solve() + test_node.send_message(msg_block(block_h2f)) - self.log.info("Second height 2 block accepted only from whitelisted peer") + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block + # can't be fully validated. + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_h2f.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) - # 4. Now send another block that builds on the forking chain. - blocks_h3 = [] - for i in range(2): - blocks_h3.append(create_block(blocks_h2f[i].sha256, create_coinbase(3), blocks_h2f[i].nTime+1)) - blocks_h3[i].solve() - test_node.send_message(msg_block(blocks_h3[0])) - white_node.send_message(msg_block(blocks_h3[1])) + # But this block should be accepted by node since it has equal work. + self.nodes[0].getblock(block_h2f.hash) + self.log.info("Second height 2 block accepted, but not reorg'ed to") - for x in [test_node, white_node]: - x.sync_with_ping() - # Since the earlier block was not processed by node0, the new block + # 4b. Now send another block that builds on the forking chain. + block_h3 = create_block(block_h2f.sha256, create_coinbase(3), block_h2f.nTime+1) + block_h3.solve() + test_node.send_message(msg_block(block_h3)) + + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block # can't be fully validated. + tip_entry_found = False for x in self.nodes[0].getchaintips(): - if x['hash'] == blocks_h3[0].hash: + if x['hash'] == block_h3.hash: assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + self.nodes[0].getblock(block_h3.hash) + + # But this block should be accepted by node since it has more work. + self.nodes[0].getblock(block_h3.hash) + self.log.info("Unrequested more-work block accepted") + + # 4c. Now mine 288 more blocks and deliver; all should be processed but + # the last (height-too-high) on node (as long as its not missing any headers) + tip = block_h3 + all_blocks = [] + for i in range(288): + next_block = create_block(tip.sha256, create_coinbase(i + 4), tip.nTime+1) + next_block.solve() + all_blocks.append(next_block) + tip = next_block + + # Now send the block at height 5 and check that it wasn't accepted (missing header) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblock, all_blocks[1].hash) + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockheader, all_blocks[1].hash) - # But this block should be accepted by node0 since it has more work. - self.nodes[0].getblock(blocks_h3[0].hash) - self.log.info("Unrequested more-work block accepted from non-whitelisted peer") + # The block at height 5 should be accepted if we provide the missing header, though + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(all_blocks[0])) + test_node.send_message(headers_message) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + self.nodes[0].getblock(all_blocks[1].hash) - # Node1 should have accepted and reorged. - assert_equal(self.nodes[1].getblockcount(), 3) - self.log.info("Successfully reorged to length 3 chain from whitelisted peer") + # Now send the blocks in all_blocks + for i in range(288): + test_node.send_message(msg_block(all_blocks[i])) + test_node.sync_with_ping() - # 4b. Now mine 288 more blocks and deliver; all should be processed but - # the last (height-too-high) on node0. Node1 should process the tip if - # we give it the headers chain leading to the tip. - tips = blocks_h3 - headers_message = msg_headers() - all_blocks = [] # node0's blocks - for j in range(2): - for i in range(288): - next_block = create_block(tips[j].sha256, create_coinbase(i + 4), tips[j].nTime+1) - next_block.solve() - if j==0: - test_node.send_message(msg_block(next_block)) - all_blocks.append(next_block) - else: - headers_message.headers.append(CBlockHeader(next_block)) - tips[j] = next_block - - self.bump_mocktime(2) - set_node_times(self.nodes, self.mocktime) # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead for x in all_blocks[:-1]: self.nodes[0].getblock(x.hash) assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) - headers_message.headers.pop() # Ensure the last block is unrequested - white_node.send_message(headers_message) # Send headers leading to tip - white_node.send_message(msg_block(tips[1])) # Now deliver the tip - white_node.sync_with_ping() - self.nodes[1].getblock(tips[1].hash) - self.log.info("Unrequested block far ahead of tip accepted from whitelisted peer") - # 5. Test handling of unrequested block on the node that didn't process # Should still not be processed (even though it has a child that has more # work). - test_node.send_message(msg_block(blocks_h2f[0])) - # Here, if the sleep is too short, the test could falsely succeed (if the - # node hasn't processed the block by the time the sleep returns, and then - # the node processes it and incorrectly advances the tip). - # But this would be caught later on, when we verify that an inv triggers - # a getdata request for this block. + # The node should have requested the blocks at some point, so + # disconnect/reconnect first + connections[0].disconnect_node() + test_node.wait_for_disconnect() + + test_node = NodeConnCB() # connects to node (not whitelisted) + connections[0] = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node) + test_node.add_connection(connections[0]) + + test_node.wait_for_verack() + test_node.send_message(msg_block(block_h1f)) + test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) self.log.info("Unrequested block that would complete more-work chain was ignored") @@ -221,27 +229,99 @@ def run_test(self): with mininode_lock: # Clear state so we can check the getdata request test_node.last_message.pop("getdata", None) - test_node.send_message(msg_inv([CInv(2, blocks_h3[0].sha256)])) + test_node.send_message(msg_inv([CInv(2, block_h3.sha256)])) test_node.sync_with_ping() with mininode_lock: getdata = test_node.last_message["getdata"] # Check that the getdata includes the right block - assert_equal(getdata.inv[0].hash, blocks_h2f[0].sha256) + assert_equal(getdata.inv[0].hash, block_h1f.sha256) self.log.info("Inv at tip triggered getdata for unprocessed block") # 7. Send the missing block for the third time (now it is requested) - test_node.send_message(msg_block(blocks_h2f[0])) + test_node.send_message(msg_block(block_h1f)) test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 290) + self.nodes[0].getblock(all_blocks[286].hash) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) self.log.info("Successfully reorged to longer chain from non-whitelisted peer") - # 8. Connect node2 to node0 and ensure it is able to sync - connect_nodes(self.nodes[0], 2) - sync_blocks([self.nodes[0], self.nodes[2]]) - self.log.info("Successfully synced nodes 2 and 0") + # 8. Create a chain which is invalid at a height longer than the + # current chain, but which has more blocks on top of that + block_289f = create_block(all_blocks[284].sha256, create_coinbase(289), all_blocks[284].nTime+1) + block_289f.solve() + block_290f = create_block(block_289f.sha256, create_coinbase(290), block_289f.nTime+1) + block_290f.solve() + block_291 = create_block(block_290f.sha256, create_coinbase(291), block_290f.nTime+1) + # block_291 spends a coinbase below maturity! + block_291.vtx.append(create_transaction(block_290f.vtx[0], 0, b"42", 1)) + block_291.hashMerkleRoot = block_291.calc_merkle_root() + block_291.solve() + block_292 = create_block(block_291.sha256, create_coinbase(292), block_291.nTime+1) + block_292.solve() + + # Now send all the headers on the chain and enough blocks to trigger reorg + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_289f)) + headers_message.headers.append(CBlockHeader(block_290f)) + headers_message.headers.append(CBlockHeader(block_291)) + headers_message.headers.append(CBlockHeader(block_292)) + test_node.send_message(headers_message) + + test_node.sync_with_ping() + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_292.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) + + test_node.send_message(msg_block(block_289f)) + test_node.send_message(msg_block(block_290f)) + + test_node.sync_with_ping() + self.nodes[0].getblock(block_289f.hash) + self.nodes[0].getblock(block_290f.hash) + + test_node.send_message(msg_block(block_291)) + + # At this point we've sent an obviously-bogus block, wait for full processing + # without assuming whether we will be disconnected or not + try: + # Only wait a short while so the test doesn't take forever if we do get + # disconnected + test_node.sync_with_ping(timeout=1) + except AssertionError: + test_node.wait_for_disconnect() + + test_node = NodeConnCB() # connects to node (not whitelisted) + connections[0] = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node) + test_node.add_connection(connections[0]) + + NetworkThread().start() # Start up network handling in another thread + test_node.wait_for_verack() + + # We should have failed reorg and switched back to 290 (but have block 291) + assert_equal(self.nodes[0].getblockcount(), 290) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_equal(self.nodes[0].getblock(block_291.hash)["confirmations"], -1) + + # Now send a new header on the invalid chain, indicating we're forked off, and expect to get disconnected + block_293 = create_block(block_292.sha256, create_coinbase(293), block_292.nTime+1) + block_293.solve() + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_293)) + test_node.send_message(headers_message) + test_node.wait_for_disconnect() + + # 9. Connect node1 to node0 and ensure it is able to sync + connect_nodes(self.nodes[0], 1) + sync_blocks([self.nodes[0], self.nodes[1]]) + self.log.info("Successfully synced nodes 1 and 0") [ c.disconnect_node() for c in connections ] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index f523410cc9527..441d7d1d7dff7 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -136,6 +136,7 @@ 'uptime.py', 'resendwallettransactions.py', 'minchainwork.py', + 'p2p-acceptblock.py', # NOTE: needs dash_hash to pass ] EXTENDED_SCRIPTS = [ @@ -163,7 +164,6 @@ 'txindex.py', 'forknotify.py', 'invalidateblock.py', - 'p2p-acceptblock.py', # NOTE: needs dash_hash to pass ] # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests From 859b962a29e7f369efe65b5c973c93498c665637 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 25 Sep 2019 13:03:45 +0200 Subject: [PATCH 19/26] Move DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN up This avoids future merge conflicts --- src/net_processing.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/net_processing.h b/src/net_processing.h index 8253938a27f3f..e29d7cb993242 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -15,6 +15,8 @@ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; static const int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; /** Minimum time between orphan transactions expire time checks in seconds */ static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; +/** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ +static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; /** Headers download timeout expressed in microseconds * Timeout = base + per_header * (expected number of headers) */ @@ -27,9 +29,6 @@ static constexpr int32_t MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT = 4; /** Timeout for (unprotected) outbound peers to sync to our chainwork, in seconds */ static constexpr int64_t CHAIN_SYNC_TIMEOUT = 20 * 60; // 20 minutes -/** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ -static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; - class PeerLogicValidation : public CValidationInterface, public NetEventsInterface { private: CConnman* connman; From 58cb7e38f49c919b76222d3995869d3697fb991f Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 2 Nov 2017 20:13:17 +0100 Subject: [PATCH 20/26] Merge #11560: Connect to a new outbound peer if our tip is stale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 6262915 Add unit test for stale tip checking (Suhas Daftuar) 83df257 Add CConnmanTest to mutate g_connman in tests (João Barbosa) ac7b37c Connect to an extra outbound peer if our tip is stale (Suhas Daftuar) db32a65 Track tip update time and last new block announcement from each peer (Suhas Daftuar) 2d4327d net: Allow connecting to extra outbound peers (Suhas Daftuar) Pull request description: This is an alternative approach to #11534. Rather than disconnect an outbound peer when our tip looks stale, instead try to connect to an additional outbound peer. Periodically, check to see if we have more outbound peers than we target (ie if any extra peers are in use), and if so, disconnect the one that least recently announced a new block (breaking ties by choosing the newest peer that we connected to). Tree-SHA512: 8f19e910e0bb36867f81783e020af225f356451899adfc7ade1895d6d3bd5afe51c83759610dfd10c62090c4fe404efa0283b2f63fde0bd7da898a1aaa7fb281 --- src/init.cpp | 2 +- src/net.cpp | 35 +++++++++- src/net.h | 20 ++++++ src/net_processing.cpp | 146 ++++++++++++++++++++++++++++++++++++++--- src/net_processing.h | 17 ++++- src/test/DoS_tests.cpp | 85 ++++++++++++++++++++++++ src/test/test_dash.cpp | 14 +++- src/test/test_dash.h | 6 ++ 8 files changed, 312 insertions(+), 13 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 965cc31442652..aaeb5e4b0b45a 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1585,7 +1585,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) g_connman = std::unique_ptr(new CConnman(GetRand(std::numeric_limits::max()), GetRand(std::numeric_limits::max()))); CConnman& connman = *g_connman; - peerLogic.reset(new PeerLogicValidation(&connman)); + peerLogic.reset(new PeerLogicValidation(&connman, scheduler)); RegisterValidationInterface(peerLogic.get()); // sanitize comments per BIP-0014, format user agent and check total size diff --git a/src/net.cpp b/src/net.cpp index df7d032a933a6..8785b97bf7b51 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1804,6 +1804,37 @@ void CConnman::ProcessOneShot() } } +bool CConnman::GetTryNewOutboundPeer() +{ + return m_try_another_outbound_peer; +} + +void CConnman::SetTryNewOutboundPeer(bool flag) +{ + m_try_another_outbound_peer = flag; + LogPrint(BCLog::NET, "net: setting try another outbound peer=%s\n", flag ? "true" : "false"); +} + +// Return the number of peers we have over our outbound connection limit +// Exclude peers that are marked for disconnect, or are going to be +// disconnected soon (eg one-shots and feelers) +// Also exclude peers that haven't finished initial connection handshake yet +// (so that we don't decide we're over our desired connection limit, and then +// evict some peer that has finished the handshake) +int CConnman::GetExtraOutboundCount() +{ + int nOutbound = 0; + { + LOCK(cs_vNodes); + for (CNode* pnode : vNodes) { + if (!pnode->fInbound && !pnode->m_manual_connection && !pnode->fFeeler && !pnode->fDisconnect && !pnode->fOneShot && pnode->fSuccessfullyConnected) { + ++nOutbound; + } + } + } + return std::max(nOutbound - nMaxOutbound, 0); +} + void CConnman::ThreadOpenConnections() { // Connect to specific addresses @@ -1893,7 +1924,8 @@ void CConnman::ThreadOpenConnections() // * Only make a feeler connection once every few minutes. // bool fFeeler = false; - if (nOutbound >= nMaxOutbound) { + + if (nOutbound >= nMaxOutbound && !GetTryNewOutboundPeer()) { int64_t nTime = GetTimeMicros(); // The current time right now (in microseconds). if (nTime > nNextFeeler) { nNextFeeler = PoissonNextSend(nTime, FEELER_INTERVAL); @@ -2424,6 +2456,7 @@ CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : semAddnode = nullptr; semMasternodeOutbound = nullptr; flagInterruptMsgProc = false; + SetTryNewOutboundPeer(false); Options connOptions; Init(connOptions); diff --git a/src/net.h b/src/net.h index 63aea36460d35..2032a3aefef48 100644 --- a/src/net.h +++ b/src/net.h @@ -379,6 +379,19 @@ class CConnman void GetBanned(banmap_t &banmap); void SetBanned(const banmap_t &banmap); + // This allows temporarily exceeding nMaxOutbound, with the goal of finding + // a peer that is better than all our current peers. + void SetTryNewOutboundPeer(bool flag); + bool GetTryNewOutboundPeer(); + + // Return the number of outbound peers we have in excess of our target (eg, + // if we previously called SetTryNewOutboundPeer(true), and have since set + // to false, we may have extra peers that we wish to disconnect). This may + // return a value less than (num_outbound_connections - num_outbound_slots) + // in cases where some outbound connections are not yet fully connected, or + // not yet fully disconnected. + int GetExtraOutboundCount(); + bool AddNode(const std::string& node); bool RemoveAddedNode(const std::string& node); std::vector GetAddedNodeInfo(); @@ -564,6 +577,13 @@ class CConnman std::thread threadOpenConnections; std::thread threadOpenMasternodeConnections; std::thread threadMessageHandler; + + /** flag for deciding to connect to an extra outbound peer, + * in excess of nMaxOutbound + * This takes the place of a feeler connection */ + std::atomic_bool m_try_another_outbound_peer; + + friend struct CConnmanTest; }; extern std::unique_ptr g_connman; void Discover(boost::thread_group& threadGroup); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6c595595b7a4b..172ea808cf0c4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -23,6 +23,7 @@ #include "primitives/transaction.h" #include "random.h" #include "reverse_iterator.h" +#include "scheduler.h" #include "tinyformat.h" #include "txmempool.h" #include "ui_interface.h" @@ -150,12 +151,8 @@ namespace { /** Number of outbound peers with m_chain_sync.m_protect. */ int g_outbound_peers_with_protect_from_disconnect = 0; - /* This comment is here to force a merge conflict when bitcoin#11560 is backported - * It introduces this member as int64_t while bitcoin#11824 changes it to atomic. - * bitcoin#11824 is partially backported already, which means you'll have to take the atomic - * version when you encounter this merge conflict! + /** When our tip was last updated. */ std::atomic g_last_tip_update(0); - */ /** Relay map, protected by cs_main. */ typedef std::map MapRelay; @@ -253,6 +250,9 @@ struct CNodeState { ChainSyncTimeoutState m_chain_sync; + //! Time of last new block announcement + int64_t m_last_block_announcement; + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; nMisbehavior = 0; @@ -274,6 +274,7 @@ struct CNodeState { fProvidesHeaderAndIDs = false; fSupportsDesiredCmpctVersion = false; m_chain_sync = { 0, nullptr, false, false }; + m_last_block_announcement = 0; } }; @@ -455,6 +456,15 @@ void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman* connman) { } } +bool TipMayBeStale(const Consensus::Params &consensusParams) +{ + AssertLockHeld(cs_main); + if (g_last_tip_update == 0) { + g_last_tip_update = GetTime(); + } + return g_last_tip_update < GetTime() - consensusParams.nPowTargetSpacing * 3 && mapBlocksInFlight.empty(); +} + // Requires cs_main bool CanDirectFetch(const Consensus::Params &consensusParams) { @@ -557,6 +567,15 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vectorm_last_block_announcement = time_in_seconds; +} + // Returns true for outbound peers, excluding manual connections, feelers, and // one-shots bool IsOutboundDisconnectionCandidate(const CNode *node) @@ -803,9 +822,17 @@ static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Para (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); } -PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) { +PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, CScheduler &scheduler) : connman(connmanIn), m_stale_tip_check_time(0) { // Initialize global variables that cannot be constructed at startup. recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); + + const Consensus::Params& consensusParams = Params().GetConsensus(); + // Stale tip checking and peer eviction are on two different timers, but we + // don't want them to get out of sync due to drift in the scheduler, so we + // combine them in one function and schedule at the quicker (peer-eviction) + // timer. + static_assert(EXTRA_PEER_CHECK_INTERVAL < STALE_CHECK_INTERVAL, "peer eviction timer should be less than stale tip check timer"); + scheduler.scheduleEvery(std::bind(&PeerLogicValidation::CheckForStaleTipAndEvictPeers, this, consensusParams), EXTRA_PEER_CHECK_INTERVAL * 1000); } void PeerLogicValidation::BlockConnected(const std::shared_ptr& pblock, const CBlockIndex* pindex, const std::vector& vtxConflicted) { @@ -836,6 +863,8 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr& pb } LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } + + g_last_tip_update = GetTime(); } // All of the following cache a recent block, and are protected by cs_most_recent_block @@ -1400,6 +1429,7 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve return true; } + bool received_new_header = false; const CBlockIndex *pindexLast = nullptr; { LOCK(cs_main); @@ -1440,6 +1470,12 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve } hashLastBlock = header.GetHash(); } + + // If we don't have the last header, then they'll have given us + // something new (if these headers are valid). + if (mapBlockIndex.find(hashLastBlock) == mapBlockIndex.end()) { + received_new_header = true; + } } CValidationState state; @@ -1504,6 +1540,10 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve // because it is set in UpdateBlockAvailability. Some nullptr checks // are still present, however, as belt-and-suspenders. + if (received_new_header && pindexLast->nChainWork > chainActive.Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + if (nCount == MAX_HEADERS_RESULTS) { // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue @@ -1586,6 +1626,7 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve // If this is an outbound peer, check to see if we should protect // it from the bad/lagging chain logic. if (g_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom->GetId()); nodestate->m_chain_sync.m_protect = true; ++g_outbound_peers_with_protect_from_disconnect; } @@ -2509,6 +2550,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr CBlockHeaderAndShortTxIDs cmpctblock; vRecv >> cmpctblock; + bool received_new_header = false; + { LOCK(cs_main); @@ -2518,6 +2561,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); return true; } + + if (mapBlockIndex.find(cmpctblock.header.GetHash()) == mapBlockIndex.end()) { + received_new_header = true; + } } const CBlockIndex *pindex = nullptr; @@ -2556,6 +2603,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr assert(pindex); UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + CNodeState *nodestate = State(pfrom->GetId()); + + // If this was a new header with more work than our tip, update the + // peer's last block announcement time + if (received_new_header && pindex->nChainWork > chainActive.Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + std::map::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); @@ -2578,8 +2633,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) return true; - CNodeState *nodestate = State(pfrom->GetId()); - // We want to be a bit conservative just to be extra careful about DoS // possibilities in compact block processing... if (pindex->nHeight <= chainActive.Height() + 2) { @@ -3290,6 +3343,83 @@ void PeerLogicValidation::ConsiderEviction(CNode *pto, int64_t time_in_seconds) } } +void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) +{ + // Check whether we have too many outbound peers + int extra_peers = connman->GetExtraOutboundCount(); + if (extra_peers > 0) { + // If we have more outbound peers than we target, disconnect one. + // Pick the outbound peer that least recently announced + // us a new block, with ties broken by choosing the more recent + // connection (higher node id) + NodeId worst_peer = -1; + int64_t oldest_block_announcement = std::numeric_limits::max(); + + LOCK(cs_main); + + connman->ForEachNode([&](CNode* pnode) { + // Ignore non-outbound peers, or nodes marked for disconnect already + if (!IsOutboundDisconnectionCandidate(pnode) || pnode->fDisconnect) return; + CNodeState *state = State(pnode->GetId()); + if (state == nullptr) return; // shouldn't be possible, but just in case + // Don't evict our protected peers + if (state->m_chain_sync.m_protect) return; + if (state->m_last_block_announcement < oldest_block_announcement || (state->m_last_block_announcement == oldest_block_announcement && pnode->GetId() > worst_peer)) { + worst_peer = pnode->GetId(); + oldest_block_announcement = state->m_last_block_announcement; + } + }); + if (worst_peer != -1) { + bool disconnected = connman->ForNode(worst_peer, [&](CNode *pnode) { + // Only disconnect a peer that has been connected to us for + // some reasonable fraction of our check-frequency, to give + // it time for new information to have arrived. + // Also don't disconnect any peer we're trying to download a + // block from. + CNodeState &state = *State(pnode->GetId()); + if (time_in_seconds - pnode->nTimeConnected > MINIMUM_CONNECT_TIME && state.nBlocksInFlight == 0) { + LogPrint(BCLog::NET, "disconnecting extra outbound peer=%d (last block announcement received at time %d)\n", pnode->GetId(), oldest_block_announcement); + pnode->fDisconnect = true; + return true; + } else { + LogPrint(BCLog::NET, "keeping outbound peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", pnode->GetId(), pnode->nTimeConnected, state.nBlocksInFlight); + return false; + } + }); + if (disconnected) { + // If we disconnected an extra peer, that means we successfully + // connected to at least one peer after the last time we + // detected a stale tip. Don't try any more extra peers until + // we next detect a stale tip, to limit the load we put on the + // network from these extra connections. + connman->SetTryNewOutboundPeer(false); + } + } + } +} + +void PeerLogicValidation::CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams) +{ + if (connman == nullptr) return; + + int64_t time_in_seconds = GetTime(); + + EvictExtraOutboundPeers(time_in_seconds); + + if (time_in_seconds > m_stale_tip_check_time) { + LOCK(cs_main); + // Check whether our tip is stale, and if so, allow using an extra + // outbound peer + if (TipMayBeStale(consensusParams)) { + LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", time_in_seconds - g_last_tip_update); + connman->SetTryNewOutboundPeer(true); + } else if (connman->GetTryNewOutboundPeer()) { + connman->SetTryNewOutboundPeer(false); + } + m_stale_tip_check_time = time_in_seconds + STALE_CHECK_INTERVAL; + } +} + class CompareInvMempoolOrder { CTxMemPool *mp; diff --git a/src/net_processing.h b/src/net_processing.h index e29d7cb993242..0afa7065a568a 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -8,6 +8,7 @@ #include "net.h" #include "validationinterface.h" +#include "consensus/params.h" /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; @@ -29,12 +30,19 @@ static constexpr int32_t MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT = 4; /** Timeout for (unprotected) outbound peers to sync to our chainwork, in seconds */ static constexpr int64_t CHAIN_SYNC_TIMEOUT = 20 * 60; // 20 minutes +/** How frequently to check for stale tips, in seconds */ +static constexpr int64_t STALE_CHECK_INTERVAL = 10 * 60; // 10 minutes +/** How frequently to check for extra outbound peers and disconnect, in seconds */ +static constexpr int64_t EXTRA_PEER_CHECK_INTERVAL = 45; +/** Minimum time an outbound-peer-eviction candidate must be connected for, in order to evict, in seconds */ +static constexpr int64_t MINIMUM_CONNECT_TIME = 30; + class PeerLogicValidation : public CValidationInterface, public NetEventsInterface { private: - CConnman* connman; + CConnman* const connman; public: - explicit PeerLogicValidation(CConnman* connmanIn); + explicit PeerLogicValidation(CConnman* connmanIn, CScheduler &scheduler); void BlockConnected(const std::shared_ptr& pblock, const CBlockIndex* pindexConnected, const std::vector& vtxConflicted) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; @@ -56,6 +64,11 @@ class PeerLogicValidation : public CValidationInterface, public NetEventsInterfa bool SendMessages(CNode* pto, std::atomic& interrupt) override; void ConsiderEviction(CNode *pto, int64_t time_in_seconds); + void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams); + void EvictExtraOutboundPeers(int64_t time_in_seconds); + +private: + int64_t m_stale_tip_check_time; //! Next time to check for stale tip }; struct CNodeStateStats { diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index b5f28ee9559e2..4cdd01b242054 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -40,6 +40,8 @@ CService ip(uint32_t i) static NodeId id = 0; +void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds); + BOOST_FIXTURE_TEST_SUITE(DoS_tests, TestingSetup) // Test eviction of an outbound peer whose chain never advances @@ -87,6 +89,89 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); } +void AddRandomOutboundPeer(std::vector &vNodes, PeerLogicValidation &peerLogic) +{ + CAddress addr(ip(GetRandInt(0xffffffff)), NODE_NONE); + vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", /*fInboundIn=*/ false)); + CNode &node = *vNodes.back(); + node.SetSendVersion(PROTOCOL_VERSION); + + peerLogic.InitializeNode(&node); + node.nVersion = 1; + node.fSuccessfullyConnected = true; + + CConnmanTest::AddNode(node); +} + +BOOST_AUTO_TEST_CASE(stale_tip_peer_management) +{ + const Consensus::Params& consensusParams = Params().GetConsensus(); + constexpr int nMaxOutbound = 8; + CConnman::Options options; + options.nMaxConnections = 125; + options.nMaxOutbound = nMaxOutbound; + options.nMaxFeeler = 1; + + connman->Init(options); + std::vector vNodes; + + // Mock some outbound peers + for (int i=0; iCheckForStaleTipAndEvictPeers(consensusParams); + + // No nodes should be marked for disconnection while we have no extra peers + for (const CNode *node : vNodes) { + BOOST_CHECK(node->fDisconnect == false); + } + + SetMockTime(GetTime() + 3*consensusParams.nPowTargetSpacing + 1); + + // Now tip should definitely be stale, and we should look for an extra + // outbound peer + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + BOOST_CHECK(connman->GetTryNewOutboundPeer()); + + // Still no peers should be marked for disconnection + for (const CNode *node : vNodes) { + BOOST_CHECK(node->fDisconnect == false); + } + + // If we add one more peer, something should get marked for eviction + // on the next check (since we're mocking the time to be in the future, the + // required time connected check should be satisfied). + AddRandomOutboundPeer(vNodes, *peerLogic); + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + for (int i=0; ifDisconnect == false); + } + // Last added node should get marked for eviction + BOOST_CHECK(vNodes.back()->fDisconnect == true); + + vNodes.back()->fDisconnect = false; + + // Update the last announced block time for the last + // peer, and check that the next newest node gets evicted. + UpdateLastBlockAnnounceTime(vNodes.back()->GetId(), GetTime()); + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + for (int i=0; ifDisconnect == false); + } + BOOST_CHECK(vNodes[nMaxOutbound-1]->fDisconnect == true); + BOOST_CHECK(vNodes.back()->fDisconnect == false); + + bool dummy; + for (const CNode *node : vNodes) { + peerLogic->FinalizeNode(node->GetId(), dummy); + } + + CConnmanTest::ClearNodes(); +} + BOOST_AUTO_TEST_CASE(DoS_banning) { std::atomic interruptDummy(false); diff --git a/src/test/test_dash.cpp b/src/test/test_dash.cpp index 7d002341226df..c494bcb3e3c6d 100644 --- a/src/test/test_dash.cpp +++ b/src/test/test_dash.cpp @@ -29,6 +29,18 @@ #include +void CConnmanTest::AddNode(CNode& node) +{ + LOCK(g_connman->cs_vNodes); + g_connman->vNodes.push_back(&node); +} + +void CConnmanTest::ClearNodes() +{ + LOCK(g_connman->cs_vNodes); + g_connman->vNodes.clear(); +} + uint256 insecure_rand_seed = GetRandHash(); FastRandomContext insecure_rand_ctx(insecure_rand_seed); @@ -95,7 +107,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha nScriptCheckThreads = 3; for (int i=0; i < nScriptCheckThreads-1; i++) threadGroup.create_thread(&ThreadScriptCheck); - peerLogic.reset(new PeerLogicValidation(connman)); + peerLogic.reset(new PeerLogicValidation(connman, scheduler)); } TestingSetup::~TestingSetup() diff --git a/src/test/test_dash.h b/src/test/test_dash.h index c911666f4fb0c..856523c19536b 100644 --- a/src/test/test_dash.h +++ b/src/test/test_dash.h @@ -50,6 +50,12 @@ struct BasicTestingSetup { * Included are data directory, coins database, script check threads setup. */ class CConnman; +class CNode; +struct CConnmanTest { + static void AddNode(CNode& node); + static void ClearNodes(); +}; + class PeerLogicValidation; struct TestingSetup: public BasicTestingSetup { CCoinsViewDB *pcoinsdbview; From 3e8962323e13af02504e8725ae4cf5eab0cd4316 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 2 Nov 2017 20:10:23 +0100 Subject: [PATCH 21/26] Merge #11593: rpc: work-around an upstream libevent bug 97932cd rpc: further constrain the libevent workaround (Cory Fields) 6b58360 rpc: work-around an upstream libevent bug (Cory Fields) Pull request description: A rare race condition may trigger while awaiting the body of a message. This may fix some reported rpc hangs/crashes. This work-around mimics what libevent does internally once a write has started, which is what usually happens, but not always due to the processing happening on a different thread: https://github.com/libevent/libevent/blob/e7ff4ef2b4fc950a765008c18e74281cdb5e7668/http.c#L373 Fixed upstream at: https://github.com/libevent/libevent/commit/5ff8eb26371c4dc56f384b2de35bea2d87814779 Tree-SHA512: b9fa97cae9da2a44101c5faf1e3be0b9cbdf722982d35541cf224be31430779c75e519c8ed18d06ab7487bfb1211069b28f22739f126d6c28ca62d3f73b79a52 --- src/httpserver.cpp | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 633c486463c82..15f29beeac464 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -240,6 +241,16 @@ static std::string RequestMethodString(HTTPRequest::RequestMethod m) /** HTTP request callback */ static void http_request_cb(struct evhttp_request* req, void* arg) { + // Disable reading to work around a libevent bug, fixed in 2.2.0. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + evhttp_connection* conn = evhttp_request_get_connection(req); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_disable(bev, EV_READ); + } + } + } std::unique_ptr hreq(new HTTPRequest(req)); LogPrint(BCLog::HTTP, "Received a %s request for %s from %s\n", @@ -606,8 +617,21 @@ void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) struct evbuffer* evb = evhttp_request_get_output_buffer(req); assert(evb); evbuffer_add(evb, strReply.data(), strReply.size()); - HTTPEvent* ev = new HTTPEvent(eventBase, true, - std::bind(evhttp_send_reply, req, nStatus, (const char*)nullptr, (struct evbuffer *)nullptr)); + auto req_copy = req; + HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus]{ + evhttp_send_reply(req_copy, nStatus, nullptr, nullptr); + // Re-enable reading from the socket. This is the second part of the libevent + // workaround above. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + evhttp_connection* conn = evhttp_request_get_connection(req_copy); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_enable(bev, EV_READ | EV_WRITE); + } + } + } + }); ev->trigger(0); replySent = true; req = 0; // transferred back to main thread From 9a96c06057b0794082198c43bcb4bfbf7594e337 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 25 Sep 2019 12:25:39 +0200 Subject: [PATCH 22/26] Remove uses of NODE_WITNESS --- src/protocol.h | 2 +- src/test/DoS_tests.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/protocol.h b/src/protocol.h index b27c7d8285ab2..128f0adb19b44 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -323,7 +323,7 @@ enum ServiceFlags : uint64_t { * Thus, generally, avoid calling with peerServices == NODE_NONE. */ static ServiceFlags GetDesirableServiceFlags(ServiceFlags services) { - return ServiceFlags(NODE_NETWORK | NODE_WITNESS); + return ServiceFlags(NODE_NETWORK); } /** diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 4cdd01b242054..df9370b06dc57 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -58,7 +58,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", /*fInboundIn=*/ false); + CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK), 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", /*fInboundIn=*/ false); dummyNode1.SetSendVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode1); @@ -92,7 +92,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) void AddRandomOutboundPeer(std::vector &vNodes, PeerLogicValidation &peerLogic) { CAddress addr(ip(GetRandInt(0xffffffff)), NODE_NONE); - vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", /*fInboundIn=*/ false)); + vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", /*fInboundIn=*/ false)); CNode &node = *vNodes.back(); node.SetSendVersion(PROTOCOL_VERSION); From 438a972a728d40273a87b0b2535b74d77213468a Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 25 Sep 2019 14:44:18 +0200 Subject: [PATCH 23/26] Fix minchainwork.py --- test/functional/minchainwork.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/functional/minchainwork.py b/test/functional/minchainwork.py index 35cd7ad141807..84cd11f72274e 100755 --- a/test/functional/minchainwork.py +++ b/test/functional/minchainwork.py @@ -32,6 +32,8 @@ def set_test_params(self): self.node_min_work = [0, 101, 101] def setup_network(self): + # Force CanDirectFetch to return false (otherwise nMinimumChainWork is ignored) + self.mocktime += 21 * 2.6 * 60 # This test relies on the chain setup being: # node0 <- node1 <- node2 # Before leaving IBD, nodes prefer to download blocks from outbound From 71d39e6a423f518975a666fcde85084e2012f8d2 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 26 Sep 2019 15:36:54 +0200 Subject: [PATCH 24/26] Don't disconnect masternodes just because they were slow in block announcement --- src/net.cpp | 4 ++++ src/net_processing.cpp | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/net.cpp b/src/net.cpp index 8785b97bf7b51..5b37104b34771 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1827,6 +1827,10 @@ int CConnman::GetExtraOutboundCount() { LOCK(cs_vNodes); for (CNode* pnode : vNodes) { + // don't count outbound masternodes + if (pnode->fMasternode) { + continue; + } if (!pnode->fInbound && !pnode->m_manual_connection && !pnode->fFeeler && !pnode->fDisconnect && !pnode->fOneShot && pnode->fSuccessfullyConnected) { ++nOutbound; } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 172ea808cf0c4..39d59f358b42c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3358,6 +3358,8 @@ void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) LOCK(cs_main); connman->ForEachNode([&](CNode* pnode) { + // Don't disconnect masternodes just because they were slow in block announcement + if (pnode->fMasternode) return; // Ignore non-outbound peers, or nodes marked for disconnect already if (!IsOutboundDisconnectionCandidate(pnode) || pnode->fDisconnect) return; CNodeState *state = State(pnode->GetId()); From 60ef97cc41a1ee3a2134cce184aaf1880b1501a1 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 26 Sep 2019 20:53:22 +0200 Subject: [PATCH 25/26] Adjust STALE_CHECK_INTERVAL to be 2.5 minutes instead of 10 minutes This value was meant to mimic the block time in Bitcoin, so should be set to 2.5 minutes in Dash. --- src/net_processing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net_processing.h b/src/net_processing.h index 0afa7065a568a..bb9a76045eb01 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -31,7 +31,7 @@ static constexpr int32_t MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT = 4; static constexpr int64_t CHAIN_SYNC_TIMEOUT = 20 * 60; // 20 minutes /** How frequently to check for stale tips, in seconds */ -static constexpr int64_t STALE_CHECK_INTERVAL = 10 * 60; // 10 minutes +static constexpr int64_t STALE_CHECK_INTERVAL = 150; // 2.5 minutes (~block interval) /** How frequently to check for extra outbound peers and disconnect, in seconds */ static constexpr int64_t EXTRA_PEER_CHECK_INTERVAL = 45; /** Minimum time an outbound-peer-eviction candidate must be connected for, in order to evict, in seconds */ From 424ed32dbbbd31303c2dee3d59f07173ced4576b Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Sun, 29 Sep 2019 12:48:49 +0200 Subject: [PATCH 26/26] Few assert_raises_jsonrpc -> assert_raises_rpc_error fixes --- test/functional/llmq-is-cl-conflicts.py | 2 +- test/functional/mining.py | 2 +- test/functional/p2p-instantsend.py | 2 +- test/functional/wallet-dump.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/functional/llmq-is-cl-conflicts.py b/test/functional/llmq-is-cl-conflicts.py index fcade3de242f8..9853704a4bdeb 100755 --- a/test/functional/llmq-is-cl-conflicts.py +++ b/test/functional/llmq-is-cl-conflicts.py @@ -9,7 +9,7 @@ from test_framework.blocktools import get_masternode_payment, create_coinbase, create_block from test_framework.mininode import * from test_framework.test_framework import DashTestFramework -from test_framework.util import sync_blocks, p2p_port, assert_raises_jsonrpc, set_node_times +from test_framework.util import sync_blocks, p2p_port, assert_raises_rpc_error, set_node_times ''' llmq-is-cl-conflicts.py diff --git a/test/functional/mining.py b/test/functional/mining.py index 239f964e8eff0..fbf478ce831d4 100755 --- a/test/functional/mining.py +++ b/test/functional/mining.py @@ -15,7 +15,7 @@ from test_framework.blocktools import create_coinbase from test_framework.mininode import CBlock from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_jsonrpc +from test_framework.util import assert_equal, assert_raises_rpc_error def b2x(b): return b2a_hex(b).decode('ascii') diff --git a/test/functional/p2p-instantsend.py b/test/functional/p2p-instantsend.py index b00452e849ded..6b83ee4a4277d 100755 --- a/test/functional/p2p-instantsend.py +++ b/test/functional/p2p-instantsend.py @@ -6,7 +6,7 @@ from test_framework.mininode import * from test_framework.test_framework import DashTestFramework from test_framework.util import isolate_node, set_node_times, reconnect_isolated_node, assert_equal, \ - assert_raises_jsonrpc + assert_raises_rpc_error ''' InstantSendTest -- test InstantSend functionality (prevent doublespend for unconfirmed transactions) diff --git a/test/functional/wallet-dump.py b/test/functional/wallet-dump.py index 0407bcca38a80..cc0d8649fb686 100755 --- a/test/functional/wallet-dump.py +++ b/test/functional/wallet-dump.py @@ -7,7 +7,7 @@ import sys from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (assert_equal, assert_raises_jsonrpc) +from test_framework.util import (assert_equal, assert_raises_rpc_error) def read_dump(file_name, addrs, hd_master_addr_old): @@ -113,7 +113,7 @@ def run_test (self): assert_equal(found_addr_rsv, 180) # keypool size # Overwriting should fail - assert_raises_jsonrpc(-8, "already exists", self.nodes[0].dumpwallet, tmpdir + "/node0/wallet.unencrypted.dump") + assert_raises_rpc_error(-8, "already exists", self.nodes[0].dumpwallet, tmpdir + "/node0/wallet.unencrypted.dump") if __name__ == '__main__': WalletDumpTest().main ()