Merge d3582f2 into merged_master (Bitcoin PR bitcoin/bitcoin#23866)

I had to make some python changes for the rpc_scantxoutset.py which are
pretty hacky but get the job done.

test/functional/p2p_filter.py

@@ -31,7 +31,7 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.wallet import (
     MiniWallet,
-    random_p2wpkh,
+    getnewdestination,
 )
 
 
@@ -169,14 +169,14 @@ def test_filter(self, filter_peer):
 
         self.log.info('Check that we only receive a merkleblock if the filter does not match a tx in a block')
         filter_peer.tx_received = False
-        block_hash = self.generatetoscriptpubkey(random_p2wpkh())
+        block_hash = self.generatetoscriptpubkey(getnewdestination()[1])
         filter_peer.wait_for_merkleblock(block_hash)
         assert not filter_peer.tx_received
 
         self.log.info('Check that we not receive a tx if the filter does not match a mempool tx')
         filter_peer.merkleblock_received = False
         filter_peer.tx_received = False
-        self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=random_p2wpkh(), amount=7 * COIN)
+        self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=7 * COIN)
         filter_peer.sync_send_with_ping()
         assert not filter_peer.merkleblock_received
         assert not filter_peer.tx_received
@@ -190,14 +190,14 @@ def test_filter(self, filter_peer):
         self.log.info('Check that after deleting filter all txs get relayed again')
         filter_peer.send_and_ping(msg_filterclear())
         for _ in range(5):
-            txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=random_p2wpkh(), amount=7 * COIN)
+            txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=7 * COIN)
             filter_peer.wait_for_tx(txid)
 
         self.log.info('Check that request for filtered blocks is ignored if no filter is set')
         filter_peer.merkleblock_received = False
         filter_peer.tx_received = False
         with self.nodes[0].assert_debug_log(expected_msgs=['received getdata']):
-            block_hash = self.generatetoscriptpubkey(random_p2wpkh())
+            block_hash = self.generatetoscriptpubkey(getnewdestination()[1])
             filter_peer.wait_for_inv([CInv(MSG_BLOCK, int(block_hash, 16))])
             filter_peer.sync_with_ping()
             assert not filter_peer.merkleblock_received

test/functional/rpc_scantxoutset.py

@@ -3,12 +3,16 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the scantxoutset rpc call."""
+from test_framework.messages import COIN
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, assert_raises_rpc_error
+from test_framework.wallet import (
+    MiniWallet,
+    address_to_scriptpubkey,
+    getnewdestination,
+)
 
 from decimal import Decimal
-import shutil
-import os
 
 
 def descriptors(out):
@@ -22,66 +26,57 @@ def set_test_params(self):
         # ELEMENTS: use bitcoin regtest prefixes to avoid having to rewrite the entire test
         self.extra_args = [["-pubkeyprefix=111", "-scriptprefix=196", "-secretprefix=239", "-extpubkeyprefix=043587CF", "-extprvkeyprefix=04358394", "-bech32_hrp=bcrt"]]
 
-    def skip_test_if_missing_module(self):
-        self.skip_if_no_wallet()
+    def sendtodestination(self, destination, amount):
+        # interpret strings as addresses, assume scriptPubKey otherwise
+        if isinstance(destination, str):
+            destination = address_to_scriptpubkey(destination)
+        self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=destination, amount=int(COIN * amount))
 
     def run_test(self):
-        self.log.info("Mining blocks...")
+        # ELEMENTS hacks
+        self.wallet = MiniWallet(self.nodes[0], hrp="bcrt")
         self.nodes[0].set_deterministic_priv_key('2Mysp7FKKe52eoC2JmU46irt1dt58TpCvhQ', 'cTNbtVJmhx75RXomhYWSZAafuNNNKPd1cr2ZiUcAeukLNGrHWjvJ')
-        self.nodes[0].importprivkey("cTNbtVJmhx75RXomhYWSZAafuNNNKPd1cr2ZiUcAeukLNGrHWjvJ")
-        self.generate(self.nodes[0], 110)
-
-        addr_P2SH_SEGWIT = self.nodes[0].getnewaddress("", "p2sh-segwit")
-        pubk1 = self.nodes[0].getaddressinfo(addr_P2SH_SEGWIT)['pubkey']
-        addr_LEGACY = self.nodes[0].getnewaddress("", "legacy")
-        pubk2 = self.nodes[0].getaddressinfo(addr_LEGACY)['pubkey']
-        addr_BECH32 = self.nodes[0].getnewaddress("", "bech32")
-        pubk3 = self.nodes[0].getaddressinfo(addr_BECH32)['pubkey']
-        txid = self.nodes[0].sendtoaddress(addr_P2SH_SEGWIT, 0.001)
-        self.nodes[0].lockunspent(unlock=False, transactions=[{"txid": txid, "vout": 0}, {"txid": txid, "vout": 1}])
-        txid = self.nodes[0].sendtoaddress(addr_LEGACY, 0.002)
-        self.nodes[0].lockunspent(unlock=False, transactions=[{"txid": txid, "vout": 0}, {"txid": txid, "vout": 1}])
-        txid = self.nodes[0].sendtoaddress(addr_BECH32, 0.004)
-        self.nodes[0].lockunspent(unlock=False, transactions=[{"txid": txid, "vout": 0}, {"txid": txid, "vout": 1}])
+        self.wallet.generate(200, invalid_call=False)
+        self.wallet.rescan_utxos()
+
+        self.log.info("Create UTXOs...")
+        pubk1, spk_P2SH_SEGWIT, addr_P2SH_SEGWIT = getnewdestination("p2sh-segwit", prefix=196)
+        pubk2, spk_LEGACY, addr_LEGACY = getnewdestination("legacy", version=111)
+        pubk3, spk_BECH32, addr_BECH32 = getnewdestination("bech32", hrp="bcrt")
+        self.sendtodestination(spk_P2SH_SEGWIT, 0.001)
+        self.sendtodestination(spk_LEGACY, 0.002)
+        self.sendtodestination(spk_BECH32, 0.004)
 
         #send to child keys of tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK
-        self.nodes[0].sendtoaddress("mkHV1C6JLheLoUSSZYk7x3FH5tnx9bu7yc", 0.008)  # (m/0'/0'/0')
-        self.nodes[0].sendtoaddress("mipUSRmJAj2KrjSvsPQtnP8ynUon7FhpCR", 0.016)  # (m/0'/0'/1')
-        self.nodes[0].sendtoaddress("n37dAGe6Mq1HGM9t4b6rFEEsDGq7Fcgfqg", 0.032)  # (m/0'/0'/1500')
-        self.nodes[0].sendtoaddress("mqS9Rpg8nNLAzxFExsgFLCnzHBsoQ3PRM6", 0.064)  # (m/0'/0'/0)
-        self.nodes[0].sendtoaddress("mnTg5gVWr3rbhHaKjJv7EEEc76ZqHgSj4S", 0.128)  # (m/0'/0'/1)
-        self.nodes[0].sendtoaddress("mketCd6B9U9Uee1iCsppDJJBHfvi6U6ukC", 0.256)  # (m/0'/0'/1500)
-        self.nodes[0].sendtoaddress("mj8zFzrbBcdaWXowCQ1oPZ4qioBVzLzAp7", 0.512)  # (m/1/1/0')
-        self.nodes[0].sendtoaddress("mfnKpKQEftniaoE1iXuMMePQU3PUpcNisA", 1.024)  # (m/1/1/1')
-        self.nodes[0].sendtoaddress("mou6cB1kaP1nNJM1sryW6YRwnd4shTbXYQ", 2.048)  # (m/1/1/1500')
-        self.nodes[0].sendtoaddress("mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", 4.096)  # (m/1/1/0)
-        self.nodes[0].sendtoaddress("mxp7w7j8S1Aq6L8StS2PqVvtt4HGxXEvdy", 8.192)  # (m/1/1/1)
-        self.nodes[0].sendtoaddress("mpQ8rokAhp1TAtJQR6F6TaUmjAWkAWYYBq", 16.384)  # (m/1/1/1500)
+        self.sendtodestination("mkHV1C6JLheLoUSSZYk7x3FH5tnx9bu7yc", 0.008)  # (m/0'/0'/0')
+        self.sendtodestination("mipUSRmJAj2KrjSvsPQtnP8ynUon7FhpCR", 0.016)  # (m/0'/0'/1')
+        self.sendtodestination("n37dAGe6Mq1HGM9t4b6rFEEsDGq7Fcgfqg", 0.032)  # (m/0'/0'/1500')
+        self.sendtodestination("mqS9Rpg8nNLAzxFExsgFLCnzHBsoQ3PRM6", 0.064)  # (m/0'/0'/0)
+        self.sendtodestination("mnTg5gVWr3rbhHaKjJv7EEEc76ZqHgSj4S", 0.128)  # (m/0'/0'/1)
+        self.sendtodestination("mketCd6B9U9Uee1iCsppDJJBHfvi6U6ukC", 0.256)  # (m/0'/0'/1500)
+        self.sendtodestination("mj8zFzrbBcdaWXowCQ1oPZ4qioBVzLzAp7", 0.512)  # (m/1/1/0')
+        self.sendtodestination("mfnKpKQEftniaoE1iXuMMePQU3PUpcNisA", 1.024)  # (m/1/1/1')
+        self.sendtodestination("mou6cB1kaP1nNJM1sryW6YRwnd4shTbXYQ", 2.048)  # (m/1/1/1500')
+        self.sendtodestination("mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", 4.096)  # (m/1/1/0)
+        self.sendtodestination("mxp7w7j8S1Aq6L8StS2PqVvtt4HGxXEvdy", 8.192)  # (m/1/1/1)
+        self.sendtodestination("mpQ8rokAhp1TAtJQR6F6TaUmjAWkAWYYBq", 16.384)  # (m/1/1/1500)
 
         self.generate(self.nodes[0], 1)
 
-        self.log.info("Stop node, remove wallet, mine again some blocks...")
-        self.stop_node(0)
-        shutil.rmtree(os.path.join(self.nodes[0].datadir, self.chain, 'wallets'))
-        self.start_node(0, self.extra_args[0] + ['-nowallet'])
-        self.import_deterministic_coinbase_privkeys()
-        self.generate(self.nodes[0], 110)
-
         scan = self.nodes[0].scantxoutset("start", [])
         info = self.nodes[0].gettxoutsetinfo()
         assert_equal(scan['success'], True)
         assert_equal(scan['height'], info['height'])
         assert_equal(scan['txouts'], info['txouts'])
         assert_equal(scan['bestblock'], info['bestblock'])
 
-        self.restart_node(0, self.extra_args[0] + ['-nowallet'])
         self.log.info("Test if we have found the non HD unspent outputs.")
-        assert_equal(self.nodes[0].scantxoutset("start", ["pkh(" + pubk1 + ")", "pkh(" + pubk2 + ")", "pkh(" + pubk3 + ")"])['total_unblinded_bitcoin_amount'], Decimal("0.002"))
-        assert_equal(self.nodes[0].scantxoutset("start", ["wpkh(" + pubk1 + ")", "wpkh(" + pubk2 + ")", "wpkh(" + pubk3 + ")"])['total_unblinded_bitcoin_amount'], Decimal("0.004"))
-        assert_equal(self.nodes[0].scantxoutset("start", ["sh(wpkh(" + pubk1 + "))", "sh(wpkh(" + pubk2 + "))", "sh(wpkh(" + pubk3 + "))"])['total_unblinded_bitcoin_amount'], Decimal("0.001"))
-        assert_equal(self.nodes[0].scantxoutset("start", ["combo(" + pubk1 + ")", "combo(" + pubk2 + ")", "combo(" + pubk3 + ")"])['total_unblinded_bitcoin_amount'], Decimal("0.007"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["pkh(" + pubk1.hex() + ")", "pkh(" + pubk2.hex() + ")", "pkh(" + pubk3.hex() + ")"])['total_unblinded_bitcoin_amount'], Decimal("0.002"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["wpkh(" + pubk1.hex() + ")", "wpkh(" + pubk2.hex() + ")", "wpkh(" + pubk3.hex() + ")"])['total_unblinded_bitcoin_amount'], Decimal("0.004"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["sh(wpkh(" + pubk1.hex() + "))", "sh(wpkh(" + pubk2.hex() + "))", "sh(wpkh(" + pubk3.hex() + "))"])['total_unblinded_bitcoin_amount'], Decimal("0.001"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["combo(" + pubk1.hex() + ")", "combo(" + pubk2.hex() + ")", "combo(" + pubk3.hex() + ")"])['total_unblinded_bitcoin_amount'], Decimal("0.007"))
         assert_equal(self.nodes[0].scantxoutset("start", ["addr(" + addr_P2SH_SEGWIT + ")", "addr(" + addr_LEGACY + ")", "addr(" + addr_BECH32 + ")"])['total_unblinded_bitcoin_amount'], Decimal("0.007"))
-        assert_equal(self.nodes[0].scantxoutset("start", ["addr(" + addr_P2SH_SEGWIT + ")", "addr(" + addr_LEGACY + ")", "combo(" + pubk3 + ")"])['total_unblinded_bitcoin_amount'], Decimal("0.007"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["addr(" + addr_P2SH_SEGWIT + ")", "addr(" + addr_LEGACY + ")", "combo(" + pubk3.hex() + ")"])['total_unblinded_bitcoin_amount'], Decimal("0.007"))
 
         self.log.info("Test range validation.")
         assert_raises_rpc_error(-8, "End of range is too high", self.nodes[0].scantxoutset, "start", [{"desc": "desc", "range": -1}])

test/functional/test_framework/address.py

@@ -38,7 +38,7 @@ class AddressType(enum.Enum):
 chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
 
 
-def create_deterministic_address_bcrt1_p2tr_op_true():
+def create_deterministic_address_bcrt1_p2tr_op_true(hrp="ert"):
     """
     Generates a deterministic bech32m address (segwit v1 output) that
     can be spent with a witness stack of OP_TRUE and the control block
@@ -48,8 +48,11 @@ def create_deterministic_address_bcrt1_p2tr_op_true():
     """
     internal_key = (2).to_bytes(32, 'big') # ELEMENTS: the given internal key from upstream failed to verify
     scriptPubKey = taproot_construct(internal_key, [(None, CScript([OP_TRUE]))]).scriptPubKey
-    address = encode_segwit_address("ert", 1, scriptPubKey[2:])
-    assert_equal(address, 'ert1pxaxh5xm2p349fg5wqstrreat4atm00ktumm6q4vfu960ls09265sf37hcj')
+    address = encode_segwit_address(hrp, 1, scriptPubKey[2:])
+    if hrp == "ert":
+        assert_equal(address, 'ert1pxaxh5xm2p349fg5wqstrreat4atm00ktumm6q4vfu960ls09265sf37hcj')
+    if hrp == "bcrt":
+        assert_equal(address, 'bcrt1pxaxh5xm2p349fg5wqstrreat4atm00ktumm6q4vfu960ls09265sczkf3k')
     return (address, internal_key)
 
 
@@ -99,44 +102,43 @@ def base58_to_byte(s):
     return res[1:-4], int(res[0])
 
 
-def keyhash_to_p2pkh(hash, main=False):
+def keyhash_to_p2pkh(hash, main=False, version=235):
     assert len(hash) == 20
-    version = 235
     return byte_to_base58(hash, version)
 
 def scripthash_to_p2sh(hash, main=False, prefix=75):
     assert len(hash) == 20
     version = prefix
     return byte_to_base58(hash, version)
 
-def key_to_p2pkh(key, main=False):
+def key_to_p2pkh(key, main=False, **kwargs):
     key = check_key(key)
-    return keyhash_to_p2pkh(hash160(key), main)
+    return keyhash_to_p2pkh(hash160(key), main, **kwargs)
 
 def script_to_p2sh(script, main=False, prefix=75):
     script = check_script(script)
     return scripthash_to_p2sh(hash160(script), main, prefix)
 
-def key_to_p2sh_p2wpkh(key, main=False):
+def key_to_p2sh_p2wpkh(key, main=False, **kwargs):
     key = check_key(key)
     p2shscript = CScript([OP_0, hash160(key)])
-    return script_to_p2sh(p2shscript, main)
+    return script_to_p2sh(p2shscript, main, **kwargs)
 
-def program_to_witness(version, program, main=False):
+def program_to_witness(version, program, main=False, hrp="ert"):
     if (type(program) is str):
         program = bytes.fromhex(program)
     assert 0 <= version <= 16
     assert 2 <= len(program) <= 40
     assert version > 0 or len(program) in [20, 32]
-    return encode_segwit_address("ert", version, program)
+    return encode_segwit_address(hrp, version, program)
 
 def script_to_p2wsh(script, main=False):
     script = check_script(script)
     return program_to_witness(0, sha256(script), main)
 
-def key_to_p2wpkh(key, main=False):
+def key_to_p2wpkh(key, main=False, **kwargs):
     key = check_key(key)
-    return program_to_witness(0, hash160(key), main)
+    return program_to_witness(0, hash160(key), main, **kwargs)
 
 def script_to_p2sh_p2wsh(script, main=False):
     script = check_script(script)

test/functional/test_framework/wallet.py

@@ -9,7 +9,13 @@
 from enum import Enum
 from random import choice
 from typing import Optional
-from test_framework.address import create_deterministic_address_bcrt1_p2tr_op_true
+from test_framework.address import (
+    base58_to_byte,
+    create_deterministic_address_bcrt1_p2tr_op_true,
+    key_to_p2pkh,
+    key_to_p2sh_p2wpkh,
+    key_to_p2wpkh,
+)
 from test_framework.descriptors import descsum_create
 from test_framework.key import ECKey
 from test_framework.messages import (
@@ -31,7 +37,11 @@
 )
 from test_framework.script_util import (
     key_to_p2pk_script,
+    key_to_p2pkh_script,
+    key_to_p2sh_p2wpkh_script,
     key_to_p2wpkh_script,
+    keyhash_to_p2pkh_script,
+    scripthash_to_p2sh_script,
 )
 from test_framework.util import (
     assert_equal,
@@ -64,7 +74,7 @@ class MiniWalletMode(Enum):
 
 
 class MiniWallet:
-    def __init__(self, test_node, *, mode=MiniWalletMode.ADDRESS_OP_TRUE):
+    def __init__(self, test_node, *, mode=MiniWalletMode.ADDRESS_OP_TRUE, hrp="ert"):
         self._test_node = test_node
         self._utxos = []
         self._priv_key = None
@@ -80,7 +90,7 @@ def __init__(self, test_node, *, mode=MiniWalletMode.ADDRESS_OP_TRUE):
             pub_key = self._priv_key.get_pubkey()
             self._scriptPubKey = key_to_p2pk_script(pub_key.get_bytes())
         elif mode == MiniWalletMode.ADDRESS_OP_TRUE:
-            self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true()
+            self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true(hrp=hrp)
             self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey'])
 
     def rescan_utxos(self):
@@ -211,12 +221,40 @@ def sendrawtransaction(self, *, from_node, tx_hex):
         return txid
 
 
-def random_p2wpkh():
-    """Generate a random P2WPKH scriptPubKey. Can be used when a random destination is needed,
-    but no compiled wallet is available (e.g. as replacement to the getnewaddress RPC)."""
+def getnewdestination(address_type='bech32', hrp='ert', version=235, prefix=75):
+    """Generate a random destination of the specified type and return the
+       corresponding public key, scriptPubKey and address. Supported types are
+       'legacy', 'p2sh-segwit' and 'bech32'. Can be used when a random
+       destination is needed, but no compiled wallet is available (e.g. as
+       replacement to the getnewaddress/getaddressinfo RPCs)."""
     key = ECKey()
     key.generate()
-    return key_to_p2wpkh_script(key.get_pubkey().get_bytes())
+    pubkey = key.get_pubkey().get_bytes()
+    if address_type == 'legacy':
+        scriptpubkey = key_to_p2pkh_script(pubkey)
+        address = key_to_p2pkh(pubkey, version=version)
+    elif address_type == 'p2sh-segwit':
+        scriptpubkey = key_to_p2sh_p2wpkh_script(pubkey)
+        address = key_to_p2sh_p2wpkh(pubkey, prefix=prefix)
+    elif address_type == 'bech32':
+        scriptpubkey = key_to_p2wpkh_script(pubkey)
+        address = key_to_p2wpkh(pubkey, hrp=hrp)
+    # TODO: also support bech32m (need to generate x-only-pubkey)
+    else:
+        assert False
+    return pubkey, scriptpubkey, address
+
+
+def address_to_scriptpubkey(address):
+    """Converts a given address to the corresponding output script (scriptPubKey)."""
+    payload, version = base58_to_byte(address)
+    if version == 111:  # testnet pubkey hash
+        return keyhash_to_p2pkh_script(payload)
+    elif version == 196:  # testnet script hash
+        return scripthash_to_p2sh_script(payload)
+    # TODO: also support other address formats
+    else:
+        assert False
 
 
 def make_chain(node, address, privkeys, parent_txid, parent_value, n=0, parent_locking_script=None, fee=DEFAULT_FEE):