test: Bump MAX_NODES to 12

Summary:
test: Speed up cache creation

This is a backport of Core [[bitcoin/bitcoin#16042 | PR16042]]

Depends on D5984

Test Plan:
  ninja all check-all

Reviewers: #bitcoin_abc, Fabien

Reviewed By: #bitcoin_abc, Fabien

Subscribers: Fabien

Differential Revision: https://reviews.bitcoinabc.org/D5985

test/functional/test_framework/test_framework.py

@@ -446,57 +446,47 @@ def _start_logging(self):
     def _initialize_chain(self):
         """Initialize a pre-mined blockchain for use by the test.
 
-        Create a cache of a 199-block-long chain (with wallet) for MAX_NODES
+        Create a cache of a 199-block-long chain
         Afterward, create num_nodes copies from the cache."""
 
+        # Use node 0 to create the cache for all other nodes
+        CACHE_NODE_ID = 0
+        cache_node_dir = get_datadir_path(self.options.cachedir, CACHE_NODE_ID)
         assert self.num_nodes <= MAX_NODES
-        create_cache = False
-        for i in range(MAX_NODES):
-            if not os.path.isdir(get_datadir_path(self.options.cachedir, i)):
-                create_cache = True
-                break
-
-        if create_cache:
-            self.log.debug("Creating data directories from cached datadir")
-
-            # find and delete old cache directories if any exist
-            for i in range(MAX_NODES):
-                if os.path.isdir(get_datadir_path(self.options.cachedir, i)):
-                    shutil.rmtree(get_datadir_path(self.options.cachedir, i))
-
-            # Create cache directories, run bitcoinds:
-            for i in range(MAX_NODES):
-                datadir = initialize_datadir(
-                    self.options.cachedir, i, self.chain)
-                self.nodes.append(TestNode(
-                    i,
-                    get_datadir_path(self.options.cachedir, i),
+
+        if not os.path.isdir(cache_node_dir):
+            self.log.debug(
+                "Creating cache directory {}".format(cache_node_dir))
+
+            initialize_datadir(
+                self.options.cachedir,
+                CACHE_NODE_ID,
+                self.chain)
+            self.nodes.append(
+                TestNode(
+                    CACHE_NODE_ID,
+                    cache_node_dir,
                     chain=self.chain,
                     extra_conf=["bind=127.0.0.1"],
-                    extra_args=[],
+                    extra_args=['-disablewallet'],
                     host=None,
-                    rpc_port=rpc_port(i),
-                    p2p_port=p2p_port(i),
+                    rpc_port=rpc_port(CACHE_NODE_ID),
+                    p2p_port=p2p_port(CACHE_NODE_ID),
                     timewait=self.rpc_timeout,
                     bitcoind=self.options.bitcoind,
                     bitcoin_cli=self.options.bitcoincli,
                     coverage_dir=None,
                     emulator=self.options.emulator,
                 ))
-                self.nodes[i].clear_default_args()
-                self.nodes[i].extend_default_args(["-datadir=" + datadir])
-                self.nodes[i].extend_default_args(["-disablewallet"])
-                if i > 0:
-                    self.nodes[i].extend_default_args(
-                        ["-connect=127.0.0.1:" + str(p2p_port(0))])
-                if self.options.phononactivation:
-                    self.nodes[i].extend_default_args(
-                        ["-phononactivationtime={}".format(TIMESTAMP_IN_THE_PAST)])
-                self.start_node(i)
+
+            if self.options.phononactivation:
+                self.nodes[CACHE_NODE_ID].extend_default_args(
+                    ["-phononactivationtime={}".format(TIMESTAMP_IN_THE_PAST)])
+
+            self.start_node(CACHE_NODE_ID)
 
             # Wait for RPC connections to be ready
-            for node in self.nodes:
-                node.wait_for_rpc_connection()
+            self.nodes[CACHE_NODE_ID].wait_for_rpc_connection()
 
             # Create a 199-block-long chain; each of the 4 first nodes
             # gets 25 mature blocks and 25 immature.
@@ -505,32 +495,34 @@ def _initialize_chain(self):
             # This is needed so that we are out of IBD when the test starts,
             # see the tip age check in IsInitialBlockDownload().
             for i in range(8):
-                self.nodes[0].generatetoaddress(
-                    25 if i != 7 else 24, self.nodes[i % 4].get_deterministic_priv_key().address)
-            sync_blocks(self.nodes)
+                self.nodes[CACHE_NODE_ID].generatetoaddress(
+                    nblocks=25 if i != 7 else 24,
+                    address=TestNode.PRIV_KEYS[i % 4].address,
+                )
 
-            for n in self.nodes:
-                assert_equal(n.getblockchaininfo()["blocks"], 199)
+            assert_equal(
+                self.nodes[CACHE_NODE_ID].getblockchaininfo()["blocks"], 199)
 
-            # Shut them down, and clean up cache directories:
+            # Shut it down, and clean up cache directories:
             self.stop_nodes()
             self.nodes = []
 
-            def cache_path(n, *paths):
-                return os.path.join(get_datadir_path(
-                    self.options.cachedir, n), self.chain, *paths)
+            def cache_path(*paths):
+                return os.path.join(cache_node_dir, "regtest", *paths)
 
-            for i in range(MAX_NODES):
-                # Remove empty wallets dir
-                os.rmdir(cache_path(i, 'wallets'))
-                for entry in os.listdir(cache_path(i)):
-                    if entry not in ['chainstate', 'blocks']:
-                        os.remove(cache_path(i, entry))
+            # Remove empty wallets dir
+            os.rmdir(cache_path('wallets'))
+            for entry in os.listdir(cache_path()):
+                # Only keep chainstate and blocks folder
+                if entry not in ['chainstate', 'blocks']:
+                    os.remove(cache_path(entry))
 
         for i in range(self.num_nodes):
-            from_dir = get_datadir_path(self.options.cachedir, i)
+            self.log.debug(
+                "Copy cache directory {} to node {}".format(
+                    cache_node_dir, i))
             to_dir = get_datadir_path(self.options.tmpdir, i)
-            shutil.copytree(from_dir, to_dir)
+            shutil.copytree(cache_node_dir, to_dir)
             # Overwrite port/rpcport in bitcoin.conf
             initialize_datadir(self.options.tmpdir, i, self.chain)
 

test/functional/test_framework/test_node.py

@@ -23,6 +23,7 @@
 from .authproxy import JSONRPCException
 from .messages import COIN, CTransaction, FromHex
 from .util import (
+    MAX_NODES,
     append_config,
     delete_cookie_file,
     get_rpc_proxy,
@@ -117,32 +118,52 @@ def __init__(self, i, datadir, *, chain, host, rpc_port, p2p_port, timewait, bit
         self.cleanup_on_exit = True
         self.p2ps = []
 
+    AddressKeyPair = collections.namedtuple(
+        'AddressKeyPair', ['address', 'key'])
+    PRIV_KEYS = [
+        # address , privkey
+        AddressKeyPair(
+            'mjTkW3DjgyZck4KbiRusZsqTgaYTxdSz6z',
+            'cVpF924EspNh8KjYsfhgY96mmxvT6DgdWiTYMtMjuM74hJaU5psW'),
+        AddressKeyPair(
+            'msX6jQXvxiNhx3Q62PKeLPrhrqZQdSimTg',
+            'cUxsWyKyZ9MAQTaAhUQWJmBbSvHMwSmuv59KgxQV7oZQU3PXN3KE'),
+        AddressKeyPair(
+            'mnonCMyH9TmAsSj3M59DsbH8H63U3RKoFP',
+            'cTrh7dkEAeJd6b3MRX9bZK8eRmNqVCMH3LSUkE3dSFDyzjU38QxK'),
+        AddressKeyPair(
+            'mqJupas8Dt2uestQDvV2NH3RU8uZh2dqQR',
+            'cVuKKa7gbehEQvVq717hYcbE9Dqmq7KEBKqWgWrYBa2CKKrhtRim'),
+        AddressKeyPair(
+            'msYac7Rvd5ywm6pEmkjyxhbCDKqWsVeYws',
+            'cQDCBuKcjanpXDpCqacNSjYfxeQj8G6CAtH1Dsk3cXyqLNC4RPuh'),
+        AddressKeyPair(
+            'n2rnuUnwLgXqf9kk2kjvVm8R5BZK1yxQBi',
+            'cQakmfPSLSqKHyMFGwAqKHgWUiofJCagVGhiB4KCainaeCSxeyYq'),
+        AddressKeyPair(
+            'myzuPxRwsf3vvGzEuzPfK9Nf2RfwauwYe6',
+            'cQMpDLJwA8DBe9NcQbdoSb1BhmFxVjWD5gRyrLZCtpuF9Zi3a9RK'),
+        AddressKeyPair(
+            'mumwTaMtbxEPUswmLBBN3vM9oGRtGBrys8',
+            'cSXmRKXVcoouhNNVpcNKFfxsTsToY5pvB9DVsFksF1ENunTzRKsy'),
+        AddressKeyPair(
+            'mpV7aGShMkJCZgbW7F6iZgrvuPHjZjH9qg',
+            'cSoXt6tm3pqy43UMabY6eUTmR3eSUYFtB2iNQDGgb3VUnRsQys2k'),
+        AddressKeyPair(
+            'mq4fBNdckGtvY2mijd9am7DRsbRB4KjUkf',
+            'cN55daf1HotwBAgAKWVgDcoppmUNDtQSfb7XLutTLeAgVc3u8hik'),
+        AddressKeyPair(
+            'mpFAHDjX7KregM3rVotdXzQmkbwtbQEnZ6',
+            'cT7qK7g1wkYEMvKowd2ZrX1E5f6JQ7TM246UfqbCiyF7kZhorpX3'),
+        AddressKeyPair(
+            'mzRe8QZMfGi58KyWCse2exxEFry2sfF2Y7',
+            'cPiRWE8KMjTRxH1MWkPerhfoHFn5iHPWVK5aPqjW8NxmdwenFinJ'),
+    ]
+
     def get_deterministic_priv_key(self):
         """Return a deterministic priv key in base58, that only depends on the node's index"""
-        AddressKeyPair = collections.namedtuple(
-            'AddressKeyPair', ['address', 'key'])
-        PRIV_KEYS = [
-            # address , privkey
-            AddressKeyPair('mjTkW3DjgyZck4KbiRusZsqTgaYTxdSz6z',
-                           'cVpF924EspNh8KjYsfhgY96mmxvT6DgdWiTYMtMjuM74hJaU5psW'),
-            AddressKeyPair('msX6jQXvxiNhx3Q62PKeLPrhrqZQdSimTg',
-                           'cUxsWyKyZ9MAQTaAhUQWJmBbSvHMwSmuv59KgxQV7oZQU3PXN3KE'),
-            AddressKeyPair('mnonCMyH9TmAsSj3M59DsbH8H63U3RKoFP',
-                           'cTrh7dkEAeJd6b3MRX9bZK8eRmNqVCMH3LSUkE3dSFDyzjU38QxK'),
-            AddressKeyPair('mqJupas8Dt2uestQDvV2NH3RU8uZh2dqQR',
-                           'cVuKKa7gbehEQvVq717hYcbE9Dqmq7KEBKqWgWrYBa2CKKrhtRim'),
-            AddressKeyPair('msYac7Rvd5ywm6pEmkjyxhbCDKqWsVeYws',
-                           'cQDCBuKcjanpXDpCqacNSjYfxeQj8G6CAtH1Dsk3cXyqLNC4RPuh'),
-            AddressKeyPair('n2rnuUnwLgXqf9kk2kjvVm8R5BZK1yxQBi',
-                           'cQakmfPSLSqKHyMFGwAqKHgWUiofJCagVGhiB4KCainaeCSxeyYq'),
-            AddressKeyPair('myzuPxRwsf3vvGzEuzPfK9Nf2RfwauwYe6',
-                           'cQMpDLJwA8DBe9NcQbdoSb1BhmFxVjWD5gRyrLZCtpuF9Zi3a9RK'),
-            AddressKeyPair('mumwTaMtbxEPUswmLBBN3vM9oGRtGBrys8',
-                           'cSXmRKXVcoouhNNVpcNKFfxsTsToY5pvB9DVsFksF1ENunTzRKsy'),
-            AddressKeyPair('mpV7aGShMkJCZgbW7F6iZgrvuPHjZjH9qg',
-                           'cSoXt6tm3pqy43UMabY6eUTmR3eSUYFtB2iNQDGgb3VUnRsQys2k'),
-        ]
-        return PRIV_KEYS[self.index]
+        assert len(self.PRIV_KEYS) == MAX_NODES
+        return self.PRIV_KEYS[self.index]
 
     def _node_msg(self, msg: str) -> str:
         """Return a modified msg that identifies this node by its index as a debugging aid."""

test/functional/test_framework/util.py

@@ -271,7 +271,7 @@ def wait_until(predicate, *, attempts=float('inf'),
 
 
 # The maximum number of nodes a single test can spawn
-MAX_NODES = 8
+MAX_NODES = 12
 # Don't assign rpc or p2p ports lower than this
 PORT_MIN = 11000
 # The number of ports to "reserve" for p2p and rpc, each