WIP: rollback testing

cardano_node_tests/split_topology.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Generate topology files for split network.
+
+For settings it uses the same env variables as when running the tests.
+"""
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+from cardano_node_tests.utils import cluster_nodes
+
+LOGGER = logging.getLogger(__name__)
+
+
+def get_args() -> argparse.Namespace:
+    """Get command line arguments."""
+    parser = argparse.ArgumentParser(description=__doc__.split("\n", maxsplit=1)[0])
+    parser.add_argument(
+        "-d",
+        "--dest-dir",
+        required=True,
+        help="Path to destination directory",
+    )
+    parser.add_argument(
+        "-i",
+        "--instance-num",
+        required=False,
+        type=int,
+        default=0,
+        help="Instance number in the sequence of cluster instances (default: 0)",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    logging.basicConfig(
+        format="%(name)s:%(levelname)s:%(message)s",
+        level=logging.INFO,
+    )
+    args = get_args()
+
+    destdir = Path(args.dest_dir)
+    destdir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        cluster_nodes.get_cluster_type().cluster_scripts.gen_split_topology_files(
+            destdir=destdir,
+            instance_num=args.instance_num,
+        )
+    except Exception as exc:
+        LOGGER.error(str(exc))
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cardano_node_tests/tests/test_rollback.py

@@ -0,0 +1,278 @@
+"""Tests for rollback."""
+import logging
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import List
+from typing import Optional
+
+import allure
+import pytest
+from cardano_clusterlib import clusterlib
+
+from cardano_node_tests.cluster_management import cluster_management
+from cardano_node_tests.tests import common
+from cardano_node_tests.utils import cluster_nodes
+from cardano_node_tests.utils import clusterlib_utils
+from cardano_node_tests.utils import configuration
+from cardano_node_tests.utils import helpers
+
+LOGGER = logging.getLogger(__name__)
+
+
+@pytest.mark.skipif(
+    cluster_nodes.get_cluster_type().type != cluster_nodes.ClusterType.LOCAL,
+    reason="runs only on local cluster",
+)
+@pytest.mark.skipif(configuration.NUM_POOLS % 2 != 0, reason="`NUM_POOLS` must be even")
+@pytest.mark.skipif(configuration.NUM_POOLS < 4, reason="`NUM_POOLS` must be at least 4")
+@pytest.mark.skipif(
+    configuration.MIXED_P2P, reason="Works only when all nodes have the same topology type"
+)
+class TestRollback:
+    """Test rollback."""
+
+    @pytest.fixture
+    def payment_addrs(
+        self,
+        cluster_manager: cluster_management.ClusterManager,
+        cluster_singleton: clusterlib.ClusterLib,
+    ) -> List[clusterlib.AddressRecord]:
+        """Create new payment addresses."""
+        cluster = cluster_singleton
+
+        with cluster_manager.cache_fixture() as fixture_cache:
+            if fixture_cache.value:
+                return fixture_cache.value  # type: ignore
+
+            addrs = clusterlib_utils.create_payment_addr_records(
+                *[f"addr_rollback_ci{cluster_manager.cluster_instance_num}_{i}" for i in range(3)],
+                cluster_obj=cluster,
+            )
+            fixture_cache.value = addrs
+
+        # Fund source addresses
+        clusterlib_utils.fund_from_faucet(
+            *addrs,
+            cluster_obj=cluster,
+            faucet_data=cluster_manager.cache.addrs_data["user1"],
+        )
+        return addrs
+
+    @pytest.fixture
+    def split_topology_dir(self) -> Path:
+        """Return path to directory with split topology files."""
+        instance_num = cluster_nodes.get_instance_num()
+
+        destdir = Path.cwd() / f"split_topology_ci{instance_num}"
+        if destdir.exists():
+            return destdir
+
+        destdir.mkdir()
+
+        cluster_nodes.get_cluster_type().cluster_scripts.gen_split_topology_files(
+            destdir=destdir,
+            instance_num=instance_num,
+        )
+
+        return destdir
+
+    @pytest.fixture
+    def backup_topology(self) -> Path:
+        """Backup the original topology files."""
+        state_dir = cluster_nodes.get_cluster_env().state_dir
+        topology_files = list(state_dir.glob("topology*.json"))
+
+        backup_dir = state_dir / f"backup_topology_{helpers.get_rand_str()}"
+        backup_dir.mkdir()
+
+        # Copy topology files to backup dir
+        for f in topology_files:
+            shutil.copy(f, backup_dir / f.name)
+
+        return backup_dir
+
+    def split_network(self, split_topology_dir: Path) -> None:
+        """Use the split topology files == split the network."""
+        state_dir = cluster_nodes.get_cluster_env().state_dir
+        topology_files = list(state_dir.glob("topology*.json"))
+
+        prefix = "p2p-split" if configuration.ENABLE_P2P else "split"
+
+        for f in topology_files:
+            shutil.copy(split_topology_dir / f"{prefix}-{f.name}", f)
+
+        cluster_nodes.restart_all_nodes()
+
+    def restore_network(self, backup_topology: Path) -> None:
+        """Restore the original topology files == restore the network."""
+        state_dir = cluster_nodes.get_cluster_env().state_dir
+        topology_files = list(state_dir.glob("topology*.json"))
+
+        for f in topology_files:
+            shutil.copy(backup_topology / f.name, f)
+
+        cluster_nodes.restart_all_nodes()
+
+    def node_query_utxo(
+        self,
+        cluster_obj: clusterlib.ClusterLib,
+        node: str,
+        address: str = "",
+        tx_raw_output: Optional[clusterlib.TxRawOutput] = None,
+    ) -> List[clusterlib.UTXOData]:
+        """Query UTxO on given node."""
+        orig_socket = os.environ.get("CARDANO_NODE_SOCKET_PATH")
+        assert orig_socket
+        new_socket = Path(orig_socket).parent / f"{node}.socket"
+
+        try:
+            os.environ["CARDANO_NODE_SOCKET_PATH"] = str(new_socket)
+            utxos = cluster_obj.g_query.get_utxo(address=address, tx_raw_output=tx_raw_output)
+            return utxos
+        finally:
+            os.environ["CARDANO_NODE_SOCKET_PATH"] = orig_socket
+
+    def node_submit_tx(
+        self,
+        cluster_obj: clusterlib.ClusterLib,
+        node: str,
+        temp_template: str,
+        src_addr: clusterlib.AddressRecord,
+        dst_addr: clusterlib.AddressRecord,
+    ) -> clusterlib.TxRawOutput:
+        """Submit transaction on given node."""
+        orig_socket = os.environ.get("CARDANO_NODE_SOCKET_PATH")
+        assert orig_socket
+        new_socket = Path(orig_socket).parent / f"{node}.socket"
+
+        curr_time = time.time()
+        destinations = [clusterlib.TxOut(address=dst_addr.address, amount=1_000_000)]
+        tx_files = clusterlib.TxFiles(signing_key_files=[src_addr.skey_file])
+
+        try:
+            os.environ["CARDANO_NODE_SOCKET_PATH"] = str(new_socket)
+            tx_raw_output = cluster_obj.g_transaction.send_tx(
+                src_address=src_addr.address,
+                tx_name=f"{temp_template}_{int(curr_time)}",
+                txouts=destinations,
+                tx_files=tx_files,
+            )
+            return tx_raw_output
+        finally:
+            os.environ["CARDANO_NODE_SOCKET_PATH"] = orig_socket
+
+    @allure.link(helpers.get_vcs_link())
+    def test_rollback(
+        self,
+        cluster_manager: cluster_management.ClusterManager,
+        cluster_singleton: clusterlib.ClusterLib,
+        payment_addrs: List[clusterlib.AddressRecord],
+        backup_topology: Path,
+        split_topology_dir: Path,
+    ):
+        """Test rollback."""
+        cluster = cluster_singleton
+        temp_template = common.get_test_id(cluster)
+        last_pool = f"pool{configuration.NUM_POOLS}"
+
+        tx_outputs = []
+
+        # Submit Tx number 1
+        tx_outputs.append(
+            self.node_submit_tx(
+                cluster_obj=cluster,
+                node="pool1",
+                temp_template=temp_template,
+                src_addr=payment_addrs[0],
+                dst_addr=payment_addrs[0],
+            )
+        )
+
+        with cluster_manager.respin_on_failure():
+            # Split the network
+            self.split_network(split_topology_dir=split_topology_dir)
+
+            # Check that the Tx number 1 exists on both parts of the split network
+            assert self.node_query_utxo(
+                cluster_obj=cluster, node="pool1", tx_raw_output=tx_outputs[-1]
+            ), "The Tx number 1 doesn't exist on network 1"
+            assert self.node_query_utxo(
+                cluster_obj=cluster, node=last_pool, tx_raw_output=tx_outputs[-1]
+            ), "The Tx number 1 doesn't exist on network 2"
+
+            # Submit a Tx number 2 in first part of the split network
+            tx_outputs.append(
+                self.node_submit_tx(
+                    cluster_obj=cluster,
+                    node="pool1",
+                    temp_template=temp_template,
+                    src_addr=payment_addrs[1],
+                    dst_addr=payment_addrs[1],
+                )
+            )
+
+            # Check that the Tx number 2 exists only on the first part of the split network
+            assert self.node_query_utxo(
+                cluster_obj=cluster, node="pool1", tx_raw_output=tx_outputs[-1]
+            ), "The Tx number 2 doesn't exist on network 1"
+            assert not self.node_query_utxo(
+                cluster_obj=cluster, node=last_pool, tx_raw_output=tx_outputs[-1]
+            ), "The Tx number 2 does exist on network 2"
+
+            # Submit a Tx number 3 in the second part of the split network
+            tx_outputs.append(
+                self.node_submit_tx(
+                    cluster_obj=cluster,
+                    node=last_pool,
+                    temp_template=temp_template,
+                    src_addr=payment_addrs[2],
+                    dst_addr=payment_addrs[2],
+                )
+            )
+
+            # Check that the Tx number 3 exists only on the second part of the split network
+            assert not self.node_query_utxo(
+                cluster_obj=cluster, node="pool1", tx_raw_output=tx_outputs[-1]
+            ), "The Tx number 3 does exist on network 1"
+            assert self.node_query_utxo(
+                cluster_obj=cluster, node=last_pool, tx_raw_output=tx_outputs[-1]
+            ), "The Tx number 3 doesn't exist on network 2"
+
+            # Wait for several new blocks to let chains progress
+            cluster.wait_for_new_block(new_blocks=3)
+
+            # Restore the network
+            self.restore_network(backup_topology=backup_topology)
+
+            # Wait a bit for rollback to happen
+            time.sleep(10)
+
+            # Check that the network is no longer split
+            utxo_tx2_net1 = self.node_query_utxo(
+                cluster_obj=cluster, node="pool1", tx_raw_output=tx_outputs[-2]
+            )
+            utxo_tx2_net2 = self.node_query_utxo(
+                cluster_obj=cluster, node=last_pool, tx_raw_output=tx_outputs[-2]
+            )
+            utxo_tx3_net1 = self.node_query_utxo(
+                cluster_obj=cluster, node="pool1", tx_raw_output=tx_outputs[-1]
+            )
+            utxo_tx3_net2 = self.node_query_utxo(
+                cluster_obj=cluster, node=last_pool, tx_raw_output=tx_outputs[-1]
+            )
+
+            assert utxo_tx2_net1 == utxo_tx2_net2, "UTxOs are not identical, chain is still split?"
+            assert utxo_tx3_net1 == utxo_tx3_net2, "UTxOs are not identical, chain is still split?"
+
+            assert (
+                utxo_tx2_net1 or utxo_tx3_net1
+            ), "Neither Tx number 2 nor Tx number 3 exists on chain"
+
+            # At this point we know that the network is not plit, so we don't need to respin
+            # the cluster if the test fails.
+
+        assert not (
+            utxo_tx2_net1 and utxo_tx3_net1
+        ), "Neither Tx number 2 nor Tx number 3 was rolled back"