refactor(test configuration): network interface configuration

We need to configure Scylla networking with multiple NIC/IP combinations.
There are a few addresses to configure Scylla connections: rpc_address, listen_address,
broadcast_address, broadcast_rpc_address.

We want to be able to use different NIC/IP for addresses, at least, for rpc_address and
listen_address:
- rpc_address: ipv4, private, nic 0
- listen_address: ipv4, public, nic 1

This commit presents SCT configuration changes for support this.

According to issue scylladb/scylla-manager#3411

configurations/network_config/all_addresses_ipv6_public.yaml

@@ -0,0 +1,28 @@
+scylla_network_config:
+- address: listen_address  # Address Scylla listens for connections from other nodes. See storage_port and ssl_storage_ports.
+  ip_type: ipv6
+  public: true
+  listen_all: false  # Should be True when multiple interfaces - Scylla should be listening on all interfaces
+  use_dns: false
+  nic: 0
+- address: rpc_address  # Address on which Scylla is going to expect Thrift and CQL client connections.
+  ip_type: ipv6
+  public: true
+  listen_all: false  # Should be True when multiple interfaces - Scylla should be listening on all interfaces
+  use_dns: false
+  nic: 0
+- address: broadcast_rpc_address  # Address that is broadcasted to tell the clients to connect to. Related to rpc_address.
+  ip_type: ipv6
+  public: true  # Should be False when multiple interfaces
+  use_dns: false
+  nic: 0
+- address: broadcast_address  # Address that is broadcasted to tell other Scylla nodes to connect to. Related to listen_address above.
+  ip_type: ipv6
+  public: true  # Should be False when multiple interfaces
+  use_dns: false
+  nic: 0  #  If ipv4 and public is True it has to be primary network interface (device index is 0)
+- address: test_communication  # Type of IP used to connect to machine instances
+  ip_type: ipv6
+  public: true
+  use_dns: false
+  nic: 0  #  If ipv4 and public is True it has to be primary network interface (device index is 0)

configurations/network_config/test_communication_public.yaml

@@ -0,0 +1,28 @@
+scylla_network_config:
+- address: listen_address  # Address Scylla listens for connections from other nodes. See storage_port and ssl_storage_ports.
+  ip_type: ipv4
+  public: false
+  listen_all: false  # Should be True when multiple interfaces - Scylla should be listening on all interfaces
+  use_dns: false
+  nic: 0
+- address: rpc_address  # Address on which Scylla is going to expect Thrift and CQL client connections.
+  ip_type: ipv4
+  public: false
+  listen_all: false  # Should be True when multiple interfaces - Scylla should be listening on all interfaces
+  use_dns: false
+  nic: 0
+- address: broadcast_rpc_address  # Address that is broadcasted to tell the clients to connect to. Related to rpc_address.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces
+  use_dns: false
+  nic: 0
+- address: broadcast_address  # Address that is broadcasted to tell other Scylla nodes to connect to. Related to listen_address above.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces
+  use_dns: false
+  nic: 0  #  If ipv4 and public is True it has to be primary network interface (device index is 0)
+- address: test_communication  # Type of IP used to connect to machine instances
+  ip_type: ipv4
+  public: true
+  use_dns: false
+  nic: 0  #  If ipv4 and public is True it has to be primary network interface (device index is 0)

configurations/network_config/two_interfaces.yaml

@@ -0,0 +1,28 @@
+scylla_network_config:
+- address: listen_address  # Address Scylla listens for connections from other nodes. See storage_port and ssl_storage_ports.
+  ip_type: ipv4
+  public: false
+  listen_all: true  # Should be True when multiple interfaces - Scylla should be listening on all interfaces
+  use_dns: false
+  nic: 1
+- address: rpc_address  # Address on which Scylla is going to expect Thrift and CQL client connections.
+  ip_type: ipv4
+  public: false
+  listen_all: true  # Should be True when multiple interfaces - Scylla should be listening on all interfaces
+  use_dns: false
+  nic: 1
+- address: broadcast_rpc_address  # Address that is broadcasted to tell the clients to connect to. Related to rpc_address.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces
+  use_dns: false
+  nic: 1
+- address: broadcast_address  # Address that is broadcasted to tell other Scylla nodes to connect to. Related to listen_address above.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces
+  use_dns: false
+  nic: 1  #  If ipv4 and public is True it has to be primary network interface (device index is 0)
+- address: test_communication  # Type of IP used to connect from test to DB/monitor instances
+  ip_type: ipv4
+  public: true
+  use_dns: false
+  nic: 0  #  If ipv4 and public is True it has to be primary network interface (device index is 0)

defaults/aws_config.yaml

@@ -37,3 +37,37 @@ data_volume_disk_size: 500
 data_volume_disk_iops: 10000 # depend on type iops could be 100-16000 for io2|io3 and 3000-16000 for gp3
 
 kms_key_rotation_interval: 60
+
+# TODO: this part should be moved to defaults/test_default.yaml when network interfaces configuration is supported for all backends
+# NOT SUPPORTED CASES:
+#   1. Multi network interfaces with IPv6 type
+
+scylla_network_config:
+- address: listen_address  # Address Scylla listens for connections from other nodes. See storage_port and ssl_storage_ports.
+  listen_all: false  # Should be True when multiple interfaces - Scylla should be listening on all interfaces
+  ip_type: ipv4
+  public: false  # Only public IPv6 is supported by AWS
+  use_dns: false
+  nic: 0
+- address: rpc_address  # Address on which Scylla is going to expect Thrift and CQL client connections.
+  listen_all: false  # Should be True when multiple interfaces - Scylla should be listening on all interfaces.
+  ip_type: ipv4
+  public: false  # Only public IPv6 is supported by AWS
+  use_dns: false
+  nic: 0
+- address: broadcast_rpc_address  # Address that is broadcasted to tell the clients to connect to. Related to rpc_address.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces; Only public IPv6 is supported by AWS
+  use_dns: false
+  nic: 0
+- address: broadcast_address  # Address that is broadcasted to tell other Scylla nodes to connect to. Related to listen_address above.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces; Only public IPv6 is supported by AWS
+  use_dns: false
+  nic: 0
+- address: test_communication  # Type of IP used to connect from test to DB/monitor instances
+  ip_type: ipv4
+  public: false
+  use_dns: false
+  nic: 0  #  If ipv4 and public is True it has to be primary network interface (device index is 0)
+# TODO: end

internal_test_data/network_config_interface_not_defined.yaml

@@ -0,0 +1,39 @@
+test_duration: 5
+
+n_db_nodes: 3
+n_loaders: 1
+n_monitor_nodes: 1
+user_prefix: 'fruch-testing'
+
+stress_cmd: ["cassandra-stress mixed cl=QUORUM duration=10m -schema 'replication(strategy=NetworkTopologyStrategy,replication_factor=3) compaction(strategy=SizeTieredCompactionStrategy)' -mode cql3 native -rate threads=2 -pop seq=1..3000 -log interval=5" ]
+
+stress_read_cmd: ["cassandra-stress user profile=/tmp/c-s_profile_4mv_5queries.yaml ops'(insert=15,read1=1,read2=1,read3=1,read4=1,read5=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10",
+                  "cassandra-stress user profile=/tmp/c-s_profile_2mv_2queries.yaml ops'(insert=6,mv_p_read1=1,mv_p_read2=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10",
+                  "cassandra-stress user profile=/tmp/c-s_profile_3si_5queries.yaml ops'(insert=25,si_read1=1,si_read2=1,si_read3=1,si_read4=1,si_read5=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10",
+                  "cassandra-stress user profile=/tmp/c-s_profile_2si_2queries.yaml ops'(insert=10,si_p_read1=1,si_p_read2=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10"
+                 ]
+
+db_type: scylla
+instance_type_db: 'i4i.large'
+
+scylla_network_config:
+- address: "listen_address"
+  ip_type: “ipv4”
+  public: false
+  use_dns: false
+  nic: 0
+- address: broadcast_rpc_address  # Address that is broadcasted to tell the clients to connect to. Related to rpc_address.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces; Only public IPv6 is supported by AWS
+  use_dns: false
+  nic: 0
+- address: broadcast_address  # Address that is broadcasted to tell other Scylla nodes to connect to. Related to listen_address above.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces; Only public IPv6 is supported by AWS
+  use_dns: false
+  nic: 0
+- address: test_communication  # Type of IP used to connect from test to DB/monitor instances
+  ip_type: ipv4
+  public: false
+  use_dns: false
+  nic: 0  #  If ipv4 and public is True it has to be primary network interface (device index is 0)

internal_test_data/network_config_interface_param_not_defined.yaml

@@ -0,0 +1,28 @@
+test_duration: 5
+
+n_db_nodes: 3
+n_loaders: 1
+n_monitor_nodes: 1
+user_prefix: 'fruch-testing'
+
+stress_cmd: ["cassandra-stress mixed cl=QUORUM duration=10m -schema 'replication(strategy=NetworkTopologyStrategy,replication_factor=3) compaction(strategy=SizeTieredCompactionStrategy)' -mode cql3 native -rate threads=2 -pop seq=1..3000 -log interval=5" ]
+
+stress_read_cmd: ["cassandra-stress user profile=/tmp/c-s_profile_4mv_5queries.yaml ops'(insert=15,read1=1,read2=1,read3=1,read4=1,read5=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10",
+                  "cassandra-stress user profile=/tmp/c-s_profile_2mv_2queries.yaml ops'(insert=6,mv_p_read1=1,mv_p_read2=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10",
+                  "cassandra-stress user profile=/tmp/c-s_profile_3si_5queries.yaml ops'(insert=25,si_read1=1,si_read2=1,si_read3=1,si_read4=1,si_read5=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10",
+                  "cassandra-stress user profile=/tmp/c-s_profile_2si_2queries.yaml ops'(insert=10,si_p_read1=1,si_p_read2=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10"
+                 ]
+
+db_type: scylla
+instance_type_db: 'i4i.large'
+
+scylla_network_config:
+- address: "listen_address"
+  ip_type: “ipv4”
+  use_dns: false
+  nic: 0
+- address: "rpc_address"
+  ip_type: “ipv6”
+  public: true
+  use_dns: false
+  nic: 0

internal_test_data/network_config_interface_param_public_not_primary.yaml

@@ -0,0 +1,46 @@
+test_duration: 5
+
+n_db_nodes: 3
+n_loaders: 1
+n_monitor_nodes: 1
+user_prefix: 'fruch-testing'
+
+stress_cmd: ["cassandra-stress mixed cl=QUORUM duration=10m -schema 'replication(strategy=NetworkTopologyStrategy,replication_factor=3) compaction(strategy=SizeTieredCompactionStrategy)' -mode cql3 native -rate threads=2 -pop seq=1..3000 -log interval=5" ]
+
+stress_read_cmd: ["cassandra-stress user profile=/tmp/c-s_profile_4mv_5queries.yaml ops'(insert=15,read1=1,read2=1,read3=1,read4=1,read5=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10",
+                  "cassandra-stress user profile=/tmp/c-s_profile_2mv_2queries.yaml ops'(insert=6,mv_p_read1=1,mv_p_read2=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10",
+                  "cassandra-stress user profile=/tmp/c-s_profile_3si_5queries.yaml ops'(insert=25,si_read1=1,si_read2=1,si_read3=1,si_read4=1,si_read5=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10",
+                  "cassandra-stress user profile=/tmp/c-s_profile_2si_2queries.yaml ops'(insert=10,si_p_read1=1,si_p_read2=1)' cl=QUORUM duration=5760m -mode cql3 native -rate threads=10"
+                 ]
+
+db_type: scylla
+instance_type_db: 'i4i.large'
+
+scylla_network_config:
+- address: listen_address  # Address Scylla listens for connections from other nodes. See storage_port and ssl_storage_ports.
+  ip_type: ipv4
+  public: false
+  listen_all: true  # Should be True when multiple interfaces - Scylla should be listening on all interfaces
+  use_dns: false
+  nic: 1
+- address: rpc_address  # Address on which Scylla is going to expect Thrift and CQL client connections.
+  ip_type: ipv4
+  public: false
+  listen_all: true  # Should be True when multiple interfaces - Scylla should be listening on all interfaces
+  use_dns: false
+  nic: 1
+- address: broadcast_rpc_address  # Address that is broadcasted to tell the clients to connect to. Related to rpc_address.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces
+  use_dns: false
+  nic: 1
+- address: broadcast_address  # Address that is broadcasted to tell other Scylla nodes to connect to. Related to listen_address above.
+  ip_type: ipv4
+  public: false  # Should be False when multiple interfaces
+  use_dns: false
+  nic: 1  #  If ipv4 and public is True it has to be primary network interface (device index is 0)
+- address: test_communication  # Type of IP used to connect from test to DB/monitor instances
+  ip_type: ipv4
+  public: true
+  use_dns: false
+  nic: 1  #  If ipv4 and public is True it has to be primary network interface (device index is 0)

jenkins-pipelines/longevity-10gb-3h-ipv6.jenkinsfile

@@ -6,8 +6,7 @@ def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm)
 longevityPipeline(
     backend: 'aws',
     region: 'eu-west-1',
-    ip_ssh_connections: 'ipv6',
     test_name: 'longevity_test.LongevityTest.test_custom_time',
-    test_config: 'test-cases/longevity/longevity-10gb-3h.yaml'
+    test_config: '''["test-cases/longevity/longevity-10gb-3h.yaml", "configurations/network_config/all_addresses_ipv6_public.yaml"]'''
 
 )

jenkins-pipelines/longevity-200gb-48h-network-monkey.jenkinsfile

@@ -7,6 +7,5 @@ longevityPipeline(
     backend: 'aws',
     region: 'eu-west-1',
     test_name: 'longevity_test.LongevityTest.test_custom_time',
-    test_config: 'test-cases/longevity/longevity-200GB-48h-network-monkey.yaml',
-    ip_ssh_connections: 'public'
+    test_config: '''["test-cases/longevity/longevity-200GB-48h-network-monkey.yaml", "configurations/network_config/two_interfaces.yaml"]'''
 )

jenkins-pipelines/manager/centos-manager-sanity-ipv6.jenkinsfile

@@ -5,10 +5,9 @@ def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm)
 
 managerPipeline(
     backend: 'aws',
-    ip_ssh_connections: 'ipv6',
     region: 'us-east-1',
     test_name: 'mgmt_cli_test.MgmtCliTest.test_manager_sanity',
-    test_config: 'test-cases/manager/manager-regression-ipv6.yaml',
+    test_config: '''["test-cases/manager/manager-regression-ipv6.yaml", "configurations/network_config/all_addresses_ipv6_public.yaml"]''',
 
     post_behavior_db_nodes: 'destroy',
     post_behavior_loader_nodes: 'destroy',

jenkins-pipelines/nemesis/longevity-5gb-1h-BlockNetworkMonkey-aws.jenkinsfile

@@ -7,8 +7,6 @@ longevityPipeline(
     backend: 'aws',
     region: 'eu-west-1',
     test_name: 'longevity_test.LongevityTest.test_custom_time',
-    test_config: """['configurations/nemesis/longevity-5gb-1h-nemesis.yaml', 'configurations/nemesis/BlockNetworkMonkey.yaml', 'configurations/nemesis/additional_configs/extra_interface_public.yaml']""",
-
-    ip_ssh_connections: "public"
+    test_config: """['configurations/nemesis/longevity-5gb-1h-nemesis.yaml', 'configurations/nemesis/BlockNetworkMonkey.yaml', 'configurations/network_config/two_interfaces.yaml']""",
 
 )

jenkins-pipelines/nemesis/longevity-5gb-1h-RandomInterruptionNetworkMonkey-aws.jenkinsfile

@@ -7,8 +7,6 @@ longevityPipeline(
     backend: 'aws',
     region: 'eu-west-1',
     test_name: 'longevity_test.LongevityTest.test_custom_time',
-    test_config: """['configurations/nemesis/longevity-5gb-1h-nemesis.yaml', 'configurations/nemesis/RandomInterruptionNetworkMonkey.yaml', 'configurations/nemesis/additional_configs/extra_interface_public.yaml']""",
-
-    ip_ssh_connections: "public"
+    test_config: """['configurations/nemesis/longevity-5gb-1h-nemesis.yaml', 'configurations/nemesis/RandomInterruptionNetworkMonkey.yaml', 'configurations/network_config/two_interfaces.yaml']""",
 
 )

jenkins-pipelines/nemesis/longevity-5gb-1h-StopStartInterfacesNetworkMonkey-aws.jenkinsfile

@@ -7,8 +7,6 @@ longevityPipeline(
     backend: 'aws',
     region: 'eu-west-1',
     test_name: 'longevity_test.LongevityTest.test_custom_time',
-    test_config: """['configurations/nemesis/longevity-5gb-1h-nemesis.yaml', 'configurations/nemesis/StopStartInterfacesNetworkMonkey.yaml', 'configurations/nemesis/additional_configs/extra_interface_public.yaml']""",
-
-    ip_ssh_connections: "public"
+    test_config: """['configurations/nemesis/longevity-5gb-1h-nemesis.yaml', 'configurations/nemesis/StopStartInterfacesNetworkMonkey.yaml', 'configurations/network_config/two_interfaces.yaml']""",
 
 )

jenkins-pipelines/raft/longevity-200gb-48h-network-monkey.jenkinsfile

@@ -7,6 +7,5 @@ longevityPipeline(
     backend: 'aws',
     region: 'eu-west-1',
     test_name: 'longevity_test.LongevityTest.test_custom_time',
-    test_config: '''["test-cases/longevity/longevity-200GB-48h-network-monkey.yaml", "configurations/raft/enable_raft_experimental.yaml"]''',
-    ip_ssh_connections: 'public'
+    test_config: '''["test-cases/longevity/longevity-200GB-48h-network-monkey.yaml", "configurations/raft/enable_raft_experimental.yaml", "configurations/network_config/two_interfaces.yaml"]'''
 )