From 973944eb0448ec67b55286707170bf983b2dbdb2 Mon Sep 17 00:00:00 2001 From: Grant Date: Mon, 24 Jun 2024 17:28:46 -0500 Subject: [PATCH 01/10] account for test networks simplifying get_peer_ip --- src/warnet/test_framework_bridge.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/warnet/test_framework_bridge.py b/src/warnet/test_framework_bridge.py index 1e4484590..ea8c9f029 100644 --- a/src/warnet/test_framework_bridge.py +++ b/src/warnet/test_framework_bridge.py @@ -337,8 +337,15 @@ def get_peer_ip(peer): try: # we encounter a regular ip address ip_addr = str(ipaddress.ip_address(peer['addr'].split(':')[0])) return ip_addr - except ValueError: # or we encounter a service name - tank_index = int(peer['addr'].split('-')[2]) # NETWORK-tank-INDEX-service + except ValueError as err: # or we encounter a service name + try: + # NETWORK-tank-TANK_INDEX-service + # NETWORK-test-TEST-tank-TANK_INDEX-service + tank_index = int(peer['addr'].split('-')[-2]) + except (ValueError, IndexError) as inner_err: + raise ValueError("could not derive tank index from service name: {} {}" + .format(peer['addr'], inner_err)) from err + ip_addr = self.warnet.tanks[tank_index].get_ip_addr() return ip_addr From e7f8de079ef567c62b48ec9cbecebd2e0b89b884 Mon Sep 17 00:00:00 2001 From: Grant Date: Mon, 24 Jun 2024 18:52:39 -0500 Subject: [PATCH 02/10] update the test to account for dns node names --- src/scenarios/connect_dag.py | 6 ++++++ test/dag_connection_test.py | 2 +- ...d.graphml => ten_semi_unconnected.graphml} | 19 ++++++++++++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) rename test/data/{eight_unconnected.graphml => ten_semi_unconnected.graphml} (83%) diff --git a/src/scenarios/connect_dag.py b/src/scenarios/connect_dag.py index 5f65842b3..59324c485 100644 --- a/src/scenarios/connect_dag.py +++ b/src/scenarios/connect_dag.py @@ -58,6 +58,12 @@ def run_test(self): self.connect_nodes(5, 4) self.connect_nodes(5, 6) self.connect_nodes(6, 7) + + # Nodes 8 & 9 shall come pre-connected. Attempt to connect them anyway to test the handling + # of dns node addresses + self.connect_nodes(8, 9) + self.connect_nodes(9, 8) + self.sync_all() zero_peers = self.nodes[0].getpeerinfo() diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index 30d6b2da6..06d36c206 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -6,7 +6,7 @@ from test_base import TestBase -graph_file_path = Path(os.path.dirname(__file__)) / "data" / "eight_unconnected.graphml" +graph_file_path = Path(os.path.dirname(__file__)) / "data" / "ten_semi_unconnected.graphml" base = TestBase() diff --git a/test/data/eight_unconnected.graphml b/test/data/ten_semi_unconnected.graphml similarity index 83% rename from test/data/eight_unconnected.graphml rename to test/data/ten_semi_unconnected.graphml index 5c2b7b10a..065be6127 100644 --- a/test/data/eight_unconnected.graphml +++ b/test/data/ten_semi_unconnected.graphml @@ -73,5 +73,22 @@ xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdr False False + + 26.0 + + + + False + False + + + 26.0 + + + + False + False + + - \ No newline at end of file + From ee900f953dffc5c9bfd5e9053a7d2f40b1e80399 Mon Sep 17 00:00:00 2001 From: Grant Date: Wed, 26 Jun 2024 14:18:16 -0500 Subject: [PATCH 03/10] clarify connect dag w/ function; update node count Also, add assertion for pre-connected nodes --- src/scenarios/connect_dag.py | 81 ++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/src/scenarios/connect_dag.py b/src/scenarios/connect_dag.py index 59324c485..0b3ea133d 100644 --- a/src/scenarios/connect_dag.py +++ b/src/scenarios/connect_dag.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +from enum import Enum, auto, unique from time import sleep from warnet.test_framework_bridge import WarnetTestFramework @@ -10,10 +11,16 @@ def cli_help(): return "Connect a complete DAG from a set of unconnected nodes" +@unique +class ConnectionType(Enum): + IP = auto() + DNS = auto() + + class ConnectDag(WarnetTestFramework): def set_test_params(self): # This is just a minimum - self.num_nodes = 8 + self.num_nodes = 10 def add_options(self, parser): parser.add_argument( @@ -74,50 +81,50 @@ def run_test(self): five_peers = self.nodes[5].getpeerinfo() six_peers = self.nodes[6].getpeerinfo() seven_peers = self.nodes[7].getpeerinfo() + eight_peers = self.nodes[8].getpeerinfo() + nine_peers = self.nodes[9].getpeerinfo() for tank in self.warnet.tanks: self.log.info(f"Tank {tank.index}: {tank.warnet.tanks[tank.index].get_dns_addr()} pod:" f" {tank.warnet.tanks[tank.index].get_ip_addr()}") - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[2].get_dns_addr() for d in - zero_peers), f"Could not find {self.options.network_name}-tank-000002-service" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[2].get_dns_addr() for d in - one_peers), f"Could not find {self.options.network_name}-tank-000002-service" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[3].get_dns_addr() for d in - one_peers), f"Could not find {self.options.network_name}-tank-000003-service" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[0].get_ip_addr() for d in - two_peers), f"Could not find Tank 0's ip addr" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[1].get_ip_addr() for d in - two_peers), f"Could not find Tank 1's ip addr" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[3].get_dns_addr() for d in - two_peers), f"Could not find {self.options.network_name}-tank-000003-service" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[4].get_dns_addr() for d in - two_peers), f"Could not find {self.options.network_name}-tank-000004-service" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[1].get_ip_addr() for d in - three_peers), f"Could not find Tank 1's ip addr" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[2].get_ip_addr() for d in - three_peers), f"Could not find Tank 2's ip addr" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[5].get_dns_addr() for d in - three_peers), f"Could not find {self.options.network_name}-tank-000005-service" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[2].get_ip_addr() for d in - four_peers), f"Could not find Tank 2's ip addr" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[5].get_ip_addr() for d in - four_peers), f"Could not find Tank 5's ip addr" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[3].get_ip_addr() for d in - five_peers), f"Could not find Tank 3's ip addr" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[4].get_dns_addr() for d in - five_peers), f"Could not find {self.options.network_name}-tank-000004-service" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[6].get_dns_addr() for d in - five_peers), f"Could not find {self.options.network_name}-tank-000006-service" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[5].get_ip_addr() for d in - six_peers), f"Could not find Tank 5's ip addr" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[7].get_dns_addr() for d in - six_peers), f"Could not find {self.options.network_name}-tank-000007-service" - assert any(d.get("addr").split(":")[0] == self.warnet.tanks[6].get_ip_addr() for d in - seven_peers), f"Could not find Tank 6's ip addr" + self.assert_connection(zero_peers, 2, ConnectionType.DNS) + self.assert_connection(one_peers, 2, ConnectionType.DNS) + self.assert_connection(one_peers, 3, ConnectionType.DNS) + self.assert_connection(two_peers, 0, ConnectionType.IP) + self.assert_connection(two_peers, 1, ConnectionType.IP) + self.assert_connection(two_peers, 3, ConnectionType.DNS) + self.assert_connection(two_peers, 4, ConnectionType.DNS) + self.assert_connection(three_peers, 1, ConnectionType.IP) + self.assert_connection(three_peers, 2, ConnectionType.IP) + self.assert_connection(three_peers, 5, ConnectionType.DNS) + self.assert_connection(four_peers, 2, ConnectionType.IP) + self.assert_connection(four_peers, 5, ConnectionType.IP) + self.assert_connection(five_peers, 3, ConnectionType.IP) + self.assert_connection(five_peers, 4, ConnectionType.DNS) + self.assert_connection(five_peers, 6, ConnectionType.DNS) + self.assert_connection(six_peers, 5, ConnectionType.IP) + self.assert_connection(six_peers, 7, ConnectionType.DNS) + self.assert_connection(seven_peers, 6, ConnectionType.IP) + # Check the pre-connected nodes + self.assert_connection(eight_peers, 9, ConnectionType.DNS) + self.assert_connection(nine_peers, 8, ConnectionType.IP) self.log.info(f"Successfully ran the {os.path.basename(__file__)} scenario.") + def assert_connection(self, connector, connectee_index, connection_type: ConnectionType): + if connection_type == ConnectionType.DNS: + assert any(d.get("addr") == + self.warnet.tanks[connectee_index].get_dns_addr() for d in connector), \ + (f"Could not find {self.options.network_name}-" + f"tank-00000{connectee_index}-service") + elif connection_type == ConnectionType.IP: + assert any(d.get("addr").split(":")[0] == + self.warnet.tanks[connectee_index].get_ip_addr() for d in connector), \ + f"Could not find Tank {connectee_index}'s ip addr" + else: + raise ValueError("ConnectionType must be of type DNS or IP") + if __name__ == "__main__": ConnectDag().main() From 5fcb33778d7ff7e56abf2150f3eec1da37b190a7 Mon Sep 17 00:00:00 2001 From: Grant Date: Fri, 28 Jun 2024 13:57:44 -0500 Subject: [PATCH 04/10] reformat to single line --- src/scenarios/connect_dag.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/scenarios/connect_dag.py b/src/scenarios/connect_dag.py index 0b3ea133d..54f59cdee 100644 --- a/src/scenarios/connect_dag.py +++ b/src/scenarios/connect_dag.py @@ -116,8 +116,7 @@ def assert_connection(self, connector, connectee_index, connection_type: Connect if connection_type == ConnectionType.DNS: assert any(d.get("addr") == self.warnet.tanks[connectee_index].get_dns_addr() for d in connector), \ - (f"Could not find {self.options.network_name}-" - f"tank-00000{connectee_index}-service") + f"Could not find {self.options.network_name}-tank-00000{connectee_index}-service" elif connection_type == ConnectionType.IP: assert any(d.get("addr").split(":")[0] == self.warnet.tanks[connectee_index].get_ip_addr() for d in connector), \ From 9e39440683e8d62692c69f18a808db58e972755c Mon Sep 17 00:00:00 2001 From: Grant Date: Fri, 28 Jun 2024 18:10:51 -0500 Subject: [PATCH 05/10] make ruff happy --- src/backend/kubernetes_backend.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/kubernetes_backend.py b/src/backend/kubernetes_backend.py index c1077d2e9..4f45b4209 100644 --- a/src/backend/kubernetes_backend.py +++ b/src/backend/kubernetes_backend.py @@ -420,9 +420,7 @@ def check_logging_crds_installed(self): logging_crd_name = "servicemonitors.monitoring.coreos.com" api = client.ApiextensionsV1Api() crds = api.list_custom_resource_definition() - if any(crd.metadata.name == logging_crd_name for crd in crds.items): - return True - return False + return bool(any(crd.metadata.name == logging_crd_name for crd in crds.items)) def apply_prometheus_service_monitors(self, tanks): for tank in tanks: From 1010638cdd6827f35b2ec6cf0539ee20cba780bd Mon Sep 17 00:00:00 2001 From: Grant Date: Fri, 28 Jun 2024 18:18:03 -0500 Subject: [PATCH 06/10] remove network name --- test/dag_connection_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index 06d36c206..a8e1c520e 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -16,7 +16,7 @@ base.wait_for_all_edges() # Start scenario -base.warcli(f"scenarios run connect_dag --network_name={base.network_name}") +base.warcli(f"scenarios run connect_dag") counter = 0 seconds = 180 From 52ba2b58e4ba353ec8a40f52d85a2eaf7119ab8d Mon Sep 17 00:00:00 2001 From: Grant Date: Fri, 28 Jun 2024 18:26:29 -0500 Subject: [PATCH 07/10] remove extraneous "f" --- test/dag_connection_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index a8e1c520e..fc1cd597d 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -16,7 +16,7 @@ base.wait_for_all_edges() # Start scenario -base.warcli(f"scenarios run connect_dag") +base.warcli("scenarios run connect_dag") counter = 0 seconds = 180 From 4b578028c4b05a74ede1fb4a8478cbe2934e11dd Mon Sep 17 00:00:00 2001 From: Grant Date: Fri, 28 Jun 2024 22:50:56 -0500 Subject: [PATCH 08/10] move connect_dag.py --- test/dag_connection_test.py | 2 +- test/framework_tests/__init__.py | 0 {src/scenarios => test/framework_tests}/connect_dag.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 test/framework_tests/__init__.py rename {src/scenarios => test/framework_tests}/connect_dag.py (98%) diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index fc1cd597d..07f5fd0cf 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -16,7 +16,7 @@ base.wait_for_all_edges() # Start scenario -base.warcli("scenarios run connect_dag") +base.warcli("scenarios run-file test/framework_tests/connect_dag.py") counter = 0 seconds = 180 diff --git a/test/framework_tests/__init__.py b/test/framework_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/scenarios/connect_dag.py b/test/framework_tests/connect_dag.py similarity index 98% rename from src/scenarios/connect_dag.py rename to test/framework_tests/connect_dag.py index 54f59cdee..a1fc2e168 100644 --- a/src/scenarios/connect_dag.py +++ b/test/framework_tests/connect_dag.py @@ -110,7 +110,7 @@ def run_test(self): self.assert_connection(eight_peers, 9, ConnectionType.DNS) self.assert_connection(nine_peers, 8, ConnectionType.IP) - self.log.info(f"Successfully ran the {os.path.basename(__file__)} scenario.") + self.log.info(f"Successfully ran the scenario file: {os.path.basename(__file__)} ") def assert_connection(self, connector, connectee_index, connection_type: ConnectionType): if connection_type == ConnectionType.DNS: From cae9923585bad89ba19cd19d27eb20921ca26501 Mon Sep 17 00:00:00 2001 From: Grant Date: Mon, 1 Jul 2024 13:25:17 -0500 Subject: [PATCH 09/10] decrement scenario count --- test/scenarios_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scenarios_test.py b/test/scenarios_test.py index cd7bd55ab..445da2615 100755 --- a/test/scenarios_test.py +++ b/test/scenarios_test.py @@ -14,7 +14,7 @@ # Use rpc instead of warcli so we get raw JSON object scenarios = base.rpc("scenarios_available") -assert len(scenarios) == 5 +assert len(scenarios) == 4 # Start scenario base.warcli("scenarios run miner_std --allnodes --interval=1") From 3a0756ae90de5541c6a7d85ba476de9c2082bb52 Mon Sep 17 00:00:00 2001 From: Grant Date: Tue, 2 Jul 2024 10:21:37 -0500 Subject: [PATCH 10/10] change "success" log entry --- test/framework_tests/connect_dag.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework_tests/connect_dag.py b/test/framework_tests/connect_dag.py index a1fc2e168..3ebd0ae21 100644 --- a/test/framework_tests/connect_dag.py +++ b/test/framework_tests/connect_dag.py @@ -110,7 +110,8 @@ def run_test(self): self.assert_connection(eight_peers, 9, ConnectionType.DNS) self.assert_connection(nine_peers, 8, ConnectionType.IP) - self.log.info(f"Successfully ran the scenario file: {os.path.basename(__file__)} ") + self.log.info(f"Successfully ran the connect_dag.py scenario using a temporary file: " + f"{os.path.basename(__file__)} ") def assert_connection(self, connector, connectee_index, connection_type: ConnectionType): if connection_type == ConnectionType.DNS: