From 769b2572ad342ac6319e8e7e8d1cedf45ccfd6da Mon Sep 17 00:00:00 2001 From: Stephen White <93026204+Steve-White-UK@users.noreply.github.com> Date: Tue, 31 Oct 2023 09:04:39 +0000 Subject: [PATCH] Fixes to the scripts and startups - does not touch cpp files. (#3849) --- tests/Zilliqa/test_zilliqa_NAT.py | 2 +- tests/Zilliqa/test_zilliqa_late.py | 2 +- tests/Zilliqa/test_zilliqa_local.py | 51 +++++++++++++++---- tests/Zilliqa/test_zilliqa_lookup.py | 72 +++++++++++---------------- tests/Zilliqa/test_zilliqa_seedpub.py | 2 +- tests/native/prepare_constants.py | 16 +++++- tests/native/start_native.sh | 3 +- 7 files changed, 91 insertions(+), 57 deletions(-) diff --git a/tests/Zilliqa/test_zilliqa_NAT.py b/tests/Zilliqa/test_zilliqa_NAT.py index d80ff23567..65532131b9 100755 --- a/tests/Zilliqa/test_zilliqa_NAT.py +++ b/tests/Zilliqa/test_zilliqa_NAT.py @@ -90,7 +90,7 @@ def run_start(): return dev_root += "/Zilliqa" - fp = dev_root + "/" + "constants_local.xml.native" + fp = dev_root + "/" + "constants.xml" if not os.path.exists(fp): print( fp +" not found") diff --git a/tests/Zilliqa/test_zilliqa_late.py b/tests/Zilliqa/test_zilliqa_late.py index f7351f19b2..9cf493cf41 100755 --- a/tests/Zilliqa/test_zilliqa_late.py +++ b/tests/Zilliqa/test_zilliqa_late.py @@ -95,7 +95,7 @@ def run_start(): keys_file = open(LOCAL_RUN_FOLDER + 'keys.txt', "w") for x in range(0, count): keys_file.write(keypairs[x] + '\n') - shutil.copyfile('constants_local.xml.native', LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') + shutil.copyfile('constants.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') keys_file.close() diff --git a/tests/Zilliqa/test_zilliqa_local.py b/tests/Zilliqa/test_zilliqa_local.py index b6bd404814..77e7f5d17e 100755 --- a/tests/Zilliqa/test_zilliqa_local.py +++ b/tests/Zilliqa/test_zilliqa_local.py @@ -26,6 +26,7 @@ import xml.etree.cElementTree as ET NODE_LISTEN_PORT = 5001 +NODE_LOOKUP_PORT = 23456 STATUS_SERVER_LISTEN_PORT = 4301 LOCAL_RUN_FOLDER = './local_run/' REJOIN_DS_GUARD_RUN_FOLDER = './dsguard_rejoin_local_run/' @@ -270,13 +271,7 @@ def run_start(numdsnodes): testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) count = len(testfolders_list) - dev_root = os.getenv("DEV_TREE_ROOT") - if dev_root is None: - print("DEV_TREE_ROOT is not set") - return - - dev_root += "/Zilliqa" - fp = dev_root + "/" + "constants_local.xml.native" + fp = "constants.xml" if not os.path.exists(fp): print( fp +" not found") @@ -296,6 +291,10 @@ def run_start(numdsnodes): shutil.copyfile(fp, LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') ipc_path = "/tmp/zilliqa" + str(NODE_LISTEN_PORT + x) + ".sock" status_server_port = str(STATUS_SERVER_LISTEN_PORT + x) + + '''patch_lookup_pubkey(LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml', keypairs)''' + '''patch_seed_pubkey(LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml', keypairs)''' + patch_param_in_xml(LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml', ipc_path, status_server_port) shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') @@ -328,6 +327,40 @@ def patch_param_in_xml(filepath, ipc_path, status_server_port): tree.write(filepath) +def patch_lookup_pubkey(filepath, keypairs): + root = ET.parse(filepath).getroot() + td = root.find('lookups') + if td: + root.remove(td) + root.append(ET.Element('lookups')) + td = root.find('lookups') + p = ET.SubElement(td, "peer") + ET.SubElement(p, "pubkey").text = keypairs[0].split(" ")[0] + ET.SubElement(p, "ip").text = '127.0.0.1' + ET.SubElement(p, "port").text = str(NODE_LISTEN_PORT) + ET.SubElement(p, "hostname").text = None + + tree = ET.ElementTree(root) + tree.write(filepath) + + +def patch_seed_pubkey(filepath, keypairs): + root = ET.parse(filepath).getroot() + td = root.find('upper_seed') + if td: + root.remove(td) + root.append(ET.Element('upper_seed')) + td = root.find('upper_seed') + p = ET.SubElement(td, "peer") + ET.SubElement(p, "pubkey").text = keypairs[0].split(" ")[0] + ET.SubElement(p, "ip").text = '127.0.0.1' + ET.SubElement(p, "port").text = str(NODE_LISTEN_PORT) + ET.SubElement(p, "hostname").text = None + + tree = ET.ElementTree(root) + tree.write(filepath) + + # To rejoin ds guard index 2 def run_start_dsguard2(): testfolders_list = get_immediate_subdirectories(REJOIN_DS_GUARD_RUN_FOLDER) @@ -335,7 +368,7 @@ def run_start_dsguard2(): for x in range(0, count): shutil.copyfile('dsnodes.xml', REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') - shutil.copyfile('constants_local.xml.native', REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x] + '/constants.xml') + shutil.copyfile('constants.xml', REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x] + '/constants.xml') # These keys are non critical and are only used for testing purposes keypairs = "021D99F2E5ACBA39ED5ACC5DCA5EE2ADDE780FFD998E1DBF440FE364C3BE360A7B 50C26000FCC08867FC3B9C03385015179E4B63282CB356014233BB1877FCDBDD" @@ -364,7 +397,7 @@ def run_start_validateBackupDB(): keypair = keypairs[0].split(" ") shutil.copyfile('ds_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/ds_whitelist.xml') shutil.copyfile('shard_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/shard_whitelist.xml') - shutil.copyfile('constants_local.xml.native', LOCAL_RUN_FOLDER + testfolders_list[0] + '/constants.xml') + shutil.copyfile('constants.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/constants.xml') shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/dsnodes.xml') shutil.copyfile('config_normal.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/config.xml') os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[0] + '; echo \"' + keypair[0] + ' ' + keypair[ diff --git a/tests/Zilliqa/test_zilliqa_lookup.py b/tests/Zilliqa/test_zilliqa_lookup.py index e5cb78acf1..ed1d7d404a 100755 --- a/tests/Zilliqa/test_zilliqa_lookup.py +++ b/tests/Zilliqa/test_zilliqa_lookup.py @@ -22,7 +22,7 @@ from subprocess import Popen, PIPE import xml.etree.cElementTree as ET -NODE_LISTEN_PORT = 40092 +NODE_LISTEN_PORT = 23456 LOCAL_RUN_FOLDER = './lookup_local_run/' LOCAL_FOLDER = "./" @@ -91,6 +91,8 @@ def run_setup(numnodes, printnodes): testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) count = len(testfolders_list) + + if printnodes: for x in range(0, count): print('[Node ' + str(x + 1).ljust(3) + '] [Port ' + str(NODE_LISTEN_PORT + x) + '] ' + LOCAL_RUN_FOLDER + @@ -105,12 +107,8 @@ def run_setup(numnodes, printnodes): keypairs.append(output.strip()) keypairs.sort() - ''' Only deals with one instance for now ''' - - patch_lookup_pubkey(LOCAL_FOLDER + "/constants_local.xml", keypairs, count) - patch_lookup_port(LOCAL_FOLDER + "/constants_local.xml", NODE_LISTEN_PORT, count) - patch_seed_pubkey(LOCAL_FOLDER + "/constants_local.xml", keypairs, count) - patch_seed_port(LOCAL_FOLDER + "/constants_local.xml", NODE_LISTEN_PORT, count) + patch_lookup_pubkey(LOCAL_FOLDER + "constants.xml", keypairs) + patch_seed_pubkey(LOCAL_FOLDER + "constants.xml", keypairs) nodes = ET.Element("nodes") @@ -118,15 +116,8 @@ def run_setup(numnodes, printnodes): keys_file = open(LOCAL_RUN_FOLDER + 'lookup_keys.txt', "w") for x in range(0, count): keys_file.write(keypairs[x] + '\n') - keypair = keypairs[x].split(" ") - if x < count: - peer = ET.SubElement(nodes, "peer") - ET.SubElement(peer, "pubk").text = keypair[0] - ET.SubElement(peer, "ip").text = '127.0.0.1' - ET.SubElement(peer, "port").text = str(NODE_LISTEN_PORT + x) keys_file.close() - def patch_constants_xml(filepath, read_txn=False): root = ET.parse(filepath).getroot() @@ -156,40 +147,36 @@ def run_gentxn(batch=100): os.system('cd ' + GENTXN_WORKING_DIR + '; ./gentxn --begin 0 --end {}'.format(batch)) -def patch_lookup_pubkey(filepath, keypairs, count): +def patch_lookup_pubkey(filepath, keypairs): root = ET.parse(filepath).getroot() td = root.find('lookups') - elems = td.findall('peer/pubkey') - for x in range(0, count): - keypair = keypairs[x].split(" ") - elems[x].text = keypair[0] + if td: + root.remove(td) + root.append(ET.Element('lookups')) + td = root.find('lookups') + p = ET.SubElement(td, "peer") + ET.SubElement(p, "pubkey").text = keypairs[0].split(" ")[0] + ET.SubElement(p, "ip").text = '127.0.0.1' + ET.SubElement(p, "port").text = str(NODE_LISTEN_PORT) + ET.SubElement(p, "hostname").text = None + tree = ET.ElementTree(root) tree.write(filepath) -def patch_seed_pubkey(filepath, keypairs, count): +def patch_seed_pubkey(filepath, keypairs): root = ET.parse(filepath).getroot() td = root.find('upper_seed') - elems = td.findall('peer/pubkey') - for x in range(0, count): - keypair = keypairs[x].split(" ") - elems[x].text = keypair[0] - tree = ET.ElementTree(root) - tree.write(filepath) -def patch_lookup_port(filepath, port, count): - root = ET.parse(filepath).getroot() - td = root.find('lookups') - elems = td.findall('peer/port') - for x in range(0, count): - elems[x].text = str(port) - tree = ET.ElementTree(root) - tree.write(filepath) -def patch_seed_port(filepath, port, count): - root = ET.parse(filepath).getroot() + if td: + root.remove(td) + root.append(ET.Element('upper_seed')) td = root.find('upper_seed') - elems = td.findall('peer/port') - for x in range(0, count): - elems[x].text = str(port) + p = ET.SubElement(td, "peer") + ET.SubElement(p, "pubkey").text = keypairs[0].split(" ")[0] + ET.SubElement(p, "ip").text = '127.0.0.1' + ET.SubElement(p, "port").text = str(NODE_LISTEN_PORT) + ET.SubElement(p, "hostname").text = None + tree = ET.ElementTree(root) tree.write(filepath) @@ -202,14 +189,13 @@ def run_start(): print("DEV_TREE_ROOT is not set") return - dev_root += "/Zilliqa" - fp = dev_root + "/" + "constants_local.xml.native" + + fp = LOCAL_FOLDER + "constants.xml" if not os.path.exists(fp): print( fp +" not found") return - # Load the keypairs keypairs = [] with open(LOCAL_RUN_FOLDER + 'lookup_keys.txt', "r") as f: @@ -230,6 +216,8 @@ def run_start(): # Launch node zilliqa process for x in range(0, count): keypair = keypairs[x].split(" ") + start_str = ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str(NODE_LISTEN_PORT + x) + ' --identity ' + 'lookup-' + str(x) + print(start_str) os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[ 1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/lzilliqa ' + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + diff --git a/tests/Zilliqa/test_zilliqa_seedpub.py b/tests/Zilliqa/test_zilliqa_seedpub.py index 123a7057e5..31f852194d 100755 --- a/tests/Zilliqa/test_zilliqa_seedpub.py +++ b/tests/Zilliqa/test_zilliqa_seedpub.py @@ -176,7 +176,7 @@ def run_start(): return dev_root += "/Zilliqa" - fp = dev_root + "/" + "constants_local.xml.native" + fp = dev_root + "/" + "constants.xml" if not os.path.exists(fp): print( fp +" not found") diff --git a/tests/native/prepare_constants.py b/tests/native/prepare_constants.py index 2cc2869f8d..9d83961f50 100644 --- a/tests/native/prepare_constants.py +++ b/tests/native/prepare_constants.py @@ -29,35 +29,49 @@ def update_xml_files(source_file, target_file): if not DEV_TREE_ROOT: print("DEV_TREE_ROOT is not set") os.abort() + else: + print("DEV_TREE_ROOT: " + DEV_TREE_ROOT) general = root.find('general') if general: general.find('LOOKUP_NODE_MODE').text = 'false' general.find('DEBUG_LEVEL').text = '4' + else: + print("Failed to find general") scilla_root = root.find('smart_contract') if scilla_root: scilla_root.find('SCILLA_ROOT').text = DEV_TREE_ROOT +'/scilla' scilla_root.find('ENABLE_SCILLA_MULTI_VERSION').text = 'false' - + else: + print("Failed to find smart_contract") jsonrpc = root.find('jsonrpc') if jsonrpc: jsonrpc.find('EVM_SERVER_BINARY').text = DEV_TREE_ROOT + '/Zilliqa/evm-ds/target/release/evm-ds' jsonrpc.find('ENABLE_STATUS_RPC').text = 'false' + jsonrpc.find('LOOKUP_RPC_PORT').text = "4201" + else: + print("Failed to find jsonrpc") metric = root.find('metric/zilliqa') if metric: metric.find('METRIC_ZILLIQA_PROVIDER').text = 'NONE' metric.find('METRIC_ZILLIQA_MASK').text = 'NONE' + else: + print("Failed to find metric") trace = root.find('trace/zilliqa') if trace: trace.find('TRACE_ZILLIQA_PROVIDER').text = 'NONE' trace.find('TRACE_ZILLIQA_MASK').text = 'NONE' + else: + print("Failed to find trace") logging = root.find('logging/zilliqa') if logging: logging.find('LOGGING_ZILLIQA_PROVIDER').text = 'NONE' + else: + print("Failed to find logging") tree = ET.ElementTree(root) tree.write(target_file) diff --git a/tests/native/start_native.sh b/tests/native/start_native.sh index 949e32b98c..a3b5e59bca 100755 --- a/tests/native/start_native.sh +++ b/tests/native/start_native.sh @@ -24,7 +24,6 @@ echo "DEV_TREE_ROOT: $DEV_TREE_ROOT" # takes a copy of constants.xml and constants_local.xml and prepares new versions with .native extension -python ./tests/native/prepare_constants.py $DEV_TREE_ROOT/Zilliqa/constants.xml $DEV_TREE_ROOT/Zilliqa/constants.xml.native -python ./tests/native/prepare_constants.py $DEV_TREE_ROOT/Zilliqa/constants_local.xml $DEV_TREE_ROOT/Zilliqa/constants_local.xml.native +python ./tests/native/prepare_constants.py constants.xml constants.xml ./tests/Node/pre_run.sh && ./tests/Node/test_node_lookup.sh && ./tests/Node/test_node_simple.sh