From d843d7c2441e7821630255da2d3767a584ad8f30 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 19 Sep 2023 14:33:42 +0200 Subject: [PATCH] feat: add custom validator count per node (#122) Working: ```json { "participants": [ { "el_client_type": "geth", "cl_client_type": "teku" }, { "el_client_type": "geth", "cl_client_type": "teku", "validator_count": 1 }, { "el_client_type": "geth", "cl_client_type": "lodestar", "validator_count": 1 }, { "el_client_type": "geth", "cl_client_type": "nimbus", "validator_count": 1 }, { "el_client_type": "geth", "cl_client_type": "prysm", "validator_count": 1 }, { "el_client_type": "geth", "cl_client_type": "lighthouse", "validator_count": 1 } ], "network_params": { }, "global_client_log_level": "info", "snooper_enabled": true } ``` Not yet working: ```json { "participants": [ { "el_client_type": "geth", "cl_client_type": "teku" }, { "el_client_type": "geth", "cl_client_type": "teku", "validator_count": 0 }, { "el_client_type": "geth", "cl_client_type": "lodestar", "validator_count": 0 }, { "el_client_type": "geth", "cl_client_type": "nimbus", "validator_count": 0 }, { "el_client_type": "geth", "cl_client_type": "prysm", "validator_count": 0 }, { "el_client_type": "geth", "cl_client_type": "lighthouse", "validator_count": 0 } ], "network_params": { }, "global_client_log_level": "info", "snooper_enabled": true } ``` @h4ck3rk3y @leeederek could you let me know how could I pass a null artifact when there are no validator keys? --------- Co-authored-by: Leandro Poroli Co-authored-by: Gyanendra Mishra --- README.md | 4 ++ package_io/constants.star | 2 +- package_io/input_parser.star | 10 +++ src/cl/lighthouse/lighthouse_launcher.star | 59 +++++++-------- src/cl/lodestar/lodestar_launcher.star | 48 ++++++------- src/cl/nimbus/nimbus_launcher.star | 45 ++++++++---- src/cl/prysm/prysm_launcher.star | 72 +++++++++---------- src/cl/teku/teku_launcher.star | 44 ++++++++---- src/participant_network.star | 15 ++-- .../cl_validator_keystore_generator.star | 47 +++++++----- 10 files changed, 206 insertions(+), 140 deletions(-) diff --git a/README.md b/README.md index 2fefe1f..eb203db 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,10 @@ these and other parameters are configurable through a json file Read more about "v_min_mem": 0, "v_max_mem": 0, + // Validator count can override the default number of validators per node + // Defaults are set by num_validator_keys_per_node + "validator_count": null, + // The number of times this participant should be repeated // defaults to 1(i.e no repetition). This is optional. "count": 1 diff --git a/package_io/constants.star b/package_io/constants.star index 1b4699b..c4a9d57 100644 --- a/package_io/constants.star +++ b/package_io/constants.star @@ -31,4 +31,4 @@ GENESIS_VALIDATORS_ROOT_PLACEHOLDER = "GENESIS_VALIDATORS_ROOT_PLACEHOLDER" DEFAULT_SNOOPER_IMAGE = "parithoshj/json_rpc_snoop:v1.0.0-x86" -ARCHIVE_MODE = True +ARCHIVE_MODE = True \ No newline at end of file diff --git a/package_io/input_parser.star b/package_io/input_parser.star index 4dc9df2..6ef6691 100644 --- a/package_io/input_parser.star +++ b/package_io/input_parser.star @@ -50,6 +50,7 @@ def get_args_with_default_values(args): v_max_cpu=participant["v_max_cpu"], v_min_mem=participant["v_min_mem"], v_max_mem=participant["v_max_mem"], + validator_count=participant["validator_count"], snooper_enabled = participant["snooper_enabled"], count=participant["count"] ) for participant in result["participants"]], @@ -96,6 +97,7 @@ def parse_input(input_args): result["participants"] = participants total_participant_count = 0 + actual_num_validators = 0 # validation of the above defaults for index, participant in enumerate(result["participants"]): el_client_type = participant["el_client_type"] @@ -123,6 +125,13 @@ def parse_input(input_args): if default_snooper_enabled: participant["snooper_enabled"] = default_snooper_enabled + validator_count = participant["validator_count"] + if validator_count == None: + default_validator_count = result["network_params"]["num_validator_keys_per_node"] + participant["validator_count"] = default_validator_count + + actual_num_validators += participant["validator_count"] + beacon_extra_params = participant.get("beacon_extra_params", []) participant["beacon_extra_params"] = beacon_extra_params @@ -228,6 +237,7 @@ def default_participant(): "v_max_cpu": 0, "v_min_mem": 0, "v_max_mem": 0, + "validator_count": None, "snooper_enabled": False, "count": 1 } diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star index f95b58a..8eb324d 100644 --- a/src/cl/lighthouse/lighthouse_launcher.star +++ b/src/cl/lighthouse/lighthouse_launcher.star @@ -105,11 +105,6 @@ def launch( bn_min_mem = int(bn_min_mem) if int(bn_min_mem) > 0 else BEACON_MIN_MEMORY bn_max_mem = int(bn_max_mem) if int(bn_max_mem) > 0 else BEACON_MAX_MEMORY - v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU - v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU - v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY - v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY - # Launch Beacon node beacon_config = get_beacon_config( launcher.genesis_data, @@ -127,26 +122,31 @@ def launch( ) beacon_service = plan.add_service(beacon_node_service_name, beacon_config) - beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID] - - # Launch validator node beacon_http_url = "http://{0}:{1}".format(beacon_service.ip_address, beacon_http_port.number) - validator_config = get_validator_config( - launcher.genesis_data, - image, - log_level, - beacon_http_url, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_validator_params, - ) - - validator_service = plan.add_service(validator_node_service_name, validator_config) + # Launch validator node if we have a keystore + validator_service = None + if node_keystore_files != None: + v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU + v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU + v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY + v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY + + validator_config = get_validator_config( + launcher.genesis_data, + image, + log_level, + beacon_http_url, + node_keystore_files, + v_min_cpu, + v_max_cpu, + v_min_mem, + v_max_mem, + extra_validator_params, + ) + + validator_service = plan.add_service(validator_node_service_name, validator_config) # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( @@ -165,13 +165,14 @@ def launch( beacon_metrics_port = beacon_service.ports[BEACON_METRICS_PORT_ID] beacon_metrics_url = "{0}:{1}".format(beacon_service.ip_address, beacon_metrics_port.number) - - validator_metrics_port = validator_service.ports[VALIDATOR_METRICS_PORT_ID] - validator_metrics_url = "{0}:{1}".format(validator_service.ip_address, validator_metrics_port.number) - beacon_node_metrics_info = cl_node_metrics.new_cl_node_metrics_info(beacon_node_service_name, METRICS_PATH, beacon_metrics_url) - validator_node_metrics_info = cl_node_metrics.new_cl_node_metrics_info(validator_node_service_name, METRICS_PATH, validator_metrics_url) - nodes_metrics_info = [beacon_node_metrics_info, validator_node_metrics_info] + nodes_metrics_info = [beacon_node_metrics_info] + + if validator_service: + validator_metrics_port = validator_service.ports[VALIDATOR_METRICS_PORT_ID] + validator_metrics_url = "{0}:{1}".format(validator_service.ip_address, validator_metrics_port.number) + validator_node_metrics_info = cl_node_metrics.new_cl_node_metrics_info(validator_node_service_name, METRICS_PATH, validator_metrics_url) + nodes_metrics_info.append(validator_node_metrics_info) return cl_client_context.new_cl_client_context( "lighthouse", @@ -316,6 +317,7 @@ def get_validator_config( # For some reason, Lighthouse takes in the parent directory of the config file (rather than the path to the config file itself) genesis_config_parent_dirpath_on_client = shared_utils.path_join(GENESIS_DATA_MOUNTPOINT_ON_CLIENTS, shared_utils.path_dir(genesis_data.config_yml_rel_filepath)) + validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, node_keystore_files.raw_keys_relative_dirpath) validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, node_keystore_files.raw_secrets_relative_dirpath) @@ -348,7 +350,6 @@ def get_validator_config( if len(extra_params): cmd.extend([param for param in extra_params]) - return ServiceConfig( image = image, ports = VALIDATOR_USED_PORTS, diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star index 42eb006..f0d9938 100644 --- a/src/cl/lodestar/lodestar_launcher.star +++ b/src/cl/lodestar/lodestar_launcher.star @@ -87,7 +87,6 @@ def launch( beacon_node_service_name = "{0}".format(service_name) validator_node_service_name = "{0}-{1}".format(service_name, VALIDATOR_SUFFIX_SERVICE_NAME) - log_level = input_parser.get_client_log_level_or_default(participant_log_level, global_log_level, LODESTAR_LOG_LEVELS) bn_min_cpu = int(bn_min_cpu) if int(bn_min_cpu) > 0 else BEACON_MIN_CPU @@ -95,12 +94,6 @@ def launch( bn_min_mem = int(bn_min_mem) if int(bn_min_mem) > 0 else BEACON_MIN_MEMORY bn_max_mem = int(bn_max_mem) if int(bn_max_mem) > 0 else BEACON_MAX_MEMORY - v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU - v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU - v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY - v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY - - # Launch Beacon node beacon_config = get_beacon_config( launcher.cl_genesis_data, @@ -121,25 +114,29 @@ def launch( beacon_http_port = beacon_service.ports[HTTP_PORT_ID] - - # Launch validator node beacon_http_url = "http://{0}:{1}".format(beacon_service.ip_address, beacon_http_port.number) - validator_config = get_validator_config( - validator_node_service_name, - launcher.cl_genesis_data, - image, - log_level, - beacon_http_url, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_validator_params, - ) - - validator_service = plan.add_service(validator_node_service_name, validator_config) + # Launch validator node if we have a keystore + if node_keystore_files != None: + v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU + v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU + v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY + v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY + validator_config = get_validator_config( + validator_node_service_name, + launcher.cl_genesis_data, + image, + log_level, + beacon_http_url, + node_keystore_files, + v_min_cpu, + v_max_cpu, + v_min_mem, + v_max_mem, + extra_validator_params, + ) + + plan.add_service(validator_node_service_name, validator_config) # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module @@ -283,9 +280,11 @@ def get_validator_config( root_dirpath = shared_utils.path_join(CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, service_name) genesis_config_filepath = shared_utils.path_join(GENESIS_DATA_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, genesis_data.config_yml_rel_filepath) + validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.raw_keys_relative_dirpath) validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.raw_secrets_relative_dirpath) + cmd = [ "validator", "--logLevel=" + log_level, @@ -306,7 +305,6 @@ def get_validator_config( # this is a repeated, we convert it into Starlark cmd.extend([param for param in extra_params]) - return ServiceConfig( image = image, ports = VALIDATOR_USED_PORTS, diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star index fc5c12b..d2edf17 100644 --- a/src/cl/nimbus/nimbus_launcher.star +++ b/src/cl/nimbus/nimbus_launcher.star @@ -187,15 +187,20 @@ def get_config( # For some reason, Nimbus takes in the parent directory of the config file (rather than the path to the config file itself) genesis_config_parent_dirpath_on_client = shared_utils.path_join(GENESIS_DATA_MOUNTPOINT_ON_CLIENT, shared_utils.path_dir(genesis_data.config_yml_rel_filepath)) jwt_secret_filepath = shared_utils.path_join(GENESIS_DATA_MOUNTPOINT_ON_CLIENT, genesis_data.jwt_secret_rel_filepath) - validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT, node_keystore_files.nimbus_keys_relative_dirpath) - validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT, node_keystore_files.raw_secrets_relative_dirpath) + + + validator_keys_dirpath = "" + validator_secrets_dirpath = "" + if node_keystore_files != None: + validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT, node_keystore_files.nimbus_keys_relative_dirpath) + validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT, node_keystore_files.raw_secrets_relative_dirpath) # Sources for these flags: # 1) https://github.com/status-im/nimbus-eth2/blob/stable/scripts/launch_local_testnet.sh # 2) https://github.com/status-im/nimbus-eth2/blob/67ab477a27e358d605e99bffeb67f98d18218eca/scripts/launch_local_testnet.sh#L417 # WARNING: Do NOT set the --max-peers flag here, as doing so to the exact number of nodes seems to mess things up! # See: https://github.com/kurtosis-tech/eth2-merge-kurtosis-module/issues/26 - cmd = [ + validator_copy = [ "mkdir", CONSENSUS_DATA_DIRPATH_IN_SERVICE_CONTAINER, "-m", @@ -218,6 +223,15 @@ def get_config( "600", VALIDATOR_SECRETS_DIRPATH_ON_SERVICE_CONTAINER + "/*", "&&", + ] + + validator_flags = [ + "--validators-dir=" + VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + "--secrets-dir=" + VALIDATOR_SECRETS_DIRPATH_ON_SERVICE_CONTAINER, + "--suggested-fee-recipient=" + package_io.VALIDATING_REWARDS_ACCOUNT, + ] + + beacon_start = [ DEFAULT_IMAGE_ENTRYPOINT, "--non-interactive=true", "--log-level=" + log_level, @@ -233,9 +247,6 @@ def get_config( "--rest-address=0.0.0.0", "--rest-allow-origin=*", "--rest-port={0}".format(HTTP_PORT_NUM), - "--validators-dir=" + VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - "--secrets-dir=" + VALIDATOR_SECRETS_DIRPATH_ON_SERVICE_CONTAINER, - "--suggested-fee-recipient=" + package_io.VALIDATING_REWARDS_ACCOUNT, # There's a bug where if we don't set this flag, the Nimbus nodes won't work: # https://discord.com/channels/641364059387854899/674288681737256970/922890280120750170 # https://github.com/status-im/nimbus-eth2/issues/2451 @@ -251,6 +262,16 @@ def get_config( "--metrics-port={0}".format(METRICS_PORT_NUM), # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^ ] + + # Depending on whether we're using a node keystore, we'll need to add the validator flags + cmd = [] + if node_keystore_files != None: + cmd.extend(validator_copy) + cmd.extend(beacon_start) + cmd.extend(validator_flags) + else: + cmd.extend(beacon_start) + if bootnode_contexts == None: # Copied from https://github.com/status-im/nimbus-eth2/blob/67ab477a27e358d605e99bffeb67f98d18218eca/scripts/launch_local_testnet.sh#L417 # See explanation there @@ -263,17 +284,18 @@ def get_config( if len(extra_params) > 0: cmd.extend([param for param in extra_params]) + files = { + GENESIS_DATA_MOUNTPOINT_ON_CLIENT: genesis_data.files_artifact_uuid, + } + if node_keystore_files: + files[VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT] = node_keystore_files.files_artifact_uuid cmd_str = " ".join(cmd) - return ServiceConfig( image = image, ports = USED_PORTS, cmd = [cmd_str], entrypoint = ENTRYPOINT_ARGS, - files = { - GENESIS_DATA_MOUNTPOINT_ON_CLIENT: genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT: node_keystore_files.files_artifact_uuid - }, + files = files, private_ip_address_placeholder = PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions = cl_node_ready_conditions.get_ready_conditions(HTTP_PORT_ID), min_cpu = bn_min_cpu, @@ -282,7 +304,6 @@ def get_config( max_memory = bn_max_mem ) - def new_nimbus_launcher(cl_genesis_data): return struct( cl_genesis_data = cl_genesis_data, diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star index 7c51c80..049413f 100644 --- a/src/cl/prysm/prysm_launcher.star +++ b/src/cl/prysm/prysm_launcher.star @@ -112,7 +112,6 @@ def launch( beacon_node_service_name = "{0}".format(service_name) validator_node_service_name = "{0}-{1}".format(service_name, VALIDATOR_SUFFIX_SERVICE_NAME) - log_level = input_parser.get_client_log_level_or_default(participant_log_level, global_log_level, PRYSM_LOG_LEVELS) bn_min_cpu = int(bn_min_cpu) if int(bn_min_cpu) > 0 else BEACON_MIN_CPU @@ -120,11 +119,6 @@ def launch( bn_min_mem = int(bn_min_mem) if int(bn_min_mem) > 0 else BEACON_MIN_MEMORY bn_max_mem = int(bn_max_mem) if int(bn_max_mem) > 0 else BEACON_MAX_MEMORY - v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU - v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU - v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY - v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY - beacon_config = get_beacon_config( launcher.genesis_data, beacon_image, @@ -144,28 +138,34 @@ def launch( beacon_http_port = beacon_service.ports[HTTP_PORT_ID] - # Launch validator node beacon_http_endpoint = "{0}:{1}".format(beacon_service.ip_address, HTTP_PORT_NUM) beacon_rpc_endpoint = "{0}:{1}".format(beacon_service.ip_address, RPC_PORT_NUM) - validator_config = get_validator_config( - launcher.genesis_data, - validator_image, - validator_node_service_name, - log_level, - beacon_rpc_endpoint, - beacon_http_endpoint, - node_keystore_files, - v_min_cpu, - v_max_cpu, - v_min_mem, - v_max_mem, - extra_validator_params, - launcher.prysm_password_relative_filepath, - launcher.prysm_password_artifact_uuid - ) - - validator_service = plan.add_service(validator_node_service_name, validator_config) + # Launch validator node if we have a keystore file + validator_service = None + if node_keystore_files != None: + v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU + v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU + v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY + v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY + validator_config = get_validator_config( + launcher.genesis_data, + validator_image, + validator_node_service_name, + log_level, + beacon_rpc_endpoint, + beacon_http_endpoint, + node_keystore_files, + v_min_cpu, + v_max_cpu, + v_min_mem, + v_max_mem, + extra_validator_params, + launcher.prysm_password_relative_filepath, + launcher.prysm_password_artifact_uuid + ) + + validator_service = plan.add_service(validator_node_service_name, validator_config) # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( @@ -184,14 +184,14 @@ def launch( beacon_metrics_port = beacon_service.ports[BEACON_MONITORING_PORT_ID] beacon_metrics_url = "{0}:{1}".format(beacon_service.ip_address, beacon_metrics_port.number) - - validator_metrics_port = validator_service.ports[VALIDATOR_MONITORING_PORT_ID] - validator_metrics_url = "{0}:{1}".format(validator_service.ip_address, validator_metrics_port.number) - beacon_node_metrics_info = cl_node_metrics.new_cl_node_metrics_info(beacon_node_service_name, METRICS_PATH, beacon_metrics_url) - validator_node_metrics_info = cl_node_metrics.new_cl_node_metrics_info(validator_node_service_name, METRICS_PATH, validator_metrics_url) - nodes_metrics_info = [beacon_node_metrics_info, validator_node_metrics_info] + nodes_metrics_info = [beacon_node_metrics_info] + if validator_service: + validator_metrics_port = validator_service.ports[VALIDATOR_MONITORING_PORT_ID] + validator_metrics_url = "{0}:{1}".format(validator_service.ip_address, validator_metrics_port.number) + validator_node_metrics_info = cl_node_metrics.new_cl_node_metrics_info(validator_node_service_name, METRICS_PATH, validator_metrics_url) + nodes_metrics_info.append(validator_node_metrics_info) return cl_client_context.new_cl_client_context( "prysm", @@ -312,17 +312,18 @@ def get_validator_config( ): consensus_data_dirpath = shared_utils.path_join(CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, service_name) - prysm_keystore_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.prysm_relative_dirpath) - prysm_password_filepath = shared_utils.path_join(PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, prysm_password_relative_filepath) genesis_config_filepath = shared_utils.path_join(GENESIS_DATA_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, genesis_data.config_yml_rel_filepath) + validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.prysm_relative_dirpath) + validator_secrets_dirpath = shared_utils.path_join(PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, prysm_password_relative_filepath) + cmd = [ "--accept-terms-of-use=true",#it's mandatory in order to run the node "--chain-config-file=" + genesis_config_filepath, "--beacon-rpc-gateway-provider=" + beacon_http_endpoint, "--beacon-rpc-provider=" + beacon_rpc_endpoint, - "--wallet-dir=" + prysm_keystore_dirpath, - "--wallet-password-file=" + prysm_password_filepath, + "--wallet-dir=" + validator_keys_dirpath, + "--wallet-password-file=" + validator_secrets_dirpath, "--datadir=" + consensus_data_dirpath, "--monitoring-port={0}".format(VALIDATOR_MONITORING_PORT_NUM), "--verbosity=" + log_level, @@ -339,7 +340,6 @@ def get_validator_config( # we do the for loop as otherwise its a proto repeated array cmd.extend([param for param in extra_params]) - return ServiceConfig( image = validator_image, ports = VALIDATOR_NODE_USED_PORTS, diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star index 3b6e063..df60f45 100644 --- a/src/cl/teku/teku_launcher.star +++ b/src/cl/teku/teku_launcher.star @@ -188,10 +188,14 @@ def get_config( genesis_config_filepath = shared_utils.path_join(GENESIS_DATA_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, genesis_data.config_yml_rel_filepath) genesis_ssz_filepath = shared_utils.path_join(GENESIS_DATA_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, genesis_data.genesis_ssz_rel_filepath) jwt_secret_filepath = shared_utils.path_join(GENESIS_DATA_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, genesis_data.jwt_secret_rel_filepath) - validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.teku_keys_relative_dirpath) - validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.teku_secrets_relative_dirpath) - cmd = [ + validator_keys_dirpath = "" + validator_secrets_dirpath = "" + if node_keystore_files: + validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.teku_keys_relative_dirpath) + validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.teku_secrets_relative_dirpath) + + validator_copy = [ # Needed because the generated keys are owned by root and the Teku image runs as the 'teku' user "cp", "-R", @@ -204,6 +208,15 @@ def get_config( validator_secrets_dirpath, DEST_VALIDATOR_SECRETS_DIRPATH_IN_SERVICE_CONTAINER, "&&", + ] + validator_flags = [ + "--validator-keys={0}:{1}".format( + DEST_VALIDATOR_KEYS_DIRPATH_IN_SERVICE_CONTAINER, + DEST_VALIDATOR_SECRETS_DIRPATH_IN_SERVICE_CONTAINER, + ), + "--validators-proposer-default-fee-recipient=" + package_io.VALIDATING_REWARDS_ACCOUNT, + ] + beacon_start = [ TEKU_BINARY_FILEPATH_IN_IMAGE, "--logging=" + log_level, "--log-destination=CONSOLE", @@ -223,13 +236,8 @@ def get_config( "--rest-api-port={0}".format(HTTP_PORT_NUM), "--rest-api-host-allowlist=*", "--data-storage-non-canonical-blocks-enabled=true", - "--validator-keys={0}:{1}".format( - DEST_VALIDATOR_KEYS_DIRPATH_IN_SERVICE_CONTAINER, - DEST_VALIDATOR_SECRETS_DIRPATH_IN_SERVICE_CONTAINER, - ), "--ee-jwt-secret-file={0}".format(jwt_secret_filepath), "--ee-endpoint=" + EXECUTION_ENGINE_ENDPOINT, - "--validators-proposer-default-fee-recipient=" + package_io.VALIDATING_REWARDS_ACCOUNT, # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv "--metrics-enabled", "--metrics-interface=0.0.0.0", @@ -239,6 +247,15 @@ def get_config( # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^ ] + # Depending on whether we're using a node keystore, we'll need to add the validator flags + cmd = [] + if node_keystore_files != None: + cmd.extend(validator_copy) + cmd.extend(beacon_start) + cmd.extend(validator_flags) + else: + cmd.extend(beacon_start) + if bootnode_contexts != None: cmd.append("--p2p-discovery-bootnodes="+",".join([ctx.enr for ctx in bootnode_contexts[:package_io.MAX_ENR_ENTRIES]])) cmd.append("--p2p-static-peers="+",".join([ctx.multiaddr for ctx in bootnode_contexts[:package_io.MAX_ENR_ENTRIES]])) @@ -247,17 +264,18 @@ def get_config( # we do the list comprehension as the default extra_params is a proto repeated string cmd.extend([param for param in extra_params]) + files = { + GENESIS_DATA_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: genesis_data.files_artifact_uuid, + } + if node_keystore_files: + files[VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER] = node_keystore_files.files_artifact_uuid cmd_str = " ".join(cmd) - return ServiceConfig( image = image, ports = USED_PORTS, cmd = [cmd_str], entrypoint = ENTRYPOINT_ARGS, - files = { - GENESIS_DATA_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, - }, + files = files, private_ip_address_placeholder = PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions = cl_node_ready_conditions.get_ready_conditions(HTTP_PORT_ID), min_cpu = bn_min_cpu, diff --git a/src/participant_network.star b/src/participant_network.star index 19f0911..eaa7dc8 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -52,15 +52,13 @@ def launch_participant_network(plan, participants, network_params, global_log_le cl_validator_data = cl_validator_keystores.generate_cl_validator_keystores( plan, network_params.preregistered_validator_keys_mnemonic, - participants, - network_params.num_validator_keys_per_node + participants ) else: cl_validator_data = cl_validator_keystores.generate_cl_valdiator_keystores_in_parallel( plan, network_params.preregistered_validator_keys_mnemonic, - participants, - network_params.num_validator_keys_per_node + participants ) plan.print(json.indent(json.encode(cl_validator_data))) @@ -94,7 +92,9 @@ def launch_participant_network(plan, participants, network_params, global_log_le genesis_generation_config_yml_template = read_file(static_files.CL_GENESIS_GENERATION_CONFIG_TEMPLATE_FILEPATH) genesis_generation_mnemonics_yml_template = read_file(static_files.CL_GENESIS_GENERATION_MNEMONICS_TEMPLATE_FILEPATH) - total_number_of_validator_keys = network_params.num_validator_keys_per_node * num_participants + total_number_of_validator_keys = 0 + for participant in participants: + total_number_of_validator_keys += participant.validator_count cl_genesis_data = cl_genesis_data_generator.generate_cl_genesis_data( plan, genesis_generation_config_yml_template, @@ -187,8 +187,9 @@ def launch_participant_network(plan, participants, network_params, global_log_le index_str = zfill_custom(index+1, zfill_calculator(participants)) cl_service_name = "cl-{0}-{1}-{2}".format(index_str, cl_client_type, el_client_type) - - new_cl_node_validator_keystores = preregistered_validator_keys_for_nodes[index] + new_cl_node_validator_keystores = None + if participant.validator_count != 0: + new_cl_node_validator_keystores = preregistered_validator_keys_for_nodes[index] el_client_context = all_el_client_contexts[index] diff --git a/src/prelaunch_data_generator/cl_validator_keystores/cl_validator_keystore_generator.star b/src/prelaunch_data_generator/cl_validator_keystores/cl_validator_keystore_generator.star index 0f865d9..17dcf84 100644 --- a/src/prelaunch_data_generator/cl_validator_keystores/cl_validator_keystore_generator.star +++ b/src/prelaunch_data_generator/cl_validator_keystores/cl_validator_keystore_generator.star @@ -33,8 +33,7 @@ KEYSTORE_GENERATION_FINISHED_FILEPATH_FORMAT = "/tmp/keystores_generated-{0}-{1} def generate_cl_validator_keystores( plan, mnemonic, - participants, - num_validators_per_node): + participants): service_name = prelaunch_data_generator_launcher.launch_prelaunch_data_generator( plan, @@ -46,12 +45,15 @@ def generate_cl_validator_keystores( all_output_dirpaths = [] all_sub_command_strs = [] - + running_total_validator_count = 0 for idx, participant in enumerate(participants): output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx) - - start_index = idx * num_validators_per_node - stop_index = (idx+1) * num_validators_per_node + if participant.validator_count == 0: + all_output_dirpaths.append(output_dirpath) + continue + start_index = running_total_validator_count + running_total_validator_count += participant.validator_count + stop_index = (start_index + participant.validator_count) generate_keystores_cmd = "{0} keystores --insecure --prysm-pass {1} --out-loc {2} --source-mnemonic \"{3}\" --source-min {4} --source-max {5}".format( KEYSTORES_GENERATION_TOOL_NAME, @@ -72,12 +74,16 @@ def generate_cl_validator_keystores( # Store outputs into files artifacts keystore_files = [] + running_total_validator_count = 0 for idx, participant in enumerate(participants): output_dirpath = all_output_dirpaths[idx] - + if participant.validator_count == 0: + keystore_files.append(None) + continue padded_idx = zfill_custom(idx+1, len(str(len(participants)))) - keystore_start_index = idx * num_validators_per_node - keystore_stop_index = (idx+1) * num_validators_per_node - 1 + keystore_start_index = running_total_validator_count + running_total_validator_count += participant.validator_count + keystore_stop_index = (keystore_start_index + participant.validator_count) - 1 artifact_name = "{0}-{1}-{2}-{3}-{4}".format( padded_idx, participant.cl_client_type, @@ -130,8 +136,7 @@ def generate_cl_validator_keystores( def generate_cl_valdiator_keystores_in_parallel( plan, mnemonic, - participants, - num_validators_per_node): + participants): service_names = prelaunch_data_generator_launcher.launch_prelaunch_data_generator_parallel( plan, @@ -143,12 +148,14 @@ def generate_cl_valdiator_keystores_in_parallel( all_output_dirpaths = [] all_generation_commands = [] finished_files_to_verify = [] - + running_total_validator_count = 0 for idx, participant in enumerate(participants): output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx) - - start_index = idx * num_validators_per_node - stop_index = (idx+1) * num_validators_per_node + if participant.validator_count == 0: + all_output_dirpaths.append(output_dirpath) + continue + start_index = idx * participant.validator_count + stop_index = (idx+1) * participant.validator_count generation_finished_filepath = KEYSTORE_GENERATION_FINISHED_FILEPATH_FORMAT.format(start_index,stop_index) finished_files_to_verify.append(generation_finished_filepath) @@ -181,13 +188,19 @@ def generate_cl_valdiator_keystores_in_parallel( # Store outputs into files artifacts keystore_files = [] + running_total_validator_count = 0 for idx, participant in enumerate(participants): + if participant.validator_count == 0: + keystore_files.append(None) + continue service_name = service_names[idx] output_dirpath = all_output_dirpaths[idx] + running_total_validator_count += participant.validator_count padded_idx = zfill_custom(idx+1, len(str(len(participants)))) - keystore_start_index = idx * num_validators_per_node - keystore_stop_index = (idx+1) * num_validators_per_node - 1 + keystore_start_index = running_total_validator_count + running_total_validator_count += participant.validator_count + keystore_stop_index = (keystore_start_index + participant.validator_count) - 1 artifact_name = "{0}-{1}-{2}-{3}-{4}".format( padded_idx, participant.cl_client_type,