Skip to content
This repository has been archived by the owner on Sep 27, 2023. It is now read-only.

Commit

Permalink
feat: add custom validator count per node (#122)
Browse files Browse the repository at this point in the history 
Working:
```json
{
  "participants": [
    {
      "el_client_type": "geth",
      "cl_client_type": "teku"
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "teku",
      "validator_count": 1
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "lodestar",
      "validator_count": 1
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "nimbus",
      "validator_count": 1
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "prysm",
      "validator_count": 1
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "lighthouse",
      "validator_count": 1
    }
  ],
  "network_params": {
  },
  "global_client_log_level": "info",
  "snooper_enabled": true
}
```

Not yet working:
```json
{
  "participants": [
    {
      "el_client_type": "geth",
      "cl_client_type": "teku"
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "teku",
      "validator_count": 0
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "lodestar",
      "validator_count": 0
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "nimbus",
      "validator_count": 0
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "prysm",
      "validator_count": 0
    },
    {
      "el_client_type": "geth",
      "cl_client_type": "lighthouse",
      "validator_count": 0
    }
  ],
  "network_params": {
  },
  "global_client_log_level": "info",
  "snooper_enabled": true
}
```

@h4ck3rk3y @leeederek could you let me know how could I pass a null
artifact when there are no validator keys?

---------

Co-authored-by: Leandro Poroli <leandro.poroli@kurtosistech.com>
Co-authored-by: Gyanendra Mishra <anomaly.the@gmail.com>
  • Loading branch information
@barnabasbusa @leoporoli @h4ck3rk3y
3 people committed Sep 19, 2023
1 parent c9a736e commit d843d7c
Show file tree
Hide file tree
Showing 10 changed files with 206 additions and 140 deletions.
4 changes: 4 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,10 @@ these and other parameters are configurable through a json file Read more about
"v_min_mem": 0,
"v_max_mem": 0,

// Validator count can override the default number of validators per node
// Defaults are set by num_validator_keys_per_node
"validator_count": null,

// The number of times this participant should be repeated
// defaults to 1(i.e no repetition). This is optional.
"count": 1
Expand Down
2 changes: 1 addition & 1 deletion package_io/constants.star
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,4 @@ GENESIS_VALIDATORS_ROOT_PLACEHOLDER = "GENESIS_VALIDATORS_ROOT_PLACEHOLDER"

DEFAULT_SNOOPER_IMAGE = "parithoshj/json_rpc_snoop:v1.0.0-x86"

ARCHIVE_MODE = True
ARCHIVE_MODE = True
10 changes: 10 additions & 0 deletions package_io/input_parser.star
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ def get_args_with_default_values(args):
v_max_cpu=participant["v_max_cpu"],
v_min_mem=participant["v_min_mem"],
v_max_mem=participant["v_max_mem"],
validator_count=participant["validator_count"],
snooper_enabled = participant["snooper_enabled"],
count=participant["count"]
) for participant in result["participants"]],
Expand Down Expand Up @@ -96,6 +97,7 @@ def parse_input(input_args):
result["participants"] = participants

total_participant_count = 0
actual_num_validators = 0
# validation of the above defaults
for index, participant in enumerate(result["participants"]):
el_client_type = participant["el_client_type"]
Expand Down Expand Up @@ -123,6 +125,13 @@ def parse_input(input_args):
if default_snooper_enabled:
participant["snooper_enabled"] = default_snooper_enabled

validator_count = participant["validator_count"]
if validator_count == None:
default_validator_count = result["network_params"]["num_validator_keys_per_node"]
participant["validator_count"] = default_validator_count

actual_num_validators += participant["validator_count"]

beacon_extra_params = participant.get("beacon_extra_params", [])
participant["beacon_extra_params"] = beacon_extra_params

Expand Down Expand Up @@ -228,6 +237,7 @@ def default_participant():
"v_max_cpu": 0,
"v_min_mem": 0,
"v_max_mem": 0,
"validator_count": None,
"snooper_enabled": False,
"count": 1
}
59 changes: 30 additions & 29 deletions src/cl/lighthouse/lighthouse_launcher.star
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,6 @@ def launch(
bn_min_mem = int(bn_min_mem) if int(bn_min_mem) > 0 else BEACON_MIN_MEMORY
bn_max_mem = int(bn_max_mem) if int(bn_max_mem) > 0 else BEACON_MAX_MEMORY

v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU
v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU
v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY
v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY

# Launch Beacon node
beacon_config = get_beacon_config(
launcher.genesis_data,
Expand All @@ -127,26 +122,31 @@ def launch(
)

beacon_service = plan.add_service(beacon_node_service_name, beacon_config)

beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID]

# Launch validator node
beacon_http_url = "http://{0}:{1}".format(beacon_service.ip_address, beacon_http_port.number)

validator_config = get_validator_config(
launcher.genesis_data,
image,
log_level,
beacon_http_url,
node_keystore_files,
v_min_cpu,
v_max_cpu,
v_min_mem,
v_max_mem,
extra_validator_params,
)

validator_service = plan.add_service(validator_node_service_name, validator_config)
# Launch validator node if we have a keystore
validator_service = None
if node_keystore_files != None:
v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU
v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU
v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY
v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY

validator_config = get_validator_config(
launcher.genesis_data,
image,
log_level,
beacon_http_url,
node_keystore_files,
v_min_cpu,
v_max_cpu,
v_min_mem,
v_max_mem,
extra_validator_params,
)

validator_service = plan.add_service(validator_node_service_name, validator_config)

# TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module
beacon_node_identity_recipe = GetHttpRequestRecipe(
Expand All @@ -165,13 +165,14 @@ def launch(

beacon_metrics_port = beacon_service.ports[BEACON_METRICS_PORT_ID]
beacon_metrics_url = "{0}:{1}".format(beacon_service.ip_address, beacon_metrics_port.number)

validator_metrics_port = validator_service.ports[VALIDATOR_METRICS_PORT_ID]
validator_metrics_url = "{0}:{1}".format(validator_service.ip_address, validator_metrics_port.number)

beacon_node_metrics_info = cl_node_metrics.new_cl_node_metrics_info(beacon_node_service_name, METRICS_PATH, beacon_metrics_url)
validator_node_metrics_info = cl_node_metrics.new_cl_node_metrics_info(validator_node_service_name, METRICS_PATH, validator_metrics_url)
nodes_metrics_info = [beacon_node_metrics_info, validator_node_metrics_info]
nodes_metrics_info = [beacon_node_metrics_info]

if validator_service:
validator_metrics_port = validator_service.ports[VALIDATOR_METRICS_PORT_ID]
validator_metrics_url = "{0}:{1}".format(validator_service.ip_address, validator_metrics_port.number)
validator_node_metrics_info = cl_node_metrics.new_cl_node_metrics_info(validator_node_service_name, METRICS_PATH, validator_metrics_url)
nodes_metrics_info.append(validator_node_metrics_info)

return cl_client_context.new_cl_client_context(
"lighthouse",
Expand Down Expand Up @@ -316,6 +317,7 @@ def get_validator_config(

# For some reason, Lighthouse takes in the parent directory of the config file (rather than the path to the config file itself)
genesis_config_parent_dirpath_on_client = shared_utils.path_join(GENESIS_DATA_MOUNTPOINT_ON_CLIENTS, shared_utils.path_dir(genesis_data.config_yml_rel_filepath))

validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, node_keystore_files.raw_keys_relative_dirpath)
validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, node_keystore_files.raw_secrets_relative_dirpath)

Expand Down Expand Up @@ -348,7 +350,6 @@ def get_validator_config(
if len(extra_params):
cmd.extend([param for param in extra_params])


return ServiceConfig(
image = image,
ports = VALIDATOR_USED_PORTS,
Expand Down
48 changes: 23 additions & 25 deletions src/cl/lodestar/lodestar_launcher.star
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,20 +87,13 @@ def launch(

beacon_node_service_name = "{0}".format(service_name)
validator_node_service_name = "{0}-{1}".format(service_name, VALIDATOR_SUFFIX_SERVICE_NAME)

log_level = input_parser.get_client_log_level_or_default(participant_log_level, global_log_level, LODESTAR_LOG_LEVELS)

bn_min_cpu = int(bn_min_cpu) if int(bn_min_cpu) > 0 else BEACON_MIN_CPU
bn_max_cpu = int(bn_max_cpu) if int(bn_max_cpu) > 0 else BEACON_MAX_CPU
bn_min_mem = int(bn_min_mem) if int(bn_min_mem) > 0 else BEACON_MIN_MEMORY
bn_max_mem = int(bn_max_mem) if int(bn_max_mem) > 0 else BEACON_MAX_MEMORY

v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU
v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU
v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY
v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY


# Launch Beacon node
beacon_config = get_beacon_config(
launcher.cl_genesis_data,
Expand All @@ -121,25 +114,29 @@ def launch(

beacon_http_port = beacon_service.ports[HTTP_PORT_ID]


# Launch validator node
beacon_http_url = "http://{0}:{1}".format(beacon_service.ip_address, beacon_http_port.number)

validator_config = get_validator_config(
validator_node_service_name,
launcher.cl_genesis_data,
image,
log_level,
beacon_http_url,
node_keystore_files,
v_min_cpu,
v_max_cpu,
v_min_mem,
v_max_mem,
extra_validator_params,
)

validator_service = plan.add_service(validator_node_service_name, validator_config)
# Launch validator node if we have a keystore
if node_keystore_files != None:
v_min_cpu = int(v_min_cpu) if int(v_min_cpu) > 0 else VALIDATOR_MIN_CPU
v_max_cpu = int(v_max_cpu) if int(v_max_cpu) > 0 else VALIDATOR_MAX_CPU
v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY
v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY
validator_config = get_validator_config(
validator_node_service_name,
launcher.cl_genesis_data,
image,
log_level,
beacon_http_url,
node_keystore_files,
v_min_cpu,
v_max_cpu,
v_min_mem,
v_max_mem,
extra_validator_params,
)

plan.add_service(validator_node_service_name, validator_config)

# TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module

Expand Down Expand Up @@ -283,9 +280,11 @@ def get_validator_config(
root_dirpath = shared_utils.path_join(CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, service_name)

genesis_config_filepath = shared_utils.path_join(GENESIS_DATA_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, genesis_data.config_yml_rel_filepath)

validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.raw_keys_relative_dirpath)
validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, node_keystore_files.raw_secrets_relative_dirpath)


cmd = [
"validator",
"--logLevel=" + log_level,
Expand All @@ -306,7 +305,6 @@ def get_validator_config(
# this is a repeated<proto type>, we convert it into Starlark
cmd.extend([param for param in extra_params])


return ServiceConfig(
image = image,
ports = VALIDATOR_USED_PORTS,
Expand Down
45 changes: 33 additions & 12 deletions src/cl/nimbus/nimbus_launcher.star
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,15 +187,20 @@ def get_config(
# For some reason, Nimbus takes in the parent directory of the config file (rather than the path to the config file itself)
genesis_config_parent_dirpath_on_client = shared_utils.path_join(GENESIS_DATA_MOUNTPOINT_ON_CLIENT, shared_utils.path_dir(genesis_data.config_yml_rel_filepath))
jwt_secret_filepath = shared_utils.path_join(GENESIS_DATA_MOUNTPOINT_ON_CLIENT, genesis_data.jwt_secret_rel_filepath)
validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT, node_keystore_files.nimbus_keys_relative_dirpath)
validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT, node_keystore_files.raw_secrets_relative_dirpath)


validator_keys_dirpath = ""
validator_secrets_dirpath = ""
if node_keystore_files != None:
validator_keys_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT, node_keystore_files.nimbus_keys_relative_dirpath)
validator_secrets_dirpath = shared_utils.path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT, node_keystore_files.raw_secrets_relative_dirpath)

# Sources for these flags:
# 1) https://github.com/status-im/nimbus-eth2/blob/stable/scripts/launch_local_testnet.sh
# 2) https://github.com/status-im/nimbus-eth2/blob/67ab477a27e358d605e99bffeb67f98d18218eca/scripts/launch_local_testnet.sh#L417
# WARNING: Do NOT set the --max-peers flag here, as doing so to the exact number of nodes seems to mess things up!
# See: https://github.com/kurtosis-tech/eth2-merge-kurtosis-module/issues/26
cmd = [
validator_copy = [
"mkdir",
CONSENSUS_DATA_DIRPATH_IN_SERVICE_CONTAINER,
"-m",
Expand All @@ -218,6 +223,15 @@ def get_config(
"600",
VALIDATOR_SECRETS_DIRPATH_ON_SERVICE_CONTAINER + "/*",
"&&",
]

validator_flags = [
"--validators-dir=" + VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER,
"--secrets-dir=" + VALIDATOR_SECRETS_DIRPATH_ON_SERVICE_CONTAINER,
"--suggested-fee-recipient=" + package_io.VALIDATING_REWARDS_ACCOUNT,
]

beacon_start = [
DEFAULT_IMAGE_ENTRYPOINT,
"--non-interactive=true",
"--log-level=" + log_level,
Expand All @@ -233,9 +247,6 @@ def get_config(
"--rest-address=0.0.0.0",
"--rest-allow-origin=*",
"--rest-port={0}".format(HTTP_PORT_NUM),
"--validators-dir=" + VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER,
"--secrets-dir=" + VALIDATOR_SECRETS_DIRPATH_ON_SERVICE_CONTAINER,
"--suggested-fee-recipient=" + package_io.VALIDATING_REWARDS_ACCOUNT,
# There's a bug where if we don't set this flag, the Nimbus nodes won't work:
# https://discord.com/channels/641364059387854899/674288681737256970/922890280120750170
# https://github.com/status-im/nimbus-eth2/issues/2451
Expand All @@ -251,6 +262,16 @@ def get_config(
"--metrics-port={0}".format(METRICS_PORT_NUM),
# ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
]

# Depending on whether we're using a node keystore, we'll need to add the validator flags
cmd = []
if node_keystore_files != None:
cmd.extend(validator_copy)
cmd.extend(beacon_start)
cmd.extend(validator_flags)
else:
cmd.extend(beacon_start)

if bootnode_contexts == None:
# Copied from https://github.com/status-im/nimbus-eth2/blob/67ab477a27e358d605e99bffeb67f98d18218eca/scripts/launch_local_testnet.sh#L417
# See explanation there
Expand All @@ -263,17 +284,18 @@ def get_config(
if len(extra_params) > 0:
cmd.extend([param for param in extra_params])

files = {
GENESIS_DATA_MOUNTPOINT_ON_CLIENT: genesis_data.files_artifact_uuid,
}
if node_keystore_files:
files[VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT] = node_keystore_files.files_artifact_uuid
cmd_str = " ".join(cmd)

return ServiceConfig(
image = image,
ports = USED_PORTS,
cmd = [cmd_str],
entrypoint = ENTRYPOINT_ARGS,
files = {
GENESIS_DATA_MOUNTPOINT_ON_CLIENT: genesis_data.files_artifact_uuid,
VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENT: node_keystore_files.files_artifact_uuid
},
files = files,
private_ip_address_placeholder = PRIVATE_IP_ADDRESS_PLACEHOLDER,
ready_conditions = cl_node_ready_conditions.get_ready_conditions(HTTP_PORT_ID),
min_cpu = bn_min_cpu,
Expand All @@ -282,7 +304,6 @@ def get_config(
max_memory = bn_max_mem
)


def new_nimbus_launcher(cl_genesis_data):
return struct(
cl_genesis_data = cl_genesis_data,
Expand Down

0 comments on commit d843d7c

Please sign in to comment.