feat!: merged genesis generation (#288)

Co-authored-by: Gyanendra Mishra <anomaly.the@gmail.com>

.circleci/config.yml

@@ -11,7 +11,7 @@ executors:
 parameters:
   should-enable-check-latest-version-workflow:
     type: boolean
-    default: false
+    default: true
   # To enable/disabled the check_code workflow execution which will be triggered by the PR's checkers
   should-enable-build-workflow:
     type: boolean

.circleci/tests/besu-all.json

@@ -25,8 +25,5 @@
       "cl_client_type": "lodestar"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/erigon-all.json

@@ -1,9 +1,5 @@
 {
   "participants": [
-    {
-      "el_client_type": "geth",
-      "cl_client_type": "teku"
-    },
     {
       "el_client_type": "erigon",
       "cl_client_type": "teku"
@@ -25,8 +21,5 @@
       "cl_client_type": "lodestar"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/ethereumjs-all.json

@@ -21,8 +21,5 @@
       "cl_client_type": "lodestar"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/geth-all.json

@@ -21,8 +21,5 @@
       "cl_client_type": "lodestar"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/lighthouse-all.json

@@ -25,8 +25,5 @@
       "cl_client_type": "lighthouse"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/lodestar-all.json

@@ -25,8 +25,5 @@
       "cl_client_type": "lodestar"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/mev.json

@@ -10,10 +10,10 @@
         "prometheus_grafana"
     ],
     "mev_params": {
-        "mev_relay_image": "flashbots/mev-boost-relay:0.27"
+        "launch_custom_flood": true,
+        "mev_relay_image": "flashbots/mev-boost-relay:0.28.0a2"
     },
     "network_params": {
-        "seconds_per_slot": 3,
-        "capella_fork_epoch": 1
+        "seconds_per_slot": 3
     }
-}
+}

.circleci/tests/mix-with-capella.json

@@ -0,0 +1,32 @@
+{
+  "participants": [
+    {
+      "el_client_type": "geth",
+      "cl_client_type": "teku"
+    },
+    {
+      "el_client_type": "nethermind",
+      "cl_client_type": "prysm"
+    },
+    {
+      "el_client_type": "erigon",
+      "cl_client_type": "nimbus"
+    },
+    {
+      "el_client_type": "besu",
+      "cl_client_type": "lighthouse"
+    },
+    {
+      "el_client_type": "reth",
+      "cl_client_type": "lodestar"
+    },
+    {
+      "el_client_type": "ethereumjs",
+      "cl_client_type": "teku"
+    }
+  ],
+  "network_params": {
+    "capella_fork_epoch": 1
+  },
+  "launch_additional_services": false
+}

.circleci/tests/mix.json

@@ -25,8 +25,5 @@
       "cl_client_type": "teku"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/nethermind-all.json

@@ -21,8 +21,5 @@
       "cl_client_type": "lodestar"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/nimbus-all.json

@@ -25,8 +25,5 @@
       "cl_client_type": "nimbus"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/nimbus-mev.json

@@ -3,14 +3,5 @@
         "el_client_type": "geth",
         "cl_client_type": "nimbus"
     }],
-    "network_params": {
-        "network_id": "3151908",
-        "deposit_contract_address": "0x4242424242424242424242424242424242424242",
-        "seconds_per_slot": 12,
-        "num_validator_keys_per_node": 64,
-        "preregistered_validator_keys_mnemonic": "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete",
-        "capella_fork_epoch": 1
-    },
-    "global_client_log_level": "info",
     "mev_type": "full"
 }

.circleci/tests/prysm-all.json

@@ -25,8 +25,5 @@
       "cl_client_type": "prysm"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/reth-all.json

@@ -1,9 +1,5 @@
 {
   "participants": [
-    {
-      "el_client_type": "geth",
-      "cl_client_type": "teku"
-    },
     {
       "el_client_type": "reth",
       "cl_client_type": "teku"
@@ -25,8 +21,5 @@
       "cl_client_type": "lodestar"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

.circleci/tests/teku-all.json

@@ -25,8 +25,5 @@
       "cl_client_type": "teku"
     }
   ],
-  "network_params": {
-    "capella_fork_epoch": 1
-  },
   "launch_additional_services": false
 }

README.md

@@ -189,7 +189,11 @@ To configure the package behaviour, you can modify your `network_params.json` fi
 
             // Count of nodes to spin up for this participant
             // Default to 1
-            "count": 1
+            "count": 1,
+
+             // Count of the number of validators you want to run for a given participant
+             // Default to null, which means that the number of validators will be using the network parameter num_validator_keys_per_node
+             "validator_count": null
         }
     ],
 
@@ -213,6 +217,16 @@ To configure the package behaviour, you can modify your `network_params.json` fi
         // How long you want the network to wait before starting up
         "genesis_delay": 120,
 
+        // Max churn rate for the network introduced by
+        // EIP-7514 https://eips.ethereum.org/EIPS/eip-7514
+        // Defaults to 8
+        "max_churn": 8,
+
+        // Ejection balance
+        // Defaults to 16ETH
+        // 16000000000 gwei
+        "ejection_balance": 16000000000,
+
         // The epoch at which the capella and deneb forks are set to occur.
         "capella_fork_epoch": 0,
         "deneb_fork_epoch": 4,
@@ -245,7 +259,7 @@ To configure the package behaviour, you can modify your `network_params.json` fi
     //  - A light beacon chain explorer will be launched
     //  - Default: ["tx_spammer", "blob_spammer", "el_forkmon", "beacon_metrics_gazer", "dora"," "prometheus_grafana"]
     "additional_services": [
-       "tx_spammer",
+        "tx_spammer",
         "blob_spammer",
         "custom_flood",
         "goomy_blob",
@@ -256,12 +270,6 @@ To configure the package behaviour, you can modify your `network_params.json` fi
         "prometheus_grafana"
     ],
 
-    // Which blockchain explorer should be used
-    // "dora" will use the dora explorer developped by pk910
-    // "full" will use the explorer developped by the beaconcha.in team
-    // defaults to "light"
-    "explorer_version": "dora",
-
     //  If set, the package will block until a finalized epoch has occurred.
     "wait_for_finalization": false,
 
@@ -270,7 +278,8 @@ To configure the package behaviour, you can modify your `network_params.json` fi
     //  This value will be overridden by participant-specific values
     "global_client_log_level": "info",
 
-    // EngineAPI Snooper
+    // EngineAPI Snooper global flags for all participants
+    // Default to false
     "snooper_enabled": false,
 
     // Parallelizes keystore generation so that each node has keystores being generated in their own container
@@ -509,14 +518,14 @@ This package comes with [seven prefunded keys for testing](https://github.com/ku
 
 Here's a table of where the keys are used
 
-| Account Index | Component Used In   | Private Key Used | Public Key Used | Comment                    |
-|---------------|---------------------|------------------|-----------------|----------------------------|
+| Account Index | Component Used In   | Private Key Used | Public Key Used | Comment                     |
+|---------------|---------------------|------------------|-----------------|-----------------------------|
 | 0             | mev_flood           | ✅                |                 | As the admin_key           |
-| 0             | mev_custom_flood    |                  | ✅               | As the receiver of balance |
+| 0             | mev_custom_flood    |                   | ✅              | As the receiver of balance |
 | 1             | blob_spammer        | ✅                |                 | As the sender of blobs     |
 | 2             | mev_flood           | ✅                |                 | As the user_key            |
 | 3             | transaction_spammer | ✅                |                 | To spam transactions with  |
-| 4              | goomy_blob          | ✅                |                 | As the sender of blobs     |
+| 4              | goomy_blob         | ✅                |                 | As the sender of blobs     |
 | 5             | eip4788_deployment  | ✅                |                 | As contract deployer       |
 | 6             | mev_custom_flood    | ✅                |                 | As the sender of balance   |
 

docs/architecture.md

@@ -53,22 +53,18 @@ The participant network is the beating heart at the center of the package. The p
 
 We'll explain these phases one by one.
 
-### Generating EL client data
+### Generating EL and CL client data
 
-All EL clients require both a genesis file and a JWT secret. The exact format of the genesis file differs per client, so we first leverage [a Docker image containing tools for generating this genesis data][ethereum-genesis-generator] to create the actual files that the EL clients-to-be will need. This is accomplished by filling in several genesis generation config files found in [the `static_files` directory][static-files].
+All EL clients require both a genesis file and a JWT secret. The exact format of the genesis file differs per client, so we first leverage [a Docker image containing tools for generating this genesis data][ethereum-genesis-generator] to create the actual files that the EL clients-to-be will need. This is accomplished by filling in a single genesis generation environment config files found in [`static_files`](../static_files/genesis-generation-config/el-cl/values.env.tmpl).
 
-The generated output files then get stored in the Kurtosis enclave, ready for use when we start the EL clients. The information about these stored files is tracked in [the `el_genesis_data` struct](https://github.com/kurtosis-tech/ethereum-package/blob/main/src/participant_network/prelaunch_data_generator/el_genesis/el_genesis_data.star).
+CL clients, like EL clients also have a genesis and config files that they need. This is created at the same time as the EL genesis files.
+
+Then the validator keys are generated. A tool called [eth2-val-tools](https://github.com/protolambda/eth2-val-tools) is used to generate the keys. The keys are then stored as a file artifact.
 
 ### Starting EL clients
 
 Next, we plug the generated genesis data [into EL client "launchers"](https://github.com/kurtosis-tech/ethereum-package/tree/main/src/participant_network/el) to start a mining network of EL nodes. The launchers come with a `launch` function that consumes EL genesis data and produces information about the running EL client node. Running EL node information is represented by [an `el_client_context` struct](https://github.com/kurtosis-tech/ethereum-package/blob/main/src/participant_network/el/el_client_context.star). Each EL client type has its own launcher (e.g. [Geth](https://github.com/kurtosis-tech/ethereum-package/tree/main/src/participant_network/el/geth), [Besu](https://github.com/kurtosis-tech/ethereum-package/tree/main/src/participant_network/el/besu)) because each EL client will require different environment variables and flags to be set when launching the client's container.
 
-### Generating CL client data
-
-CL clients, like EL clients, also have genesis and config files that they need. We use [the same Docker image with tools for generating genesis data][ethereum-genesis-generator] to create the necessary CL files, we provide the genesis generation config using templates in [the `static_files` directory][static-files], and we store the generated output files in the Kurtosis enclave in the same way as the EL client genesis files. Like with EL nodes, CL genesis data information is tracked in [the `cl_client_context` struct](https://github.com/kurtosis-tech/ethereum-package/blob/main/src/participant_network/el/el_client_context.star).
-
-The major difference with CL clients is that CL clients have validator keys, so we need to generate keys for the CL clients to use during validation. The generation happens using [the same genesis-generating Docker image][ethereum-genesis-generator], but via a call to [a different tool bundled inside the Docker image](https://github.com/protolambda/eth2-val-tools).
-
 ### Starting CL clients
 
 Once CL genesis data and keys have been created, the CL client nodes are started via [the CL client launchers](https://github.com/kurtosis-tech/ethereum-package/tree/main/src/participant_network/cl). Just as with EL clients:

main.star

@@ -75,6 +75,7 @@ def run(plan, args={}):
         all_participants,
         final_genesis_timestamp,
         genesis_validators_root,
+        el_cl_data_files_artifact_uuid,
     ) = participant_network.launch_participant_network(
         plan,
         args_with_right_defaults.participants,
@@ -290,7 +291,12 @@ def run(plan, args={}):
         elif additional_service == "dora":
             plan.print("Launching dora")
             dora_config_template = read_file(static_files.DORA_CONFIG_TEMPLATE_FILEPATH)
-            dora.launch_dora(plan, dora_config_template, all_cl_client_contexts)
+            dora.launch_dora(
+                plan,
+                dora_config_template,
+                all_cl_client_contexts,
+                el_cl_data_files_artifact_uuid,
+            )
             plan.print("Succesfully launched dora")
         elif additional_service == "full_beaconchain_explorer":
             plan.print("Launching full-beaconchain-explorer")

network_params.json

@@ -11,7 +11,8 @@
 			"beacon_extra_params": [],
 			"validator_extra_params": [],
 			"builder_network_params": null,
-			"count": 1
+      "validator_count": null,
+			"count": 2
 		}
 	],
 	"network_params": {
@@ -22,7 +23,7 @@
 		"preregistered_validator_keys_mnemonic": "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete",
 		"genesis_delay": 120,
 		"capella_fork_epoch": 0,
-		"deneb_fork_epoch": 500,
+		"deneb_fork_epoch": 4,
 		"electra_fork_epoch": null
 	},
 	"launch_additional_services": true,