feat(configs): Implement new format of configs and implement protobuf…

… for it (#1501)

## What ❔

Loading new format of config from files. 
Only main node is fully moved to the ability to work from files.
Init process is intentionally has not be adapted for the file-based
configurtaion
This format will be used with the new configuration 


## Why ❔

For having an opportunity not to rely on envs we can load files with
necessary configuration.

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [x] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [x] Tests for the changes have been added / updated.
- [x] Documentation comments have been added / updated.
- [x] Code has been formatted via `zk fmt` and `zk lint`.
- [x] Spellcheck has been run via `zk spellcheck`.
- [x] Linkcheck has been run via `zk linkcheck`.

---------

Signed-off-by: Danil <deniallugo@gmail.com>
Co-authored-by: Grzegorz Prusak <pompon.pompon@gmail.com>

.gitignore

@@ -33,6 +33,7 @@ Cargo.lock
 
 /etc/env/*
 !/etc/env/base
+!/etc/env/file_based
 !/etc/env/dev.toml
 !/etc/env/dev_validium.toml
 !/etc/env/dev_validium_docker.toml

Cargo.toml

@@ -43,13 +43,12 @@ members = [
     "core/lib/vm_utils",
     "core/lib/web3_decl",
     "core/lib/snapshots_applier",
-
+    "core/lib/crypto_primitives",
     # Test infrastructure
     "core/tests/test_account",
     "core/tests/loadnext",
     "core/tests/vm-benchmark",
     "core/tests/vm-benchmark/harness",
-
     # SDK section
     "sdk/zksync-rs",
 ]
@@ -167,16 +166,16 @@ zk_evm_1_3_1 = { package = "zk_evm", git = "https://github.com/matter-labs/era-z
 zk_evm_1_3_3 = { package = "zk_evm", git = "https://github.com/matter-labs/era-zk_evm.git", tag = "v1.3.3-rc2" }
 zk_evm_1_4_0 = { package = "zk_evm", git = "https://github.com/matter-labs/era-zk_evm.git", branch = "v1.4.0" }
 zk_evm_1_4_1 = { package = "zk_evm", git = "https://github.com/matter-labs/era-zk_evm.git", branch = "v1.4.1" }
-zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
-zksync_consensus_bft = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
-zksync_consensus_crypto = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
-zksync_consensus_executor = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
-zksync_consensus_network = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
-zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
-zksync_consensus_storage = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
-zksync_consensus_utils = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
-zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
-zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "ce9279566ae9653c6da61b499e89c0779d601484" }
+zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
+zksync_consensus_bft = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
+zksync_consensus_crypto = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
+zksync_consensus_executor = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
+zksync_consensus_network = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
+zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
+zksync_consensus_storage = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
+zksync_consensus_utils = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
+zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
+zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "8833a9b7fef89d1ffc5c08d53a3560164bc1c694" }
 
 # "Local" dependencies
 multivm = { path = "core/lib/multivm" }
@@ -215,3 +214,4 @@ zksync_test_account = { path = "core/tests/test_account" }
 zksync_types = { path = "core/lib/types" }
 zksync_utils = { path = "core/lib/utils" }
 zksync_web3_decl = { path = "core/lib/web3_decl" }
+zksync_crypto_primitives = { path = "core/lib/crypto_primitives" }

core/bin/block_reverter/src/main.rs

@@ -2,8 +2,7 @@ use anyhow::Context as _;
 use clap::{Parser, Subcommand};
 use tokio::io::{self, AsyncReadExt};
 use zksync_config::{
-    configs::ObservabilityConfig, ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig,
-    PostgresConfig,
+    configs::ObservabilityConfig, ContractsConfig, DBConfig, ETHConfig, PostgresConfig,
 };
 use zksync_core::block_reverter::{
     BlockReverter, BlockReverterEthConfig, BlockReverterFlags, L1ExecutedBatchesRevert, NodeRole,
@@ -87,14 +86,17 @@ async fn main() -> anyhow::Result<()> {
     }
     let _guard = builder.build();
 
-    let eth_sender = ETHSenderConfig::from_env().context("ETHSenderConfig::from_env()")?;
+    let eth_sender = ETHConfig::from_env().context("ETHSenderConfig::from_env()")?;
     let db_config = DBConfig::from_env().context("DBConfig::from_env()")?;
-    let eth_client = ETHClientConfig::from_env().context("ETHClientConfig::from_env()")?;
-    let default_priority_fee_per_gas =
-        U256::from(eth_sender.gas_adjuster.default_priority_fee_per_gas);
+    let default_priority_fee_per_gas = U256::from(
+        eth_sender
+            .gas_adjuster
+            .context("gas_adjuster")?
+            .default_priority_fee_per_gas,
+    );
     let contracts = ContractsConfig::from_env().context("ContractsConfig::from_env()")?;
     let postgres_config = PostgresConfig::from_env().context("PostgresConfig::from_env()")?;
-    let config = BlockReverterEthConfig::new(eth_sender, contracts, eth_client.web3_url.clone());
+    let config = BlockReverterEthConfig::new(eth_sender, contracts);
 
     let connection_pool = ConnectionPool::<Core>::builder(
         postgres_config.master_url()?,

core/bin/external_node/src/config/mod.rs

@@ -17,7 +17,7 @@ use zksync_types::{api::BridgeAddresses, fee_model::FeeParams};
 use zksync_web3_decl::{
     error::ClientRpcContext,
     jsonrpsee::http_client::{HttpClient, HttpClientBuilder},
-    namespaces::{EthNamespaceClient, ZksNamespaceClient},
+    namespaces::{EnNamespaceClient, EthNamespaceClient, ZksNamespaceClient},
 };
 
 pub(crate) mod observability;
@@ -30,6 +30,8 @@ const BYTES_IN_MEGABYTE: usize = 1_024 * 1_024;
 #[derive(Debug, Deserialize, Clone, PartialEq)]
 pub struct RemoteENConfig {
     pub bridgehub_proxy_addr: Option<Address>,
+    pub state_transition_proxy_addr: Option<Address>,
+    pub transparent_proxy_admin_addr: Option<Address>,
     pub diamond_proxy_addr: Address,
     pub l1_erc20_bridge_proxy_addr: Address,
     pub l2_erc20_bridge_addr: Address,
@@ -39,6 +41,8 @@ pub struct RemoteENConfig {
     pub l2_chain_id: L2ChainId,
     pub l1_chain_id: L1ChainId,
     pub max_pubdata_per_batch: u64,
+    pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode,
+    pub dummy_verifier: bool,
 }
 
 impl RemoteENConfig {
@@ -51,8 +55,8 @@ impl RemoteENConfig {
             .get_testnet_paymaster()
             .rpc_context("get_testnet_paymaster")
             .await?;
-        // In case EN is connected to the old server version without `get_bridgehub_contract` method.
-        let bridgehub_proxy_addr = client.get_bridgehub_contract().await.ok().flatten();
+        let genesis = client.genesis_config().rpc_context("genesis").await.ok();
+        let shared_bridge = genesis.as_ref().and_then(|a| a.shared_bridge.clone());
         let diamond_proxy_addr = client
             .get_main_contract()
             .rpc_context("get_main_contract")
@@ -77,7 +81,13 @@ impl RemoteENConfig {
         };
 
         Ok(Self {
-            bridgehub_proxy_addr,
+            bridgehub_proxy_addr: shared_bridge.as_ref().map(|a| a.bridgehub_proxy_addr),
+            state_transition_proxy_addr: shared_bridge
+                .as_ref()
+                .map(|a| a.state_transition_proxy_addr),
+            transparent_proxy_admin_addr: shared_bridge
+                .as_ref()
+                .map(|a| a.transparent_proxy_admin_addr),
             diamond_proxy_addr,
             l2_testnet_paymaster_addr,
             l1_erc20_bridge_proxy_addr: bridges.l1_erc20_default_bridge,
@@ -87,6 +97,14 @@ impl RemoteENConfig {
             l2_chain_id,
             l1_chain_id,
             max_pubdata_per_batch,
+            l1_batch_commit_data_generator_mode: genesis
+                .as_ref()
+                .map(|a| a.l1_batch_commit_data_generator_mode)
+                .unwrap_or_default(),
+            dummy_verifier: genesis
+                .as_ref()
+                .map(|a| a.dummy_verifier)
+                .unwrap_or_default(),
         })
     }
 }
@@ -676,13 +694,17 @@ impl From<ExternalNodeConfig> for InternalApiConfig {
                 l2_weth_bridge: config.remote.l2_weth_bridge_addr,
             },
             bridgehub_proxy_addr: config.remote.bridgehub_proxy_addr,
+            state_transition_proxy_addr: config.remote.state_transition_proxy_addr,
+            transparent_proxy_admin_addr: config.remote.transparent_proxy_admin_addr,
             diamond_proxy_addr: config.remote.diamond_proxy_addr,
             l2_testnet_paymaster_addr: config.remote.l2_testnet_paymaster_addr,
             req_entities_limit: config.optional.req_entities_limit,
             fee_history_limit: config.optional.fee_history_limit,
             filters_disabled: config.optional.filters_disabled,
             mempool_cache_update_interval: config.optional.mempool_cache_update_interval(),
             mempool_cache_size: config.optional.mempool_cache_size,
+            dummy_verifier: config.remote.dummy_verifier,
+            l1_batch_commit_data_generator_mode: config.remote.l1_batch_commit_data_generator_mode,
         }
     }
 }

core/bin/external_node/src/config/observability.rs

@@ -36,5 +36,6 @@ pub fn observability_config_from_env() -> anyhow::Result<ObservabilityConfig> {
         sentry_environment,
         log_format,
         opentelemetry: None,
+        sporadic_crypto_errors_substrs: vec![],
     })
 }

core/bin/snapshots_creator/src/tests.rs

@@ -27,10 +27,12 @@ use super::*;
 const TEST_CONFIG: SnapshotsCreatorConfig = SnapshotsCreatorConfig {
     storage_logs_chunk_size: 1_000_000,
     concurrent_queries_count: 10,
+    object_store: None,
 };
 const SEQUENTIAL_TEST_CONFIG: SnapshotsCreatorConfig = SnapshotsCreatorConfig {
     storage_logs_chunk_size: 1_000_000,
     concurrent_queries_count: 1,
+    object_store: None,
 };
 
 #[derive(Debug)]

core/bin/zksync_server/Cargo.toml

@@ -13,6 +13,7 @@ publish = false
 [dependencies]
 zksync_config.workspace = true
 zksync_env_config.workspace = true
+zksync_protobuf_config.workspace = true
 zksync_storage.workspace = true
 zksync_utils.workspace = true
 zksync_types.workspace = true