feat: consensus component config for main node and external node (#881)

## What ❔

consensus component config for main node and external node

## Why ❔

Consensus component has been implemented, now we need a way to enable
it.

.github/workflows/ci-core-reusable.yml

@@ -118,8 +118,13 @@ jobs:
           ci_run sccache --show-stats
           ci_run cat /tmp/sccache_log.txt
   integration:
-    runs-on: [matterlabs-ci-runner]
+    strategy:
+      matrix:
+        consensus: [false,true]
+    env:
+      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,basic_witness_input_producer${{ matrix.consensus && ',consensus' || '' }}"
 
+    runs-on: [matterlabs-ci-runner]
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
         with:
@@ -179,7 +184,7 @@ jobs:
       # `sleep 5` because we need to wait until server started properly
       - name: Run server
         run: |
-          ci_run zk server &>server.log &
+          ci_run zk server --components=$SERVER_COMPONENTS &>server.log &
           ci_run sleep 5
 
       - name: Run contract verifier
@@ -197,7 +202,7 @@ jobs:
         run: |
           ci_run pkill zksync_server || true
           ci_run sleep 2
-          ci_run zk test i revert
+          ENABLE_CONSENSUS=${{ matrix.consensus }} PASSED_ENV_VARS=ENABLE_CONSENSUS ci_run zk test i revert
 
         # This test should be the last one as soon as it
         # finished bootloader will be different
@@ -230,7 +235,14 @@ jobs:
           ci_run cat /tmp/sccache_log.txt
 
   external-node:
+    strategy:
+      matrix:
+        consensus: [false,true]
     runs-on: [matterlabs-ci-runner]
+
+    env:
+      SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,basic_witness_input_producer${{ matrix.consensus && ',consensus' || '' }}"
+      EXT_NODE_FLAGS: "${{ matrix.consensus && '--enable-consensus' || '' }}"
 
     steps:
       - name: Checkout code # Checks out the repository under $GITHUB_WORKSPACE, so the job can access it.
@@ -291,14 +303,14 @@ jobs:
       # `sleep 30` because we need to wait until server started properly
       - name: Run server
         run: |
-          ci_run zk server &>server.log &
+          ci_run zk server --components=$SERVER_COMPONENTS &>server.log &
           ci_run sleep 30
-
+      
       - name: Run external node
         run: |
           ci_run zk env ext-node-docker
           ci_run zk db setup
-          ci_run zk external-node &>ext-node.log &
+          ci_run zk external-node $EXT_NODE_FLAGS &>ext-node.log &
           ci_run sleep 30
 
       # TODO(PLA-653): Restore bridge tests for EN.
@@ -315,14 +327,14 @@ jobs:
           ci_run pkill zksync_server || true
           ci_run sleep 2
           ci_run zk env
-          ci_run zk test i revert
+          ENABLE_CONSENSUS=${{ matrix.consensus }} PASSED_ENV_VARS=ENABLE_CONSENSUS ci_run zk test i revert
           # Check that the rollback was performed on the EN
           ci_run sleep 20
           ci_run grep -q 'Rollback successfully completed' ext-node.log
           # Restart the EN
-          ci_run zk server &>>server.log &
+          ci_run zk server --components=$SERVER_COMPONENTS &>>server.log &
           ci_run sleep 30
-          ZKSYNC_ENV=ext-node-docker ci_run zk external-node &>>ext-node.log &
+          ZKSYNC_ENV=ext-node-docker ci_run zk external-node $EXT_NODE_FLAGS &>>ext-node.log &
           ci_run sleep 30
 
       - name: Run upgrade test

.gitignore

@@ -35,6 +35,8 @@ Cargo.lock
 !/etc/env/docker.toml
 !/etc/env/ext-node.toml
 !/etc/env/ext-node-docker.toml
+!/etc/env/consensus_config.json
+!/etc/env/en_consensus_config.json
 /etc/tokens/localhost.json
 /etc/zksolc-bin/*
 /etc/zkvyper-bin/*

checks-config/era.dic

@@ -306,6 +306,7 @@ tokenomics
 validator
 validator's
 validator
+validators
 Validators
 CHAINID
 PREVRANDAO
@@ -804,6 +805,7 @@ hacky
 ceil
 Infura
 synth
+proto
 
 AUTOGENERATED
 x19Ethereum

core/bin/external_node/Cargo.toml

@@ -20,6 +20,9 @@ zksync_state = { path = "../../lib/state" }
 zksync_basic_types = { path = "../../lib/basic_types" }
 zksync_contracts = { path = "../../lib/contracts" }
 
+zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5b3d383d7a65b0fbe2a771fecf4313f5083be9ae" }
+zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5b3d383d7a65b0fbe2a771fecf4313f5083be9ae" }
+
 prometheus_exporter = { path = "../../lib/prometheus_exporter" }
 zksync_health_check = { path = "../../lib/health_check" }
 zksync_web3_decl = { path = "../../lib/web3_decl" }

core/bin/external_node/src/config/mod.rs

@@ -4,9 +4,13 @@ use anyhow::Context;
 use serde::Deserialize;
 use url::Url;
 use zksync_basic_types::{Address, L1ChainId, L2ChainId};
-use zksync_core::api_server::{
-    tx_sender::TxSenderConfig,
-    web3::{state::InternalApiConfig, Namespace},
+use zksync_consensus_roles::node;
+use zksync_core::{
+    api_server::{
+        tx_sender::TxSenderConfig,
+        web3::{state::InternalApiConfig, Namespace},
+    },
+    consensus,
 };
 use zksync_types::api::BridgeAddresses;
 use zksync_web3_decl::{
@@ -75,6 +79,12 @@ impl RemoteENConfig {
     }
 }
 
+#[derive(Debug, Deserialize, Clone, PartialEq)]
+pub enum BlockFetcher {
+    ServerAPI,
+    Consensus,
+}
+
 /// This part of the external node config is completely optional to provide.
 /// It can tweak limits of the API, delay intervals of certain components, etc.
 /// If any of the fields are not provided, the default values will be used.
@@ -406,14 +416,32 @@ impl PostgresConfig {
     }
 }
 
+fn read_operator_address() -> anyhow::Result<Address> {
+    Ok(std::env::var("EN_OPERATOR_ADDR")?.parse()?)
+}
+
+pub(crate) fn read_consensus_config() -> anyhow::Result<consensus::FetcherConfig> {
+    let path = std::env::var("EN_CONSENSUS_CONFIG_PATH")
+        .context("EN_CONSENSUS_CONFIG_PATH env variable is not set")?;
+    let cfg = std::fs::read_to_string(&path).context(path)?;
+    let cfg: consensus::config::Config =
+        consensus::config::decode_json(&cfg).context("failed decoding JSON")?;
+    let node_key: node::SecretKey = consensus::config::read_secret("EN_CONSENSUS_NODE_KEY")?;
+    Ok(consensus::FetcherConfig {
+        executor: cfg.executor_config(node_key),
+        operator_address: read_operator_address().context("read_operator_address()")?,
+    })
+}
+
 /// External Node Config contains all the configuration required for the EN operation.
 /// It is split into three parts: required, optional and remote for easier navigation.
-#[derive(Debug, Deserialize, Clone, PartialEq)]
+#[derive(Debug, Clone)]
 pub struct ExternalNodeConfig {
     pub required: RequiredENConfig,
     pub postgres: PostgresConfig,
     pub optional: OptionalENConfig,
     pub remote: RemoteENConfig,
+    pub consensus: Option<consensus::FetcherConfig>,
 }
 
 impl ExternalNodeConfig {
@@ -434,7 +462,6 @@ impl ExternalNodeConfig {
         let remote = RemoteENConfig::fetch(&client)
             .await
             .context("Unable to fetch required config values from the main node")?;
-
         // We can query them from main node, but it's better to set them explicitly
         // as well to avoid connecting to wrong environment variables unintentionally.
         let eth_chain_id = HttpClientBuilder::default()
@@ -479,6 +506,7 @@ impl ExternalNodeConfig {
             postgres,
             required,
             optional,
+            consensus: None,
         })
     }
 }