feat: support running consensus from snapshot (BFT-418) (#1429)

Support for running external node p2p fetcher from snapshot.
Implemented taking storage snapshot and restoring it in unit tests.

.github/workflows/ci-core-reusable.yml

@@ -200,11 +200,10 @@ jobs:
         run: ci_run zk test i server
 
       - name: Snapshot recovery test
-        if: ${{ ! matrix.consensus }}
         # We use `yarn` directly because the test launches `zk` commands in both server and EN envs.
         # An empty topmost environment helps avoid a mess when redefining env vars shared between both envs
         # (e.g., DATABASE_URL).
-        run: ci_run yarn snapshot-recovery-test snapshot-recovery-test
+        run: ENABLE_CONSENSUS=${{ matrix.consensus }} PASSED_ENV_VARS=ENABLE_CONSENSUS ci_run yarn snapshot-recovery-test snapshot-recovery-test
 
       - name: Fee projection tests
         run: ci_run zk test i fees

core/bin/external_node/Cargo.toml

@@ -28,8 +28,8 @@ zksync_web3_decl = { path = "../../lib/web3_decl" }
 zksync_types = { path = "../../lib/types" }
 vlog = { path = "../../lib/vlog" }
 
-zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
-zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
+zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
+zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
 vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "1c9cc500e92cf9ea052b230e114a6f9cce4fb2c1" }
 
 anyhow = "1.0"

core/bin/external_node/src/main.rs

@@ -456,7 +456,7 @@ struct Cli {
     /// or was synced from genesis.
     ///
     /// This is an experimental and incomplete feature; do not use unless you know what you're doing.
-    #[arg(long, conflicts_with = "enable_consensus")]
+    #[arg(long)]
     enable_snapshots_recovery: bool,
 }
 
@@ -491,17 +491,9 @@ async fn main() -> anyhow::Result<()> {
     let mut config = ExternalNodeConfig::collect()
         .await
         .context("Failed to load external node config")?;
-    if opt.enable_consensus {
-        // This is more of a sanity check; the mutual exclusion of `enable_consensus` and `enable_snapshots_recovery`
-        // should be ensured by `clap`.
-        anyhow::ensure!(
-            !opt.enable_snapshots_recovery,
-            "Consensus logic does not support snapshot recovery yet"
-        );
-    } else {
+    if !opt.enable_consensus {
         config.consensus = None;
     }
-
     if let Some(threshold) = config.optional.slow_query_threshold() {
         ConnectionPool::global_config().set_slow_query_threshold(threshold)?;
     }

core/bin/zksync_server/Cargo.toml

@@ -19,10 +19,10 @@ zksync_types = { path = "../../lib/types" }
 zksync_core = { path = "../../lib/zksync_core" }
 
 # Consensus dependenices
-zksync_consensus_crypto = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
-zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
-zksync_consensus_executor = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
-zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
+zksync_consensus_crypto = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
+zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
+zksync_consensus_executor = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
+zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
 vlog = { path = "../../lib/vlog" }
 
 anyhow = "1.0"

core/lib/dal/Cargo.toml

@@ -18,9 +18,9 @@ zksync_system_constants = { path = "../constants" }
 zksync_contracts = { path = "../contracts" }
 zksync_types = { path = "../types" }
 zksync_health_check = { path = "../health_check" }
-zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
-zksync_consensus_storage = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
-zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
+zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
+zksync_consensus_storage = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
+zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
 
 itertools = "0.10.1"
 thiserror = "1.0"
@@ -55,4 +55,4 @@ chrono = { version = "0.4", features = ["serde"] }
 assert_matches = "1.5.0"
 
 [build-dependencies]
-zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
+zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }

core/lib/dal/src/consensus_dal.rs

@@ -53,10 +53,6 @@ impl ConsensusDal<'_, '_> {
                 got.fork.number,
                 genesis.fork.number,
             );
-            anyhow::ensure!(
-                got.fork.first_parent.is_none(),
-                "fork with first_parent != None not supported",
-            );
         }
         let genesis =
             zksync_protobuf::serde::serialize(genesis, serde_json::value::Serializer).unwrap();
@@ -95,27 +91,55 @@ impl ConsensusDal<'_, '_> {
         Ok(())
     }
 
+    /// Fetches the range of miniblocks present in storage.
+    /// If storage was recovered from snapshot, the range doesn't need to start at 0.
+    pub async fn block_range(&mut self) -> anyhow::Result<std::ops::Range<validator::BlockNumber>> {
+        let mut txn = self
+            .storage
+            .start_transaction()
+            .await
+            .context("start_transaction")?;
+        let snapshot = txn
+            .snapshot_recovery_dal()
+            .get_applied_snapshot_status()
+            .await
+            .context("get_applied_snapshot_status()")?;
+        // `snapshot.miniblock_number` indicates the last block processed.
+        // This block is NOT present in storage. Therefore the first block
+        // that will appear in storage is `snapshot.miniblock_number+1`.
+        let start = validator::BlockNumber(snapshot.map_or(0, |s| s.miniblock_number.0 + 1).into());
+        let end = txn
+            .blocks_dal()
+            .get_sealed_miniblock_number()
+            .await
+            .context("get_sealed_miniblock_number")?
+            .map_or(start, |last| validator::BlockNumber(last.0.into()).next());
+        Ok(start..end)
+    }
+
     /// [Main node only] creates a new consensus fork starting at
     /// the last sealed miniblock. Resets the state of the consensus
     /// by calling `try_update_genesis()`.
     pub async fn fork(&mut self) -> anyhow::Result<()> {
-        let mut txn = self.storage.start_transaction().await?;
-        let Some(old) = txn.consensus_dal().genesis().await? else {
+        let mut txn = self
+            .storage
+            .start_transaction()
+            .await
+            .context("start_transaction")?;
+        let Some(old) = txn.consensus_dal().genesis().await.context("genesis()")? else {
             return Ok(());
         };
-        let last = txn
-            .blocks_dal()
-            .get_sealed_miniblock_number()
-            .await?
-            .context("forking without any blocks in storage is not supported yet")?;
-        let first_block = validator::BlockNumber(last.0.into());
-
+        let first_block = txn
+            .consensus_dal()
+            .block_range()
+            .await
+            .context("get_block_range()")?
+            .end;
         let new = validator::Genesis {
             validators: old.validators,
             fork: validator::Fork {
                 number: old.fork.number.next(),
                 first_block,
-                first_parent: None,
             },
         };
         txn.consensus_dal().try_update_genesis(&new).await?;
@@ -159,6 +183,30 @@ impl ConsensusDal<'_, '_> {
         Ok(())
     }
 
+    /// Fetches the first consensus certificate.
+    /// It might NOT be the certificate for the first miniblock:
+    /// see `validator::Genesis.first_block`.
+    pub async fn first_certificate(&mut self) -> anyhow::Result<Option<validator::CommitQC>> {
+        let Some(row) = sqlx::query!(
+            r#"
+            SELECT
+                certificate
+            FROM
+                miniblocks_consensus
+            ORDER BY
+                number ASC
+            LIMIT
+                1
+            "#
+        )
+        .fetch_optional(self.storage.conn())
+        .await?
+        else {
+            return Ok(None);
+        };
+        Ok(Some(zksync_protobuf::serde::deserialize(row.certificate)?))
+    }
+
     /// Fetches the last consensus certificate.
     /// Currently certificates are NOT generated synchronously with miniblocks,
     /// so it might NOT be the certificate for the last miniblock.
@@ -244,14 +292,10 @@ impl ConsensusDal<'_, '_> {
         let header = &cert.message.proposal;
         let mut txn = self.storage.start_transaction().await?;
         if let Some(last) = txn.consensus_dal().last_certificate().await? {
-            let last = &last.message.proposal;
             anyhow::ensure!(
-                last.number.next() == header.number,
+                last.header().number.next() == header.number,
                 "expected certificate for a block after the current head block"
             );
-            anyhow::ensure!(Some(last.hash()) == header.parent, "parent block mismatch");
-        } else {
-            anyhow::ensure!(header.parent.is_none(), "inserting first block with parent");
         }
         let want_payload = txn
             .consensus_dal()
@@ -297,7 +341,6 @@ mod tests {
             let fork = validator::Fork {
                 number: validator::ForkNumber(n),
                 first_block: rng.gen(),
-                first_parent: None,
             };
             let setup = validator::testonly::Setup::new_with_fork(rng, 3, fork);
             conn.consensus_dal()

core/lib/dal/src/snapshots_creator_dal.rs

@@ -40,6 +40,8 @@ impl SnapshotsCreatorDal<'_, '_> {
         Ok(count as u64)
     }
 
+    /// Constructs a `storate_logs` chunk of the state AFTER processing `[0..l1_batch_number]`
+    /// batches. `miniblock_number` MUST be the last miniblock of the `l1_batch_number` batch.
     pub async fn get_storage_logs_chunk(
         &mut self,
         miniblock_number: MiniblockNumber,

core/lib/object_store/Cargo.toml

@@ -13,7 +13,7 @@ categories = ["cryptography"]
 vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "1c9cc500e92cf9ea052b230e114a6f9cce4fb2c1" }
 zksync_config = { path = "../config" }
 zksync_types = { path = "../types" }
-zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
+zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
 anyhow = "1.0"
 async-trait = "0.1"
 bincode = "1"

core/lib/protobuf_config/Cargo.toml

@@ -16,7 +16,7 @@ serde_json = "1.0"
 serde_yaml = "0.9"
 zksync_basic_types = { path = "../basic_types" }
 zksync_config = { path = "../config" }
-zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
+zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
 zksync_types = { path = "../types" }
 
 anyhow = "1.0"
@@ -25,5 +25,5 @@ prost = "0.12.1"
 rand = "0.8"
 
 [build-dependencies]
-zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
+zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
 

core/lib/types/Cargo.toml

@@ -17,7 +17,7 @@ zksync_basic_types = { path = "../basic_types" }
 zksync_contracts = { path = "../contracts" }
 zksync_mini_merkle_tree = { path = "../mini_merkle_tree" }
 zksync_config = { path = "../config" }
-zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
+zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }
 
 anyhow = "1.0.75"
 chrono = { version = "0.4", features = ["serde"] }
@@ -43,4 +43,4 @@ tokio = { version = "1", features = ["rt", "macros"] }
 serde_with = { version = "1", features = ["hex"] }
 
 [build-dependencies]
-zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" }
+zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5329a809cfc06d4939fb5ece26c9ad1e1741c50a" }