feat(prover)!: Add EIP4844 support for provers subsystem (#1200)

This PR adds support for EIP4844 as part of 1.4.2 release.

To support EIP4844, the prover needs to:
- prove EIP4844 blobs
- inject the result into scheduler proof

EIP4844 is special cased for 1.4.2. Later (1.5.0) it will behave like
any other circuit.

How EIP4844 works?

Blob proofs are generated at basic/ aggregation_round 0 stage and are
marked as final nodes to be inserted into scheduler. If there is a
single blob, the [witness for the second blob needs to be zeroed
out](https://github.com/matter-labs/era-zkevm_circuits/blob/v1.4.1/src/scheduler/mod.rs#L1149)
whilst the [proof still needs to be
injected](https://github.com/matter-labs/era-zkevm_circuits/blob/v1.4.1/src/scheduler/mod.rs#L1165)
(any proof will work). Blobs are stored in `witness_inputs_fri` and
`scheduler_dependency_tracker` is aware of the proofs as
`circuit_255_final_prover_job_id_0` for the first blob and
`circuit_255_final_prover_job_id_1` for the second blob. (circuit id 255
is blob, until the code-base migrates to 1.5.0, when it will become a
regular circuit).

---------

Co-authored-by: mm-zk <mm@matterlabs.dev>
Co-authored-by: Roman Brodetski <rb@matterlabs.dev>
Co-authored-by: zksync-admin-bot2 <temp-bot@matterlabs.dev>

core/lib/basic_types/src/basic_fri_types.rs

@@ -6,6 +6,66 @@ use std::{convert::TryFrom, str::FromStr};
 
 use serde::{Deserialize, Serialize};
 
+const BLOB_CHUNK_SIZE: usize = 31;
+const ELEMENTS_PER_4844_BLOCK: usize = 4096;
+pub const MAX_4844_BLOBS_PER_BLOCK: usize = 2;
+
+pub const EIP_4844_BLOB_SIZE: usize = BLOB_CHUNK_SIZE * ELEMENTS_PER_4844_BLOCK;
+
+/// Wrapper struct over Vec<u8>, represents 1 EIP4844 Blob
+pub type Blob = Vec<u8>;
+
+/// Wrapper struct, containing EIP 4844 blobs and enforcing their invariants.
+/// Current invariants:
+///   - there are between [1, 2] blobs
+///   - all blobs are of the same size [`EIP_4844_BLOB_SIZE`]
+/// Creating a structure violating these constraints will panic.
+///
+/// Note: blobs are padded to fit the correct size.
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct Eip4844Blobs {
+    blobs: Vec<Blob>,
+}
+
+impl Eip4844Blobs {
+    pub fn blobs(self) -> Vec<Blob> {
+        self.blobs
+    }
+}
+
+impl From<Vec<u8>> for Eip4844Blobs {
+    fn from(blobs: Vec<u8>) -> Self {
+        let mut chunks: Vec<Blob> = blobs
+            .chunks(EIP_4844_BLOB_SIZE)
+            .map(|chunk| chunk.into())
+            .collect();
+
+        if let Some(last_chunk) = chunks.last_mut() {
+            last_chunk.resize(EIP_4844_BLOB_SIZE, 0u8);
+        } else {
+            panic!("cannot create Eip4844Blobs, received empty pubdata");
+        }
+
+        assert!(
+            chunks.len() <= MAX_4844_BLOBS_PER_BLOCK,
+            "cannot create Eip4844Blobs, received too many blobs"
+        );
+
+        Self { blobs: chunks }
+    }
+}
+
+impl From<Eip4844Blobs> for Vec<u8> {
+    fn from(eip_4844_blobs: Eip4844Blobs) -> Self {
+        eip_4844_blobs.blobs.iter().flatten().copied().collect()
+    }
+}
+
+pub struct FinalProofIds {
+    pub node_proof_ids: [u32; 13],
+    pub eip_4844_proof_ids: [u32; 2],
+}
+
 #[derive(Debug, Deserialize, Serialize, Clone, Eq, Hash, PartialEq, PartialOrd, Ord)]
 pub struct CircuitIdRoundTuple {
     pub circuit_id: u8,
@@ -99,3 +159,77 @@ impl TryFrom<i32> for AggregationRound {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    fn test_eip_4844_blobs_empty_pubdata() {
+        let payload = vec![];
+        let _eip_4844_blobs: Eip4844Blobs = payload.into();
+    }
+
+    #[test]
+    fn test_eip_4844_blobs_less_than_a_blob() {
+        // blob size (126976) - 1
+        let payload = vec![1; 126975];
+        let eip_4844_blobs: Eip4844Blobs = payload.into();
+        let blobs = eip_4844_blobs.blobs();
+        assert_eq!(blobs.len(), 1);
+        let blob = &blobs[0];
+        assert_eq!(blob[blob.len() - 1], 0);
+        assert_eq!(blob[0], 1);
+    }
+
+    #[test]
+    fn test_eip_4844_blobs_exactly_a_blob() {
+        // blob size (126976)
+        let payload = vec![1; 126976];
+        let eip_4844_blobs: Eip4844Blobs = payload.into();
+        let blobs = eip_4844_blobs.blobs();
+        assert_eq!(blobs.len(), 1);
+        let blob = &blobs[0];
+        assert_eq!(blob[blob.len() - 1], 1);
+        assert_eq!(blob[0], 1);
+    }
+
+    #[test]
+    fn test_eip_4844_blobs_less_than_two_blobs() {
+        // blob size (126976) * 2 (max number of blobs) - 1
+        let payload = vec![1; 253951];
+        let eip_4844_blobs: Eip4844Blobs = payload.into();
+        let blobs = eip_4844_blobs.blobs();
+        assert_eq!(blobs.len(), 2);
+        let first_blob = &blobs[0];
+        assert_eq!(first_blob[first_blob.len() - 1], 1);
+        assert_eq!(first_blob[0], 1);
+        let second_blob = &blobs[1];
+        assert_eq!(second_blob[second_blob.len() - 1], 0);
+        assert_eq!(second_blob[0], 1);
+    }
+
+    #[test]
+    fn test_eip_4844_blobs_exactly_two_blobs() {
+        // blob size (126976) * 2 (max number of blobs)
+        let payload = vec![1; 253952];
+        let eip_4844_blobs: Eip4844Blobs = payload.into();
+        let blobs = eip_4844_blobs.blobs();
+        assert_eq!(blobs.len(), 2);
+        let first_blob = &blobs[0];
+        assert_eq!(first_blob[first_blob.len() - 1], 1);
+        assert_eq!(first_blob[0], 1);
+        let second_blob = &blobs[1];
+        assert_eq!(second_blob[second_blob.len() - 1], 1);
+        assert_eq!(second_blob[0], 1);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_eip_4844_blobs_more_than_two_blobs() {
+        // blob size (126976) * 2 (max number of blobs) + 1
+        let payload = vec![1; 253953];
+        let _eip_4844_blobs: Eip4844Blobs = payload.into();
+    }
+}

core/lib/dal/migrations/20240226120530_add_support_for_eip4844.down.sql

@@ -0,0 +1,4 @@
+ALTER TABLE scheduler_dependency_tracker_fri DROP COLUMN IF EXISTS eip_4844_final_prover_job_id_1;
+ALTER TABLE scheduler_dependency_tracker_fri DROP COLUMN IF EXISTS eip_4844_final_prover_job_id_0;
+
+ALTER TABLE witness_inputs_fri DROP COLUMN IF EXISTS eip_4844_blobs;

core/lib/dal/migrations/20240226120530_add_support_for_eip4844.up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE witness_inputs_fri ADD COLUMN IF NOT EXISTS eip_4844_blobs BYTEA;
+
+ALTER TABLE scheduler_dependency_tracker_fri ADD COLUMN IF NOT EXISTS eip_4844_final_prover_job_id_0 BIGINT;
+ALTER TABLE scheduler_dependency_tracker_fri ADD COLUMN IF NOT EXISTS eip_4844_final_prover_job_id_1 BIGINT;

core/lib/dal/src/fri_prover_dal.rs

@@ -23,6 +23,10 @@ pub mod types {
     use sqlx::types::chrono::{DateTime, Utc};
     use zksync_types::{basic_fri_types::AggregationRound, L1BatchNumber};
 
+    // This currently lives in `zksync_prover_types` -- we don't want a dependency between prover types (`zkevm_test_harness`) and DAL.
+    // This will be gone as part of 1.5.0, when EIP4844 becomes normal jobs, rather than special cased ones.
+    pub(crate) const EIP_4844_CIRCUIT_ID: u8 = 255;
+
     #[derive(Debug, Clone)]
     pub struct FriProverJobMetadata {
         pub id: u32,
@@ -249,14 +253,18 @@ impl FriProverDal<'_, '_> {
         for (sequence_number, (circuit_id, circuit_blob_url)) in
             circuit_ids_and_urls.iter().enumerate()
         {
+            // EIP 4844 are special cased.
+            // There exist only 2 blobs that are calculated at basic layer and injected straight into scheduler proof (as of 1.4.2).
+            // As part of 1.5.0, these will be treated as regular circuits, having basic, leaf, node and finally being attached as regular node proofs to the scheduler.
+            let is_node_final_proof = *circuit_id == types::EIP_4844_CIRCUIT_ID;
             self.insert_prover_job(
                 l1_batch_number,
                 *circuit_id,
                 depth,
                 sequence_number,
                 aggregation_round,
                 circuit_blob_url,
-                false,
+                is_node_final_proof,
                 protocol_version_id,
             )
             .await;

core/lib/dal/src/fri_scheduler_dependency_tracker_dal.rs

@@ -1,6 +1,6 @@
-use zksync_types::L1BatchNumber;
+use zksync_types::{basic_fri_types::FinalProofIds, L1BatchNumber};
 
-use crate::StorageProcessor;
+use crate::{fri_prover_dal::types, StorageProcessor};
 
 #[derive(Debug)]
 pub struct FriSchedulerDependencyTrackerDal<'a, 'c> {
@@ -35,6 +35,8 @@ impl FriSchedulerDependencyTrackerDal<'_, '_> {
                         AND circuit_11_final_prover_job_id IS NOT NULL
                         AND circuit_12_final_prover_job_id IS NOT NULL
                         AND circuit_13_final_prover_job_id IS NOT NULL
+                        AND eip_4844_final_prover_job_id_0 IS NOT NULL
+                        AND eip_4844_final_prover_job_id_0 IS NOT NULL
                 )
             RETURNING
                 l1_batch_number;
@@ -69,15 +71,30 @@ impl FriSchedulerDependencyTrackerDal<'_, '_> {
         circuit_id: u8,
         final_prover_job_id: u32,
         l1_batch_number: L1BatchNumber,
+        // As of 1.4.2, there exist only 2 blobs. Their order matter.
+        // `blob_ordering` is used to determine which blob is the first one and which is the second.
+        // This will be changed when 1.5.0 will land and there will be a single node proof for blobs.
+        blob_ordering: usize,
     ) {
-        let query = format!(
-            r#"
+        let query = if circuit_id != types::EIP_4844_CIRCUIT_ID {
+            format!(
+                r#"
                 UPDATE scheduler_dependency_tracker_fri
                 SET circuit_{}_final_prover_job_id = $1
                 WHERE l1_batch_number = $2
             "#,
-            circuit_id
-        );
+                circuit_id
+            )
+        } else {
+            format!(
+                r#"
+                    UPDATE scheduler_dependency_tracker_fri
+                    SET eip_4844_final_prover_job_id_{} = $1
+                    WHERE l1_batch_number = $2
+                "#,
+                blob_ordering,
+            )
+        };
         sqlx::query(&query)
             .bind(final_prover_job_id as i64)
             .bind(l1_batch_number.0 as i64)
@@ -89,7 +106,7 @@ impl FriSchedulerDependencyTrackerDal<'_, '_> {
     pub async fn get_final_prover_job_ids_for(
         &mut self,
         l1_batch_number: L1BatchNumber,
-    ) -> [u32; 13] {
+    ) -> FinalProofIds {
         sqlx::query!(
             r#"
             SELECT
@@ -106,8 +123,8 @@ impl FriSchedulerDependencyTrackerDal<'_, '_> {
         .unwrap()
         .into_iter()
         .next()
-        .map(|row| {
-            [
+        .map(|row| FinalProofIds {
+            node_proof_ids: [
                 row.circuit_1_final_prover_job_id.unwrap() as u32,
                 row.circuit_2_final_prover_job_id.unwrap() as u32,
                 row.circuit_3_final_prover_job_id.unwrap() as u32,
@@ -121,7 +138,11 @@ impl FriSchedulerDependencyTrackerDal<'_, '_> {
                 row.circuit_11_final_prover_job_id.unwrap() as u32,
                 row.circuit_12_final_prover_job_id.unwrap() as u32,
                 row.circuit_13_final_prover_job_id.unwrap() as u32,
-            ]
+            ],
+            eip_4844_proof_ids: [
+                row.eip_4844_final_prover_job_id_0.unwrap() as u32,
+                row.eip_4844_final_prover_job_id_1.unwrap() as u32,
+            ],
         })
         .unwrap()
     }