Merge remote-tracking branch 'origin/main' into daniyar-pla-747-state…

…-keeper-async-cache

.github/release-please/manifest.json

@@ -1,4 +1,4 @@
 {
-  "core": "20.8.0",
+  "core": "21.0.0",
   "prover": "11.0.0"
 }

.github/workflows/build-docker-from-tag.yml

@@ -29,8 +29,6 @@ jobs:
       prover_fri_gpu_key_id: ${{ steps.extract-prover-fri-setup-key-ids.outputs.gpu_short_commit_sha }}
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
-        with:
-
       - name: Generate output with git tag
         id: set
         run: |

.github/workflows/vm-perf-comparison.yml

@@ -28,7 +28,7 @@ jobs:
         run: echo "sha=$(git merge-base HEAD FETCH_HEAD)" >> $GITHUB_OUTPUT
 
       - name: checkout divergence point
-        run: git checkout ${{ steps.merge_base.outputs.sha }}
+        run: git checkout ${{ steps.merge_base.outputs.sha }} --recurse-submodules
 
       - name: setup-env
         run: |

core/CHANGELOG.md

@@ -1,5 +1,45 @@
 # Changelog
 
+## [21.0.0](https://github.com/matter-labs/zksync-era/compare/core-v20.8.0...core-v21.0.0) (2024-03-01)
+
+
+### ⚠ BREAKING CHANGES
+
+* **prover:** Add EIP4844 support for provers subsystem ([#1200](https://github.com/matter-labs/zksync-era/issues/1200))
+* Set 21 as latest protocol version ([#1262](https://github.com/matter-labs/zksync-era/issues/1262))
+
+### Features
+
+* A way to disable slow query logging for some queries ([#1297](https://github.com/matter-labs/zksync-era/issues/1297)) ([13b82a0](https://github.com/matter-labs/zksync-era/commit/13b82a0db88d64b579e79cb824c71af8d14a9e64))
+* **aggregator:** support two operator addresses in sender and aggregator ([#1201](https://github.com/matter-labs/zksync-era/issues/1201)) ([7be56e5](https://github.com/matter-labs/zksync-era/commit/7be56e576fbb4d404319a6c099dcf1d5cb0b476a))
+* change EN main node fetcher concurrency factor ([#1317](https://github.com/matter-labs/zksync-era/issues/1317)) ([d4235b5](https://github.com/matter-labs/zksync-era/commit/d4235b541034b300a7043e83bb9fc44ab196dad5))
+* **commitment-generator:** Commitment for 1.4.2 ([#1234](https://github.com/matter-labs/zksync-era/issues/1234)) ([9b773eb](https://github.com/matter-labs/zksync-era/commit/9b773ebb3c170dc3d503ec83fb33d8d463614b4f))
+* **config:** add pubdata sending method to config ([#1261](https://github.com/matter-labs/zksync-era/issues/1261)) ([cebf55a](https://github.com/matter-labs/zksync-era/commit/cebf55acde566064d365840933515d79535af18c))
+* **config:** Added blobs to commit batches and made da source part of config ([#1243](https://github.com/matter-labs/zksync-era/issues/1243)) ([e01d1b6](https://github.com/matter-labs/zksync-era/commit/e01d1b64a20eabf7c43f310978730ad6f3223fb2))
+* **config:** update eth_sender to use blobs ([#1295](https://github.com/matter-labs/zksync-era/issues/1295)) ([e81f080](https://github.com/matter-labs/zksync-era/commit/e81f0800ad072e4a1433e1fdce6866bb7568f7da))
+* **core:** Adds support for 4844 transaction signing and rlp encoding ([#1254](https://github.com/matter-labs/zksync-era/issues/1254)) ([10e3a3e](https://github.com/matter-labs/zksync-era/commit/10e3a3ec50c84b3533af111d532e0b528c0b4966))
+* **dal:** `zksync_types::Transaction` to use protobuf for wire encoding (BFT-407) ([#1047](https://github.com/matter-labs/zksync-era/issues/1047)) ([ee94bee](https://github.com/matter-labs/zksync-era/commit/ee94beed9e610adee94ea5073f0c748df7447135))
+* **eth_sender:** set blob gas fee when blobs DA is used ([#1285](https://github.com/matter-labs/zksync-era/issues/1285)) ([57c5526](https://github.com/matter-labs/zksync-era/commit/57c5526602b89ce0fff9bbf06d840f7858b2ce99))
+* **gas-adjuster:** gas adjuster for EIP4844 ([#1255](https://github.com/matter-labs/zksync-era/issues/1255)) ([1da97ed](https://github.com/matter-labs/zksync-era/commit/1da97edea5bc323fc40d76c96d6ac48be77f0f7b))
+* Metrics for block tip ([#1144](https://github.com/matter-labs/zksync-era/issues/1144)) ([85d4b12](https://github.com/matter-labs/zksync-era/commit/85d4b124f59ee6596aaa8e6db3bbbb86fa829c90))
+* **node_framework:** Add Web3 API layers ([#1258](https://github.com/matter-labs/zksync-era/issues/1258)) ([105f4cc](https://github.com/matter-labs/zksync-era/commit/105f4cc2aebce958c3e9d2bfe4a4277fc26ebb4b))
+* **node_framework:** Support Proof Data Handler in the framework ([#1233](https://github.com/matter-labs/zksync-era/issues/1233)) ([2191218](https://github.com/matter-labs/zksync-era/commit/21912182b72f17c40d259f2bc0de9ce8e8241420))
+* **prover:** Add EIP4844 support for provers subsystem ([#1200](https://github.com/matter-labs/zksync-era/issues/1200)) ([6953e89](https://github.com/matter-labs/zksync-era/commit/6953e8989571d65230c66f3731c8831fd7abb274))
+* Remove methods to sign an arbitraty message ([#1294](https://github.com/matter-labs/zksync-era/issues/1294)) ([8904123](https://github.com/matter-labs/zksync-era/commit/89041236282e4b85de5eec3a4d9471f0fd3db58d))
+* Set 21 as latest protocol version ([#1262](https://github.com/matter-labs/zksync-era/issues/1262)) ([30579ef](https://github.com/matter-labs/zksync-era/commit/30579ef6cf5e8f96572d9b215fceec79556ce0ea))
+* support loading verifier and protocol version from db ([#1293](https://github.com/matter-labs/zksync-era/issues/1293)) ([533f013](https://github.com/matter-labs/zksync-era/commit/533f0131342dc40eea1bb82334828d0007043d19))
+
+
+### Bug Fixes
+
+* Add EIP4844 to fri_prover_group_config ([#1309](https://github.com/matter-labs/zksync-era/issues/1309)) ([edf9397](https://github.com/matter-labs/zksync-era/commit/edf9397ea65ddcb41b68f9732e496f38cadf3e58))
+* **api:** Use better gas per pubdata in API (pure server changes) ([#1311](https://github.com/matter-labs/zksync-era/issues/1311)) ([54f8d8c](https://github.com/matter-labs/zksync-era/commit/54f8d8caa702e0676b31cfc048d8d331fa961547))
+* **docker:** change default password for postgres ([#1250](https://github.com/matter-labs/zksync-era/issues/1250)) ([f6bff74](https://github.com/matter-labs/zksync-era/commit/f6bff74845983518026ec1d91a47523a3ec0f48c))
+* **en:** fail fast if we don't request correct number of txs from man node ([#1269](https://github.com/matter-labs/zksync-era/issues/1269)) ([1bcbf17](https://github.com/matter-labs/zksync-era/commit/1bcbf1751c6224619873373c7cf071cdebdf56ee))
+* Fix scheduler enqueuer bug ([#1322](https://github.com/matter-labs/zksync-era/issues/1322)) ([95deb92](https://github.com/matter-labs/zksync-era/commit/95deb92a2da59417470084894588a97c98bc65b8))
+* **snapshots_creator:** Fix snapshot generation query ([#1289](https://github.com/matter-labs/zksync-era/issues/1289)) ([e279456](https://github.com/matter-labs/zksync-era/commit/e2794564ce61e52b0f31079e9a0f67c27037002c))
+* **vm:** fix match clause in `get_max_gas_per_pubdata_byte` ([#1292](https://github.com/matter-labs/zksync-era/issues/1292)) ([eaf5a50](https://github.com/matter-labs/zksync-era/commit/eaf5a506266b5292df9b277b6c020170ca1f59db))
+
 ## [20.8.0](https://github.com/matter-labs/zksync-era/compare/core-v20.7.0...core-v20.8.0) (2024-02-26)
 
 

core/bin/external_node/src/main.rs

@@ -341,10 +341,13 @@ async fn init_tasks(
             .with_filter_limit(config.optional.filters_limit)
             .with_batch_request_size_limit(config.optional.max_batch_request_size)
             .with_response_body_size_limit(config.optional.max_response_body_size())
-            .with_tx_sender(tx_sender.clone(), vm_barrier.clone())
+            .with_tx_sender(tx_sender.clone())
+            .with_vm_barrier(vm_barrier.clone())
             .with_sync_state(sync_state.clone())
             .enable_api_namespaces(config.optional.api_namespaces())
-            .build(stop_receiver.clone())
+            .build()
+            .context("failed to build HTTP JSON-RPC server")?
+            .run(stop_receiver.clone())
             .await
             .context("Failed initializing HTTP JSON-RPC server")?;
 
@@ -356,10 +359,13 @@ async fn init_tasks(
             .with_batch_request_size_limit(config.optional.max_batch_request_size)
             .with_response_body_size_limit(config.optional.max_response_body_size())
             .with_polling_interval(config.optional.polling_interval())
-            .with_tx_sender(tx_sender, vm_barrier)
+            .with_tx_sender(tx_sender)
+            .with_vm_barrier(vm_barrier)
             .with_sync_state(sync_state)
             .enable_api_namespaces(config.optional.api_namespaces())
-            .build(stop_receiver.clone())
+            .build()
+            .context("failed to build WS JSON-RPC server")?
+            .run(stop_receiver.clone())
             .await
             .context("Failed initializing WS JSON-RPC server")?;
 

core/lib/basic_types/src/basic_fri_types.rs

@@ -6,6 +6,66 @@ use std::{convert::TryFrom, str::FromStr};
 
 use serde::{Deserialize, Serialize};
 
+const BLOB_CHUNK_SIZE: usize = 31;
+const ELEMENTS_PER_4844_BLOCK: usize = 4096;
+pub const MAX_4844_BLOBS_PER_BLOCK: usize = 2;
+
+pub const EIP_4844_BLOB_SIZE: usize = BLOB_CHUNK_SIZE * ELEMENTS_PER_4844_BLOCK;
+
+/// Wrapper struct over Vec<u8>, represents 1 EIP4844 Blob
+pub type Blob = Vec<u8>;
+
+/// Wrapper struct, containing EIP 4844 blobs and enforcing their invariants.
+/// Current invariants:
+///   - there are between [1, 2] blobs
+///   - all blobs are of the same size [`EIP_4844_BLOB_SIZE`]
+/// Creating a structure violating these constraints will panic.
+///
+/// Note: blobs are padded to fit the correct size.
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct Eip4844Blobs {
+    blobs: Vec<Blob>,
+}
+
+impl Eip4844Blobs {
+    pub fn blobs(self) -> Vec<Blob> {
+        self.blobs
+    }
+}
+
+impl From<Vec<u8>> for Eip4844Blobs {
+    fn from(blobs: Vec<u8>) -> Self {
+        let mut chunks: Vec<Blob> = blobs
+            .chunks(EIP_4844_BLOB_SIZE)
+            .map(|chunk| chunk.into())
+            .collect();
+
+        if let Some(last_chunk) = chunks.last_mut() {
+            last_chunk.resize(EIP_4844_BLOB_SIZE, 0u8);
+        } else {
+            panic!("cannot create Eip4844Blobs, received empty pubdata");
+        }
+
+        assert!(
+            chunks.len() <= MAX_4844_BLOBS_PER_BLOCK,
+            "cannot create Eip4844Blobs, received too many blobs"
+        );
+
+        Self { blobs: chunks }
+    }
+}
+
+impl From<Eip4844Blobs> for Vec<u8> {
+    fn from(eip_4844_blobs: Eip4844Blobs) -> Self {
+        eip_4844_blobs.blobs.iter().flatten().copied().collect()
+    }
+}
+
+pub struct FinalProofIds {
+    pub node_proof_ids: [u32; 13],
+    pub eip_4844_proof_ids: [u32; 2],
+}
+
 #[derive(Debug, Deserialize, Serialize, Clone, Eq, Hash, PartialEq, PartialOrd, Ord)]
 pub struct CircuitIdRoundTuple {
     pub circuit_id: u8,
@@ -99,3 +159,77 @@ impl TryFrom<i32> for AggregationRound {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    fn test_eip_4844_blobs_empty_pubdata() {
+        let payload = vec![];
+        let _eip_4844_blobs: Eip4844Blobs = payload.into();
+    }
+
+    #[test]
+    fn test_eip_4844_blobs_less_than_a_blob() {
+        // blob size (126976) - 1
+        let payload = vec![1; 126975];
+        let eip_4844_blobs: Eip4844Blobs = payload.into();
+        let blobs = eip_4844_blobs.blobs();
+        assert_eq!(blobs.len(), 1);
+        let blob = &blobs[0];
+        assert_eq!(blob[blob.len() - 1], 0);
+        assert_eq!(blob[0], 1);
+    }
+
+    #[test]
+    fn test_eip_4844_blobs_exactly_a_blob() {
+        // blob size (126976)
+        let payload = vec![1; 126976];
+        let eip_4844_blobs: Eip4844Blobs = payload.into();
+        let blobs = eip_4844_blobs.blobs();
+        assert_eq!(blobs.len(), 1);
+        let blob = &blobs[0];
+        assert_eq!(blob[blob.len() - 1], 1);
+        assert_eq!(blob[0], 1);
+    }
+
+    #[test]
+    fn test_eip_4844_blobs_less_than_two_blobs() {
+        // blob size (126976) * 2 (max number of blobs) - 1
+        let payload = vec![1; 253951];
+        let eip_4844_blobs: Eip4844Blobs = payload.into();
+        let blobs = eip_4844_blobs.blobs();
+        assert_eq!(blobs.len(), 2);
+        let first_blob = &blobs[0];
+        assert_eq!(first_blob[first_blob.len() - 1], 1);
+        assert_eq!(first_blob[0], 1);
+        let second_blob = &blobs[1];
+        assert_eq!(second_blob[second_blob.len() - 1], 0);
+        assert_eq!(second_blob[0], 1);
+    }
+
+    #[test]
+    fn test_eip_4844_blobs_exactly_two_blobs() {
+        // blob size (126976) * 2 (max number of blobs)
+        let payload = vec![1; 253952];
+        let eip_4844_blobs: Eip4844Blobs = payload.into();
+        let blobs = eip_4844_blobs.blobs();
+        assert_eq!(blobs.len(), 2);
+        let first_blob = &blobs[0];
+        assert_eq!(first_blob[first_blob.len() - 1], 1);
+        assert_eq!(first_blob[0], 1);
+        let second_blob = &blobs[1];
+        assert_eq!(second_blob[second_blob.len() - 1], 1);
+        assert_eq!(second_blob[0], 1);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_eip_4844_blobs_more_than_two_blobs() {
+        // blob size (126976) * 2 (max number of blobs) + 1
+        let payload = vec![1; 253953];
+        let _eip_4844_blobs: Eip4844Blobs = payload.into();
+    }
+}