feat: upgraded encoding of transactions in consensus Payload. (matter…

…-labs#2245)

Currently the encoded transaction is up to 3x larger than its rlp
encoding. This is caused by data duplication between: raw, input and
factory_deps fields. In the new encoding we use the rlp directly. It
will be used starting with protocol version 25.

---------

Co-authored-by: Bruno França <bruno@franca.xyz>

core/lib/basic_types/src/protocol_version.rs

@@ -71,11 +71,11 @@ pub enum ProtocolVersionId {
 }
 
 impl ProtocolVersionId {
-    pub fn latest() -> Self {
+    pub const fn latest() -> Self {
         Self::Version24
     }
 
-    pub fn next() -> Self {
+    pub const fn next() -> Self {
         Self::Version25
     }
 

core/lib/dal/Cargo.toml

@@ -49,5 +49,9 @@ strum = { workspace = true, features = ["derive"] }
 tracing.workspace = true
 chrono = { workspace = true, features = ["serde"] }
 
+[dev-dependencies]
+zksync_test_account.workspace = true
+zksync_concurrency.workspace = true
+
 [build-dependencies]
 zksync_protobuf_build.workspace = true

core/lib/dal/src/consensus/mod.rs

@@ -7,6 +7,7 @@ use anyhow::{anyhow, Context as _};
 use zksync_consensus_roles::validator;
 use zksync_protobuf::{required, ProtoFmt, ProtoRepr};
 use zksync_types::{
+    abi, ethabi,
     fee::Fee,
     l1::{OpProcessingType, PriorityQueueType},
     l2::TransactionType,
@@ -38,38 +39,59 @@ pub struct Payload {
 impl ProtoFmt for Payload {
     type Proto = proto::Payload;
 
-    fn read(message: &Self::Proto) -> anyhow::Result<Self> {
-        let mut transactions = Vec::with_capacity(message.transactions.len());
-        for (i, tx) in message.transactions.iter().enumerate() {
-            transactions.push(tx.read().with_context(|| format!("transactions[{i}]"))?)
+    fn read(r: &Self::Proto) -> anyhow::Result<Self> {
+        let protocol_version = required(&r.protocol_version)
+            .and_then(|x| Ok(ProtocolVersionId::try_from(u16::try_from(*x)?)?))
+            .context("protocol_version")?;
+        let mut transactions = vec![];
+
+        match protocol_version {
+            v if v >= ProtocolVersionId::Version25 => {
+                anyhow::ensure!(
+                    r.transactions.is_empty(),
+                    "transactions should be empty in protocol_version {v}"
+                );
+                for (i, tx) in r.transactions_v25.iter().enumerate() {
+                    transactions.push(
+                        tx.read()
+                            .with_context(|| format!("transactions_v25[{i}]"))?,
+                    );
+                }
+            }
+            v => {
+                anyhow::ensure!(
+                    r.transactions_v25.is_empty(),
+                    "transactions_v25 should be empty in protocol_version {v}"
+                );
+                for (i, tx) in r.transactions.iter().enumerate() {
+                    transactions.push(tx.read().with_context(|| format!("transactions[{i}]"))?)
+                }
+            }
         }
 
         Ok(Self {
-            protocol_version: required(&message.protocol_version)
-                .and_then(|x| Ok(ProtocolVersionId::try_from(u16::try_from(*x)?)?))
-                .context("protocol_version")?,
-            hash: required(&message.hash)
+            protocol_version,
+            hash: required(&r.hash)
                 .and_then(|h| parse_h256(h))
                 .context("hash")?,
             l1_batch_number: L1BatchNumber(
-                *required(&message.l1_batch_number).context("l1_batch_number")?,
+                *required(&r.l1_batch_number).context("l1_batch_number")?,
             ),
-            timestamp: *required(&message.timestamp).context("timestamp")?,
-            l1_gas_price: *required(&message.l1_gas_price).context("l1_gas_price")?,
-            l2_fair_gas_price: *required(&message.l2_fair_gas_price)
-                .context("l2_fair_gas_price")?,
-            fair_pubdata_price: message.fair_pubdata_price,
-            virtual_blocks: *required(&message.virtual_blocks).context("virtual_blocks")?,
-            operator_address: required(&message.operator_address)
+            timestamp: *required(&r.timestamp).context("timestamp")?,
+            l1_gas_price: *required(&r.l1_gas_price).context("l1_gas_price")?,
+            l2_fair_gas_price: *required(&r.l2_fair_gas_price).context("l2_fair_gas_price")?,
+            fair_pubdata_price: r.fair_pubdata_price,
+            virtual_blocks: *required(&r.virtual_blocks).context("virtual_blocks")?,
+            operator_address: required(&r.operator_address)
                 .and_then(|a| parse_h160(a))
                 .context("operator_address")?,
             transactions,
-            last_in_batch: *required(&message.last_in_batch).context("last_in_batch")?,
+            last_in_batch: *required(&r.last_in_batch).context("last_in_batch")?,
         })
     }
 
     fn build(&self) -> Self::Proto {
-        Self::Proto {
+        let mut x = Self::Proto {
             protocol_version: Some((self.protocol_version as u16).into()),
             hash: Some(self.hash.as_bytes().into()),
             l1_batch_number: Some(self.l1_batch_number.0),
@@ -80,13 +102,19 @@ impl ProtoFmt for Payload {
             virtual_blocks: Some(self.virtual_blocks),
             operator_address: Some(self.operator_address.as_bytes().into()),
             // Transactions are stored in execution order, therefore order is deterministic.
-            transactions: self
-                .transactions
-                .iter()
-                .map(proto::Transaction::build)
-                .collect(),
+            transactions: vec![],
+            transactions_v25: vec![],
             last_in_batch: Some(self.last_in_batch),
+        };
+        match self.protocol_version {
+            v if v >= ProtocolVersionId::Version25 => {
+                x.transactions_v25 = self.transactions.iter().map(ProtoRepr::build).collect();
+            }
+            _ => {
+                x.transactions = self.transactions.iter().map(ProtoRepr::build).collect();
+            }
         }
+        x
     }
 }
 
@@ -100,6 +128,50 @@ impl Payload {
     }
 }
 
+impl ProtoRepr for proto::TransactionV25 {
+    type Type = Transaction;
+
+    fn read(&self) -> anyhow::Result<Self::Type> {
+        use proto::transaction_v25::T;
+        let tx = match required(&self.t)? {
+            T::L1(l1) => abi::Transaction::L1 {
+                tx: required(&l1.rlp)
+                    .and_then(|x| {
+                        let tokens = ethabi::decode(&[abi::L2CanonicalTransaction::schema()], x)
+                            .context("ethabi::decode()")?;
+                        // Unwrap is safe because `ethabi::decode` does the verification.
+                        let tx =
+                            abi::L2CanonicalTransaction::decode(tokens.into_iter().next().unwrap())
+                                .context("L2CanonicalTransaction::decode()")?;
+                        Ok(tx)
+                    })
+                    .context("rlp")?
+                    .into(),
+                factory_deps: l1.factory_deps.clone(),
+                eth_block: 0,
+            },
+            T::L2(l2) => abi::Transaction::L2(required(&l2.rlp).context("rlp")?.clone()),
+        };
+        tx.try_into()
+    }
+
+    fn build(tx: &Self::Type) -> Self {
+        let tx = abi::Transaction::try_from(tx.clone()).unwrap();
+        use proto::transaction_v25::T;
+        Self {
+            t: Some(match tx {
+                abi::Transaction::L1 {
+                    tx, factory_deps, ..
+                } => T::L1(proto::L1Transaction {
+                    rlp: Some(ethabi::encode(&[tx.encode()])),
+                    factory_deps,
+                }),
+                abi::Transaction::L2(tx) => T::L2(proto::L2Transaction { rlp: Some(tx) }),
+            }),
+        }
+    }
+}
+
 impl ProtoRepr for proto::Transaction {
     type Type = Transaction;
 

core/lib/dal/src/consensus/proto/mod.proto

@@ -13,10 +13,30 @@ message Payload {
   optional uint64 fair_pubdata_price = 11; // required since 1.4.1; gwei
   optional uint32 virtual_blocks = 6; // required
   optional bytes operator_address = 7; // required; H160
+  // Set for protocol_version < 25.
   repeated Transaction transactions = 8;
+  // Set for protocol_version >= 25.
+  repeated TransactionV25 transactions_v25 = 12;
   optional bool last_in_batch = 10; // required
 }
 
+message L1Transaction {
+  optional bytes rlp = 1; // required; RLP encoded L2CanonicalTransaction
+  repeated bytes factory_deps = 2;
+}
+
+message L2Transaction {
+  optional bytes rlp = 1; // required; RLP encoded TransactionRequest
+}
+
+message TransactionV25 {
+  // required
+  oneof t {
+    L1Transaction l1 = 1;
+    L2Transaction l2 = 2;
+  }
+}
+
 message Transaction {
   reserved 5;
   reserved "received_timestamp_ms";

core/lib/dal/src/consensus/tests.rs

@@ -1,21 +1,75 @@
 use std::fmt::Debug;
 
+use rand::Rng;
+use zksync_concurrency::ctx;
 use zksync_protobuf::{
     repr::{decode, encode},
+    testonly::test_encode,
     ProtoRepr,
 };
-use zksync_types::{web3::Bytes, Execute, ExecuteTransactionCommon, Transaction};
+use zksync_test_account::Account;
+use zksync_types::{
+    web3::Bytes, Execute, ExecuteTransactionCommon, L1BatchNumber, ProtocolVersionId, Transaction,
+};
+
+use super::{proto, Payload};
+use crate::tests::mock_protocol_upgrade_transaction;
+
+fn execute(rng: &mut impl Rng) -> Execute {
+    Execute {
+        contract_address: rng.gen(),
+        value: rng.gen::<u128>().into(),
+        calldata: (0..10 * 32).map(|_| rng.gen()).collect(),
+        // TODO: find a way to generate valid random bytecode.
+        factory_deps: vec![],
+    }
+}
 
-use crate::tests::{mock_l1_execute, mock_l2_transaction, mock_protocol_upgrade_transaction};
+fn l1_transaction(rng: &mut impl Rng) -> Transaction {
+    Account::random_using(rng).get_l1_tx(execute(rng), rng.gen())
+}
+
+fn l2_transaction(rng: &mut impl Rng) -> Transaction {
+    Account::random_using(rng).get_l2_tx_for_execute(execute(rng), None)
+}
+
+fn payload(rng: &mut impl Rng, protocol_version: ProtocolVersionId) -> Payload {
+    Payload {
+        protocol_version,
+        hash: rng.gen(),
+        l1_batch_number: L1BatchNumber(rng.gen()),
+        timestamp: rng.gen(),
+        l1_gas_price: rng.gen(),
+        l2_fair_gas_price: rng.gen(),
+        fair_pubdata_price: Some(rng.gen()),
+        virtual_blocks: rng.gen(),
+        operator_address: rng.gen(),
+        transactions: (0..10)
+            .map(|_| match rng.gen() {
+                true => l1_transaction(rng),
+                false => l2_transaction(rng),
+            })
+            .collect(),
+        last_in_batch: rng.gen(),
+    }
+}
 
 /// Tests struct <-> proto struct conversions.
 #[test]
 fn test_encoding() {
-    encode_decode::<super::proto::Transaction, ComparableTransaction>(mock_l1_execute().into());
-    encode_decode::<super::proto::Transaction, ComparableTransaction>(mock_l2_transaction().into());
-    encode_decode::<super::proto::Transaction, ComparableTransaction>(
+    let ctx = &ctx::test_root(&ctx::RealClock);
+    let rng = &mut ctx.rng();
+    encode_decode::<proto::TransactionV25, ComparableTransaction>(l1_transaction(rng));
+    encode_decode::<proto::TransactionV25, ComparableTransaction>(l2_transaction(rng));
+    encode_decode::<proto::Transaction, ComparableTransaction>(l1_transaction(rng));
+    encode_decode::<proto::Transaction, ComparableTransaction>(l2_transaction(rng));
+    encode_decode::<proto::Transaction, ComparableTransaction>(
         mock_protocol_upgrade_transaction().into(),
     );
+    let p = payload(rng, ProtocolVersionId::Version24);
+    test_encode(rng, &p);
+    let p = payload(rng, ProtocolVersionId::Version25);
+    test_encode(rng, &p);
 }
 
 fn encode_decode<P, C>(msg: P::Type)

core/node/consensus/Cargo.toml

@@ -43,6 +43,7 @@ zksync_node_genesis.workspace = true
 zksync_node_test_utils.workspace = true
 zksync_node_api_server.workspace = true
 zksync_test_account.workspace = true
+zksync_contracts.workspace=  true
 
 tokio.workspace = true
 test-casing.workspace = true

core/node/consensus/src/storage/mod.rs

@@ -18,7 +18,7 @@ use zksync_types::{commitment::L1BatchWithMetadata, L1BatchNumber, L2BlockNumber
 use super::config;
 
 #[cfg(test)]
-mod testonly;
+pub(crate) mod testonly;
 
 /// Context-aware `zksync_dal::ConnectionPool<Core>` wrapper.
 #[derive(Debug, Clone)]