feat(vm)!: fee model updates + 1.4.1 (#791)

## What ❔

Adds a new VM version `VM_1_4_1` that will work with `zk_evm@1.4.1` and
the [following
contracts](matter-labs/era-contracts#159).

Generally, it contains the following features (on the server side):

- The new fee model
- 1.4.1 integration
- The Increase of the maximal number of transactions per batch to 10
thousand.

❗ Requires DB migration and new config params before being deployed❗

## Why ❔

<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.
- [ ] Spellcheck has been run via `cargo spellcheck
--cfg=./spellcheck/era.cfg --code 1`.

---------

Co-authored-by: AntonD3 <antonv.dyadyuk@gmail.com>
Co-authored-by: AntonD3 <74021421+AntonD3@users.noreply.github.com>
Co-authored-by: Emil <evl@matterlabs.dev>
Co-authored-by: Fedor Sakharov <fedor.sakharov@gmail.com>
Co-authored-by: perekopskiy <mikeson.dp@gmail.com>
Co-authored-by: zksync-admin-bot2 <temp-bot@matterlabs.dev>
Co-authored-by: perekopskiy <53865202+perekopskiy@users.noreply.github.com>

checks-config/era.dic

@@ -879,3 +879,5 @@ decommitment
 hardcoded
 plookup
 shivini
+EIP4844
+KZG

core/bin/external_node/src/config/mod.rs

@@ -3,15 +3,15 @@ use std::{env, time::Duration};
 use anyhow::Context;
 use serde::Deserialize;
 use url::Url;
-use zksync_basic_types::{Address, L1ChainId, L2ChainId, MiniblockNumber};
+use zksync_basic_types::{Address, L1ChainId, L2ChainId};
 use zksync_core::api_server::{
     tx_sender::TxSenderConfig,
     web3::{state::InternalApiConfig, Namespace},
 };
 use zksync_types::api::BridgeAddresses;
 use zksync_web3_decl::{
     jsonrpsee::http_client::{HttpClient, HttpClientBuilder},
-    namespaces::{EnNamespaceClient, EthNamespaceClient, ZksNamespaceClient},
+    namespaces::{EthNamespaceClient, ZksNamespaceClient},
 };
 
 #[cfg(test)]
@@ -30,8 +30,6 @@ pub struct RemoteENConfig {
     pub l2_testnet_paymaster_addr: Option<Address>,
     pub l2_chain_id: L2ChainId,
     pub l1_chain_id: L1ChainId,
-
-    pub fair_l2_gas_price: u64,
 }
 
 impl RemoteENConfig {
@@ -63,15 +61,6 @@ impl RemoteENConfig {
                 .context("Failed to fetch L1 chain ID")?
                 .as_u64(),
         );
-        let current_miniblock = client
-            .get_block_number()
-            .await
-            .context("Failed to fetch block number")?;
-        let block_header = client
-            .sync_l2_block(MiniblockNumber(current_miniblock.as_u32()), false)
-            .await
-            .context("Failed to fetch last miniblock header")?
-            .expect("Block is known to exist");
 
         Ok(Self {
             diamond_proxy_addr,
@@ -82,7 +71,6 @@ impl RemoteENConfig {
             l2_weth_bridge_addr: bridges.l2_weth_bridge,
             l2_chain_id,
             l1_chain_id,
-            fair_l2_gas_price: block_header.l2_fair_gas_price,
         })
     }
 }
@@ -154,6 +142,15 @@ pub struct OptionalENConfig {
     /// The max possible number of gas that `eth_estimateGas` is allowed to overestimate.
     #[serde(default = "OptionalENConfig::default_estimate_gas_acceptable_overestimation")]
     pub estimate_gas_acceptable_overestimation: u32,
+    /// Whether to use the compatibility mode for gas estimation for L1->L2 transactions.
+    /// During the migration to the 1.4.1 fee model, there will be a period, when the server
+    /// will already have the 1.4.1 fee model, while the L1 contracts will still expect the transactions
+    /// to use the previous fee model with much higher overhead.
+    ///
+    /// When set to `true`, the API will ensure to return gasLimit is high enough overhead for both the old
+    /// and the new fee model when estimating L1->L2 transactions.  
+    #[serde(default = "OptionalENConfig::default_l1_to_l2_transactions_compatibility_mode")]
+    pub l1_to_l2_transactions_compatibility_mode: bool,
     /// The multiplier to use when suggesting gas price. Should be higher than one,
     /// otherwise if the L1 prices soar, the suggested gas price won't be sufficient to be included in block
     #[serde(default = "OptionalENConfig::default_gas_price_scale_factor")]
@@ -226,6 +223,10 @@ impl OptionalENConfig {
         1_000
     }
 
+    const fn default_l1_to_l2_transactions_compatibility_mode() -> bool {
+        true
+    }
+
     const fn default_gas_price_scale_factor() -> f64 {
         1.2
     }
@@ -527,13 +528,15 @@ impl From<ExternalNodeConfig> for TxSenderConfig {
                 .unwrap(),
             gas_price_scale_factor: config.optional.gas_price_scale_factor,
             max_nonce_ahead: config.optional.max_nonce_ahead,
-            fair_l2_gas_price: config.remote.fair_l2_gas_price,
             vm_execution_cache_misses_limit: config.optional.vm_execution_cache_misses_limit,
             // We set these values to the maximum since we don't know the actual values
             // and they will be enforced by the main node anyway.
             max_allowed_l2_tx_gas_limit: u32::MAX,
             validation_computational_gas_limit: u32::MAX,
             chain_id: config.remote.l2_chain_id,
+            l1_to_l2_transactions_compatibility_mode: config
+                .optional
+                .l1_to_l2_transactions_compatibility_mode,
         }
     }
 }

core/bin/snapshots_creator/src/tests.rs

@@ -143,6 +143,7 @@ async fn create_miniblock(
         l1_tx_count: 0,
         l2_tx_count: 0,
         base_fee_per_gas: 0,
+        gas_per_pubdata_limit: 0,
         batch_fee_input: Default::default(),
         base_system_contracts_hashes: Default::default(),
         protocol_version: Some(Default::default()),

core/bin/system-constants-generator/src/main.rs

@@ -3,7 +3,7 @@ use std::fs;
 use codegen::{Block, Scope};
 use multivm::{
     utils::{get_bootloader_encoding_space, get_bootloader_max_txs_in_batch},
-    vm_latest::constants::{BLOCK_OVERHEAD_GAS, BLOCK_OVERHEAD_L1_GAS, MAX_PUBDATA_PER_BLOCK},
+    vm_latest::constants::MAX_PUBDATA_PER_BLOCK,
 };
 use serde::{Deserialize, Serialize};
 use zksync_types::{
@@ -15,9 +15,13 @@ use zksync_types::{
         system_params::MAX_TX_ERGS_LIMIT,
     },
     IntrinsicSystemGasConstants, ProtocolVersionId, GUARANTEED_PUBDATA_IN_TX,
-    L1_GAS_PER_PUBDATA_BYTE, MAX_GAS_PER_PUBDATA_BYTE, MAX_NEW_FACTORY_DEPS,
+    L1_GAS_PER_PUBDATA_BYTE, MAX_NEW_FACTORY_DEPS, REQUIRED_L1_TO_L2_GAS_PER_PUBDATA_BYTE,
 };
 
+// For configs we will use the default value of `800_000` to represent the rough amount of L1 gas
+// needed to cover the batch expenses.
+const BLOCK_OVERHEAD_L1_GAS: u32 = 800_000;
+
 mod intrinsic_costs;
 mod utils;
 
@@ -30,7 +34,6 @@ struct L1SystemConfig {
     priority_tx_max_pubdata: u32,
     fair_l2_gas_price: u64,
     l1_gas_per_pubdata_byte: u32,
-    block_overhead_l2_gas: u32,
     block_overhead_l1_gas: u32,
     max_transactions_in_block: u32,
     bootloader_tx_encoding_space: u32,
@@ -56,23 +59,21 @@ pub fn generate_l1_contracts_system_config(gas_constants: &IntrinsicSystemGasCon
         priority_tx_max_pubdata: (L1_TX_DECREASE * (MAX_PUBDATA_PER_BLOCK as f64)) as u32,
         fair_l2_gas_price: FAIR_L2_GAS_PRICE_ON_L1_CONTRACT,
         l1_gas_per_pubdata_byte: L1_GAS_PER_PUBDATA_BYTE,
-        block_overhead_l2_gas: BLOCK_OVERHEAD_GAS,
         block_overhead_l1_gas: BLOCK_OVERHEAD_L1_GAS,
         max_transactions_in_block: get_bootloader_max_txs_in_batch(
             ProtocolVersionId::latest().into(),
         ) as u32,
         bootloader_tx_encoding_space: get_bootloader_encoding_space(
             ProtocolVersionId::latest().into(),
         ),
-
         l1_tx_intrinsic_l2_gas: gas_constants.l1_tx_intrinsic_gas,
         l1_tx_intrinsic_pubdata: gas_constants.l1_tx_intrinsic_pubdata,
         l1_tx_min_l2_gas_base: gas_constants.l1_tx_min_gas_base,
         l1_tx_delta_544_encoding_bytes: gas_constants.l1_tx_delta_544_encoding_bytes,
         l1_tx_delta_factory_deps_l2_gas: gas_constants.l1_tx_delta_factory_dep_gas,
         l1_tx_delta_factory_deps_pubdata: gas_constants.l1_tx_delta_factory_dep_pubdata,
         max_new_factory_deps: MAX_NEW_FACTORY_DEPS as u32,
-        required_l2_gas_price_per_pubdata: MAX_GAS_PER_PUBDATA_BYTE,
+        required_l2_gas_price_per_pubdata: REQUIRED_L1_TO_L2_GAS_PER_PUBDATA_BYTE,
     };
 
     serde_json::to_string_pretty(&l1_contracts_config).unwrap()
@@ -85,7 +86,6 @@ struct L2SystemConfig {
     guaranteed_pubdata_bytes: u32,
     max_pubdata_per_block: u32,
     max_transactions_in_block: u32,
-    block_overhead_l2_gas: u32,
     block_overhead_l1_gas: u32,
     l2_tx_intrinsic_gas: u32,
     l2_tx_intrinsic_pubdata: u32,
@@ -106,7 +106,6 @@ pub fn generate_l2_contracts_system_config(gas_constants: &IntrinsicSystemGasCon
         max_transactions_in_block: get_bootloader_max_txs_in_batch(
             ProtocolVersionId::latest().into(),
         ) as u32,
-        block_overhead_l2_gas: BLOCK_OVERHEAD_GAS,
         block_overhead_l1_gas: BLOCK_OVERHEAD_L1_GAS,
         l2_tx_intrinsic_gas: gas_constants.l2_tx_intrinsic_gas,
         l2_tx_intrinsic_pubdata: gas_constants.l2_tx_intrinsic_pubdata,

core/bin/system-constants-generator/src/utils.rs

@@ -2,14 +2,15 @@ use std::{cell::RefCell, rc::Rc};
 
 use multivm::{
     interface::{
-        dyn_tracers::vm_1_4_0::DynTracer, tracer::VmExecutionStopReason, L1BatchEnv, L2BlockEnv,
+        dyn_tracers::vm_1_4_1::DynTracer, tracer::VmExecutionStopReason, L1BatchEnv, L2BlockEnv,
         SystemEnv, TxExecutionMode, VmExecutionMode, VmInterface,
     },
     vm_latest::{
         constants::{BLOCK_GAS_LIMIT, BOOTLOADER_HEAP_PAGE},
         BootloaderState, HistoryEnabled, HistoryMode, SimpleMemory, ToTracerPointer, Vm, VmTracer,
         ZkSyncVmState,
     },
+    zk_evm_1_4_1::aux_structures::Timestamp,
 };
 use once_cell::sync::Lazy;
 use zksync_contracts::{
@@ -20,10 +21,9 @@ use zksync_state::{InMemoryStorage, StorageView, WriteStorage};
 use zksync_types::{
     block::MiniblockHasher, ethabi::Token, fee::Fee, fee_model::BatchFeeInput, l1::L1Tx, l2::L2Tx,
     utils::storage_key_for_eth_balance, AccountTreeId, Address, Execute, L1BatchNumber,
-    L1TxCommonData, L2ChainId, MiniblockNumber, Nonce, ProtocolVersionId, StorageKey, Timestamp,
-    Transaction, BOOTLOADER_ADDRESS, H256, SYSTEM_CONTEXT_ADDRESS,
-    SYSTEM_CONTEXT_GAS_PRICE_POSITION, SYSTEM_CONTEXT_TX_ORIGIN_POSITION, U256,
-    ZKPORTER_IS_AVAILABLE,
+    L1TxCommonData, L2ChainId, MiniblockNumber, Nonce, ProtocolVersionId, StorageKey, Transaction,
+    BOOTLOADER_ADDRESS, H256, SYSTEM_CONTEXT_ADDRESS, SYSTEM_CONTEXT_GAS_PRICE_POSITION,
+    SYSTEM_CONTEXT_TX_ORIGIN_POSITION, U256, ZKPORTER_IS_AVAILABLE,
 };
 use zksync_utils::{bytecode::hash_bytecode, bytes_to_be_words, u256_to_h256};
 

core/lib/commitment_utils/Cargo.toml

@@ -13,3 +13,4 @@ categories = ["cryptography"]
 zksync_types = { path = "../../lib/types" }
 zksync_utils = { path = "../../lib/utils" }
 zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.0" }
+multivm = { path = "../../lib/multivm" }

core/lib/commitment_utils/src/lib.rs

@@ -1,23 +1,33 @@
 //! Utils for commitment calculation.
+use multivm::utils::get_used_bootloader_memory_bytes;
 use zkevm_test_harness::witness::utils::{
     events_queue_commitment_fixed, initial_heap_content_commitment_fixed,
 };
-use zksync_types::{LogQuery, H256, U256, USED_BOOTLOADER_MEMORY_BYTES};
+use zksync_types::{LogQuery, ProtocolVersionId, H256, U256};
 use zksync_utils::expand_memory_contents;
 
-pub fn events_queue_commitment(events_queue: &Vec<LogQuery>, is_pre_boojum: bool) -> Option<H256> {
-    (!is_pre_boojum).then(|| H256(events_queue_commitment_fixed(events_queue)))
+pub fn events_queue_commitment(
+    events_queue: &Vec<LogQuery>,
+    protocol_version: ProtocolVersionId,
+) -> Option<H256> {
+    (!protocol_version.is_pre_boojum()).then(|| H256(events_queue_commitment_fixed(events_queue)))
 }
 
 pub fn bootloader_initial_content_commitment(
     initial_bootloader_contents: &[(usize, U256)],
-    is_pre_boojum: bool,
+    protocol_version: ProtocolVersionId,
 ) -> Option<H256> {
-    (!is_pre_boojum).then(|| {
-        let full_bootloader_memory =
-            expand_memory_contents(initial_bootloader_contents, USED_BOOTLOADER_MEMORY_BYTES);
-        H256(initial_heap_content_commitment_fixed(
-            &full_bootloader_memory,
-        ))
-    })
+    let expanded_memory_size = if protocol_version.is_pre_boojum() {
+        return None;
+    } else {
+        get_used_bootloader_memory_bytes(protocol_version.into())
+    };
+
+    let full_bootloader_memory =
+        expand_memory_contents(initial_bootloader_contents, expanded_memory_size);
+    let commitment = H256(initial_heap_content_commitment_fixed(
+        &full_bootloader_memory,
+    ));
+
+    Some(commitment)
 }

core/lib/config/src/configs/api.rs

@@ -51,6 +51,14 @@ pub struct Web3JsonRpcConfig {
     pub estimate_gas_scale_factor: f64,
     /// The max possible number of gas that `eth_estimateGas` is allowed to overestimate.
     pub estimate_gas_acceptable_overestimation: u32,
+    /// Whether to use the compatibility mode for gas estimation for L1->L2 transactions.
+    /// During the migration to the 1.4.1 fee model, there will be a period, when the server
+    /// will already have the 1.4.1 fee model, while the L1 contracts will still expect the transactions
+    /// to use the previous fee model with much higher overhead.
+    ///
+    /// When set to `true`, the API will ensure to return gasLimit is high enough overhead for both the old
+    /// and the new fee model when estimating L1->L2 transactions.  
+    pub l1_to_l2_transactions_compatibility_mode: bool,
     ///  Max possible size of an ABI encoded tx (in bytes).
     pub max_tx_size: usize,
     /// Max number of cache misses during one VM execution. If the number of cache misses exceeds this value, the API server panics.
@@ -101,6 +109,7 @@ impl Web3JsonRpcConfig {
             account_pks: Default::default(),
             estimate_gas_scale_factor: 1.2,
             estimate_gas_acceptable_overestimation: 1000,
+            l1_to_l2_transactions_compatibility_mode: true,
             max_tx_size: 1000000,
             vm_execution_cache_misses_limit: Default::default(),
             vm_concurrency_limit: Default::default(),

core/lib/config/src/configs/chain.rs

@@ -26,6 +26,25 @@ impl NetworkConfig {
     }
 }
 
+/// An enum that represents the version of the fee model to use.
+///  - `V1`, the first model that was used in zkSync Era. In this fee model, the pubdata price must be pegged to the L1 gas price.
+///  Also, the fair L2 gas price is expected to only include the proving/computation price for the operator and not the costs that come from
+///  processing the batch on L1.
+///  - `V2`, the second model that was used in zkSync Era. There the pubdata price might be independent from the L1 gas price. Also,
+///  The fair L2 gas price is expected to both the proving/computation price for the operator and the costs that come from
+///  processing the batch on L1.
+#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
+pub enum FeeModelVersion {
+    V1,
+    V2,
+}
+
+impl Default for FeeModelVersion {
+    fn default() -> Self {
+        Self::V1
+    }
+}
+
 #[derive(Debug, Deserialize, Clone, PartialEq, Default)]
 pub struct StateKeeperConfig {
     /// The max number of slots for txs in a block before it should be sealed by the slots sealer.
@@ -63,8 +82,26 @@ pub struct StateKeeperConfig {
 
     pub fee_account_addr: Address,
 
-    /// The price the operator spends on 1 gas of computation in wei.
-    pub fair_l2_gas_price: u64,
+    /// The minimal acceptable L2 gas price, i.e. the price that should include the cost of computation/proving as well
+    /// as potentially premium for congestion.
+    pub minimal_l2_gas_price: u64,
+    /// The constant that represents the possibility that a batch can be sealed because of overuse of computation resources.
+    /// It has range from 0 to 1. If it is 0, the compute will not depend on the cost for closing the batch.
+    /// If it is 1, the gas limit per batch will have to cover the entire cost of closing the batch.
+    pub compute_overhead_part: f64,
+    /// The constant that represents the possibility that a batch can be sealed because of overuse of pubdata.
+    /// It has range from 0 to 1. If it is 0, the pubdata will not depend on the cost for closing the batch.
+    /// If it is 1, the pubdata limit per batch will have to cover the entire cost of closing the batch.
+    pub pubdata_overhead_part: f64,
+    /// The constant amount of L1 gas that is used as the overhead for the batch. It includes the price for batch verification, etc.
+    pub batch_overhead_l1_gas: u64,
+    /// The maximum amount of gas that can be used by the batch. This value is derived from the circuits limitation per batch.
+    pub max_gas_per_batch: u64,
+    /// The maximum amount of pubdata that can be used by the batch. Note that if the calldata is used as pubdata, this variable should not exceed 128kb.
+    pub max_pubdata_per_batch: u64,
+
+    /// The version of the fee model to use.
+    pub fee_model_version: FeeModelVersion,
 
     /// Max number of computational gas that validation step is allowed to take.
     pub validation_computational_gas_limit: u32,
@@ -100,7 +137,13 @@ impl StateKeeperConfig {
             close_block_at_gas_percentage: 0.95,
             fee_account_addr: Address::from_str("0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7")
                 .unwrap(),
-            fair_l2_gas_price: 250000000,
+            compute_overhead_part: 0.0,
+            pubdata_overhead_part: 1.0,
+            batch_overhead_l1_gas: 800_000,
+            max_gas_per_batch: 200_000_000,
+            max_pubdata_per_batch: 100_000,
+            minimal_l2_gas_price: 100000000,
+            fee_model_version: FeeModelVersion::V2,
             validation_computational_gas_limit: 300000,
             save_call_traces: true,
             virtual_blocks_interval: 1,

core/lib/constants/src/crypto.rs

@@ -23,8 +23,6 @@ pub const MAX_NEW_FACTORY_DEPS: usize = 32;
 
 pub const PAD_MSG_BEFORE_HASH_BITS_LEN: usize = 736;
 
-/// The size of the bootloader memory in bytes which is used by the protocol.
-/// While the maximal possible size is a lot higher, we restrict ourselves to a certain limit to reduce
-/// the requirements on RAM.
-pub const USED_BOOTLOADER_MEMORY_BYTES: usize = 1 << 24;
-pub const USED_BOOTLOADER_MEMORY_WORDS: usize = USED_BOOTLOADER_MEMORY_BYTES / 32;
+/// To avoid DDoS we limit the size of the transactions size.
+/// TODO(X): remove this as a constant and introduce a config.
+pub const MAX_ENCODED_TX_SIZE: usize = 1 << 24;

core/lib/constants/src/ethereum.rs

@@ -5,11 +5,6 @@ pub const PRIORITY_EXPIRATION: u64 = 50000;
 pub const MAX_L1_TRANSACTION_GAS_LIMIT: u64 = 300000;
 pub static ETHEREUM_ADDRESS: Address = Address::zero();
 
-/// This the number of pubdata such that it should be always possible to publish
-/// from a single transaction. Note, that these pubdata bytes include only bytes that are
-/// to be published inside the body of transaction (i.e. excluding of factory deps).
-pub const GUARANTEED_PUBDATA_PER_L1_BATCH: u64 = 4000;
-
 /// The maximum number of pubdata per L1 batch. This limit is due to the fact that the Ethereum
 /// nodes do not accept transactions that have more than 128kb of pubdata.
 /// The 18kb margin is left in case of any impreciseness of the pubdata calculation.
@@ -18,10 +13,10 @@ pub const MAX_PUBDATA_PER_L1_BATCH: u64 = 110000;
 // TODO: import from `zkevm_opcode_defs` once `VM1.3` is supported
 pub const MAX_L2_TX_GAS_LIMIT: u64 = 80000000;
 
-// The users should always be able to provide `MAX_GAS_PER_PUBDATA_BYTE` gas per pubdata in their
-// transactions so that they are able to send at least `GUARANTEED_PUBDATA_PER_L1_BATCH` bytes per
-// transaction.
-pub const MAX_GAS_PER_PUBDATA_BYTE: u64 = MAX_L2_TX_GAS_LIMIT / GUARANTEED_PUBDATA_PER_L1_BATCH;
-
 // The L1->L2 are required to have the following gas per pubdata byte.
 pub const REQUIRED_L1_TO_L2_GAS_PER_PUBDATA_BYTE: u64 = 800;
+
+// The default gas per pubdata byte for L2 transactions, that is used, for instance, when we need to
+// insert some default value for type 2 transactions.
+// It is a realistic value, but it is large enough to fill into any batch regardless of the pubdata price.
+pub const DEFAULT_L2_TX_GAS_PER_PUBDATA_BYTE: u64 = 50_000;