feat(state-keeper): circuits seal criterion (#729)

## What ❔ - the number of circuits needed is estimated in VM - replaces `ComputationGasCriterion` with more precise `CircuitsCriterion` ## Why ❔ Introduce `CircuitsCriterion` for efficient batch sealing ## Checklist   - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `cargo spellcheck --cfg=./spellcheck/era.cfg --code 1`.
matter-labs · Jan 5, 2024 · c4a86bb · c4a86bb
1 parent 0fb2877
commit c4a86bb
Show file tree

Hide file tree

Showing 60 changed files with 1,262 additions and 325 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/core/bin/snapshots_creator/src/tests.rs b/core/bin/snapshots_creator/src/tests.rs
@@ -173,7 +173,7 @@ async fn create_l1_batch(
     );
     header.is_finished = true;
     conn.blocks_dal()
-        .insert_l1_batch(&header, &[], BlockGasCount::default(), &[], &[])
+        .insert_l1_batch(&header, &[], BlockGasCount::default(), &[], &[], 0)
         .await
         .unwrap();
     conn.blocks_dal()

diff --git a/core/lib/dal/migrations/20240104121833_l1-batch-predicted-circuits.down.sql b/core/lib/dal/migrations/20240104121833_l1-batch-predicted-circuits.down.sql
@@ -0,0 +1,2 @@
+ALTER TABLE l1_batches
+    DROP COLUMN IF EXISTS predicted_circuits;
diff --git a/core/lib/dal/migrations/20240104121833_l1-batch-predicted-circuits.up.sql b/core/lib/dal/migrations/20240104121833_l1-batch-predicted-circuits.up.sql
@@ -0,0 +1,2 @@
+ALTER TABLE l1_batches
+    ADD COLUMN IF NOT EXISTS predicted_circuits INT;
diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json
@@ -5788,6 +5788,42 @@
     },
     "query": "\n            SELECT\n                number,\n                hash\n            FROM\n                miniblocks\n            WHERE\n                number >= $1\n            ORDER BY\n                number ASC\n            LIMIT\n                $2\n            "
   },
+  "70979db81f473950b2fae7816dbad7fe3464f2619cee2d583accaa829aa12b94": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Int8",
+          "Int4",
+          "Int4",
+          "Int8",
+          "Bool",
+          "Bytea",
+          "ByteaArray",
+          "ByteaArray",
+          "Bytea",
+          "ByteaArray",
+          "Int8",
+          "Int8",
+          "Int8",
+          "Jsonb",
+          "Jsonb",
+          "Numeric",
+          "Int8",
+          "Int8",
+          "Bytea",
+          "Bytea",
+          "Int4",
+          "ByteaArray",
+          "Int8Array",
+          "Bytea",
+          "Int4"
+        ]
+      }
+    },
+    "query": "\n            INSERT INTO\n                l1_batches (\n                    number,\n                    l1_tx_count,\n                    l2_tx_count,\n                    timestamp,\n                    is_finished,\n                    fee_account_address,\n                    l2_to_l1_logs,\n                    l2_to_l1_messages,\n                    bloom,\n                    priority_ops_onchain_data,\n                    predicted_commit_gas_cost,\n                    predicted_prove_gas_cost,\n                    predicted_execute_gas_cost,\n                    initial_bootloader_heap_content,\n                    used_contract_hashes,\n                    base_fee_per_gas,\n                    l1_gas_price,\n                    l2_fair_gas_price,\n                    bootloader_code_hash,\n                    default_aa_code_hash,\n                    protocol_version,\n                    system_logs,\n                    storage_refunds,\n                    pubdata_input,\n                    predicted_circuits,\n                    created_at,\n                    updated_at\n                )\n            VALUES\n                (\n                    $1,\n                    $2,\n                    $3,\n                    $4,\n                    $5,\n                    $6,\n                    $7,\n                    $8,\n                    $9,\n                    $10,\n                    $11,\n                    $12,\n                    $13,\n                    $14,\n                    $15,\n                    $16,\n                    $17,\n                    $18,\n                    $19,\n                    $20,\n                    $21,\n                    $22,\n                    $23,\n                    $24,\n                    $25,\n                    NOW(),\n                    NOW()\n                )\n            "
+  },
   "72a4f50355324cce85ebaef9fa32826095e9290f0c1157094bd0c44e06012e42": {
     "describe": {
       "columns": [
@@ -8002,41 +8038,6 @@
     },
     "query": "\n            SELECT\n                upgrade_tx_hash\n            FROM\n                protocol_versions\n            WHERE\n                id = $1\n            "
   },
-  "aa8e569cf406cd0975a6ffaeeafa92f632186181ba8b93518e549e0643f58da8": {
-    "describe": {
-      "columns": [],
-      "nullable": [],
-      "parameters": {
-        "Left": [
-          "Int8",
-          "Int4",
-          "Int4",
-          "Int8",
-          "Bool",
-          "Bytea",
-          "ByteaArray",
-          "ByteaArray",
-          "Bytea",
-          "ByteaArray",
-          "Int8",
-          "Int8",
-          "Int8",
-          "Jsonb",
-          "Jsonb",
-          "Numeric",
-          "Int8",
-          "Int8",
-          "Bytea",
-          "Bytea",
-          "Int4",
-          "ByteaArray",
-          "Int8Array",
-          "Bytea"
-        ]
-      }
-    },
-    "query": "\n            INSERT INTO\n                l1_batches (\n                    number,\n                    l1_tx_count,\n                    l2_tx_count,\n                    timestamp,\n                    is_finished,\n                    fee_account_address,\n                    l2_to_l1_logs,\n                    l2_to_l1_messages,\n                    bloom,\n                    priority_ops_onchain_data,\n                    predicted_commit_gas_cost,\n                    predicted_prove_gas_cost,\n                    predicted_execute_gas_cost,\n                    initial_bootloader_heap_content,\n                    used_contract_hashes,\n                    base_fee_per_gas,\n                    l1_gas_price,\n                    l2_fair_gas_price,\n                    bootloader_code_hash,\n                    default_aa_code_hash,\n                    protocol_version,\n                    system_logs,\n                    storage_refunds,\n                    pubdata_input,\n                    created_at,\n                    updated_at\n                )\n            VALUES\n                (\n                    $1,\n                    $2,\n                    $3,\n                    $4,\n                    $5,\n                    $6,\n                    $7,\n                    $8,\n                    $9,\n                    $10,\n                    $11,\n                    $12,\n                    $13,\n                    $14,\n                    $15,\n                    $16,\n                    $17,\n                    $18,\n                    $19,\n                    $20,\n                    $21,\n                    $22,\n                    $23,\n                    $24,\n                    NOW(),\n                    NOW()\n                )\n            "
-  },
   "aa91697157517322b0dbb53dca99f41220c51f58a03c61d6b7789eab0504e320": {
     "describe": {
       "columns": [

diff --git a/core/lib/dal/src/blocks_dal.rs b/core/lib/dal/src/blocks_dal.rs
@@ -450,6 +450,7 @@ impl BlocksDal<'_, '_> {
         predicted_block_gas: BlockGasCount,
         events_queue: &[LogQuery],
         storage_refunds: &[u32],
+        predicted_circuits: u32,
     ) -> anyhow::Result<()> {
         let priority_onchain_data: Vec<Vec<u8>> = header
             .priority_ops_onchain_data
@@ -509,6 +510,7 @@ impl BlocksDal<'_, '_> {
                     system_logs,
                     storage_refunds,
                     pubdata_input,
+                    predicted_circuits,
                     created_at,
                     updated_at
                 )
@@ -538,6 +540,7 @@ impl BlocksDal<'_, '_> {
                     $22,
                     $23,
                     $24,
+                    $25,
                     NOW(),
                     NOW()
                 )
@@ -566,6 +569,7 @@ impl BlocksDal<'_, '_> {
             &system_logs,
             &storage_refunds,
             pubdata_input,
+            predicted_circuits as i32,
         )
         .execute(transaction.conn())
         .await?;
@@ -2341,7 +2345,7 @@ mod tests {
         header.l2_to_l1_messages.push(vec![33; 33]);
 
         conn.blocks_dal()
-            .insert_l1_batch(&header, &[], BlockGasCount::default(), &[], &[])
+            .insert_l1_batch(&header, &[], BlockGasCount::default(), &[], &[], 0)
             .await
             .unwrap();
 
@@ -2390,15 +2394,15 @@ mod tests {
             execute: 10,
         };
         conn.blocks_dal()
-            .insert_l1_batch(&header, &[], predicted_gas, &[], &[])
+            .insert_l1_batch(&header, &[], predicted_gas, &[], &[], 0)
             .await
             .unwrap();
 
         header.number = L1BatchNumber(2);
         header.timestamp += 100;
         predicted_gas += predicted_gas;
         conn.blocks_dal()
-            .insert_l1_batch(&header, &[], predicted_gas, &[], &[])
+            .insert_l1_batch(&header, &[], predicted_gas, &[], &[], 0)
             .await
             .unwrap();
 

diff --git a/core/lib/dal/src/storage_logs_dal.rs b/core/lib/dal/src/storage_logs_dal.rs
@@ -733,7 +733,7 @@ mod tests {
         );
         header.is_finished = true;
         conn.blocks_dal()
-            .insert_l1_batch(&header, &[], BlockGasCount::default(), &[], &[])
+            .insert_l1_batch(&header, &[], BlockGasCount::default(), &[], &[], 0)
             .await
             .unwrap();
         conn.blocks_dal()

diff --git a/core/lib/dal/src/sync_dal.rs b/core/lib/dal/src/sync_dal.rs
@@ -136,7 +136,7 @@ mod tests {
             ProtocolVersionId::latest(),
         );
         conn.blocks_dal()
-            .insert_l1_batch(&l1_batch_header, &[], BlockGasCount::default(), &[], &[])
+            .insert_l1_batch(&l1_batch_header, &[], BlockGasCount::default(), &[], &[], 0)
             .await
             .unwrap();
         conn.blocks_dal()
@@ -205,7 +205,7 @@ mod tests {
         l1_batch_header.number = L1BatchNumber(1);
         l1_batch_header.timestamp = 1;
         conn.blocks_dal()
-            .insert_l1_batch(&l1_batch_header, &[], BlockGasCount::default(), &[], &[])
+            .insert_l1_batch(&l1_batch_header, &[], BlockGasCount::default(), &[], &[], 0)
             .await
             .unwrap();
         conn.blocks_dal()

diff --git a/core/lib/multivm/Cargo.toml b/core/lib/multivm/Cargo.toml
@@ -14,6 +14,8 @@ zk_evm_1_4_0 = { package = "zk_evm", git = "https://github.com/matter-labs/era-z
 zk_evm_1_3_3 = { package = "zk_evm", git = "https://github.com/matter-labs/era-zk_evm.git", tag= "v1.3.3-rc2" }
 zk_evm_1_3_1 = { package = "zk_evm", git = "https://github.com/matter-labs/era-zk_evm.git", tag= "v1.3.1-rc2" }
 
+zkevm_test_harness_1_4_0 = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.0", package = "zkevm_test_harness" }
+
 zksync_types = { path = "../types" }
 zksync_state = { path = "../state" }
 zksync_contracts = { path = "../contracts" }
@@ -29,7 +31,6 @@ thiserror = "1.0"
 tracing = "0.1"
 vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "1c9cc500e92cf9ea052b230e114a6f9cce4fb2c1" }
 
-
 [dev-dependencies]
 tokio = { version = "1", features = ["time"] }
 zksync_test_account = { path = "../test_account" }

diff --git a/core/lib/multivm/src/glue/types/vm/vm_block_result.rs b/core/lib/multivm/src/glue/types/vm/vm_block_result.rs
@@ -26,6 +26,7 @@ impl GlueFrom<crate::vm_m5::vm_instance::VmBlockResult> for crate::interface::Fi
                     computational_gas_used: value.full_result.gas_used,
                     gas_used: value.full_result.gas_used,
                     pubdata_published: 0,
+                    estimated_circuits_used: 0.0,
                 },
                 refunds: Refunds::default(),
             },
@@ -64,6 +65,7 @@ impl GlueFrom<crate::vm_m6::vm_instance::VmBlockResult> for crate::interface::Fi
                     computational_gas_used: value.full_result.computational_gas_used,
                     gas_used: value.full_result.gas_used,
                     pubdata_published: 0,
+                    estimated_circuits_used: 0.0,
                 },
                 refunds: Refunds::default(),
             },
@@ -108,6 +110,7 @@ impl GlueFrom<crate::vm_1_3_2::vm_instance::VmBlockResult> for crate::interface:
                     computational_gas_used: value.full_result.computational_gas_used,
                     gas_used: value.full_result.gas_used,
                     pubdata_published: 0,
+                    estimated_circuits_used: 0.0,
                 },
                 refunds: Refunds::default(),
             },
@@ -168,6 +171,7 @@ impl GlueFrom<crate::vm_1_3_2::vm_instance::VmBlockResult>
                 computational_gas_used: value.full_result.computational_gas_used,
                 gas_used: value.full_result.gas_used,
                 pubdata_published: 0,
+                estimated_circuits_used: 0.0,
             },
             refunds: Refunds::default(),
         }
@@ -198,6 +202,7 @@ impl GlueFrom<crate::vm_m5::vm_instance::VmBlockResult>
                 computational_gas_used: 0,
                 gas_used: value.full_result.gas_used,
                 pubdata_published: 0,
+                estimated_circuits_used: 0.0,
             },
             refunds: Refunds::default(),
         }
@@ -239,6 +244,7 @@ impl GlueFrom<crate::vm_m6::vm_instance::VmBlockResult>
                 computational_gas_used: value.full_result.computational_gas_used,
                 gas_used: value.full_result.gas_used,
                 pubdata_published: 0,
+                estimated_circuits_used: 0.0,
             },
             refunds: Refunds::default(),
         }

diff --git a/core/lib/multivm/src/glue/types/vm/vm_partial_execution_result.rs b/core/lib/multivm/src/glue/types/vm/vm_partial_execution_result.rs
@@ -16,6 +16,7 @@ impl GlueFrom<crate::vm_m5::vm_instance::VmPartialExecutionResult>
                 // There are no such fields in m5
                 computational_gas_used: 0,
                 pubdata_published: 0,
+                estimated_circuits_used: 0.0,
             },
             refunds: crate::interface::Refunds {
                 gas_refunded: 0,
@@ -39,6 +40,7 @@ impl GlueFrom<crate::vm_m6::vm_instance::VmPartialExecutionResult>
                 computational_gas_used: value.computational_gas_used,
                 total_log_queries: value.logs.total_log_queries_count,
                 pubdata_published: 0,
+                estimated_circuits_used: 0.0,
             },
             refunds: crate::interface::Refunds {
                 gas_refunded: 0,
@@ -62,6 +64,7 @@ impl GlueFrom<crate::vm_1_3_2::vm_instance::VmPartialExecutionResult>
                 computational_gas_used: value.computational_gas_used,
                 total_log_queries: value.logs.total_log_queries_count,
                 pubdata_published: 0,
+                estimated_circuits_used: 0.0,
             },
             refunds: crate::interface::Refunds {
                 gas_refunded: 0,

diff --git a/core/lib/multivm/src/interface/types/outputs/execution_result.rs b/core/lib/multivm/src/interface/types/outputs/execution_result.rs
@@ -101,6 +101,7 @@ impl VmExecutionResultAndLogs {
             cycles_used: self.statistics.cycles_used,
             computational_gas_used: self.statistics.computational_gas_used,
             pubdata_published: self.statistics.pubdata_published,
+            estimated_circuits_used: self.statistics.estimated_circuits_used,
         }
     }
 }
diff --git a/core/lib/multivm/src/interface/types/outputs/statistic.rs b/core/lib/multivm/src/interface/types/outputs/statistic.rs
@@ -12,6 +12,7 @@ pub struct VmExecutionStatistics {
     /// Number of log queries produced by the VM during the tx execution.
     pub total_log_queries: usize,
     pub pubdata_published: u32,
+    pub estimated_circuits_used: f32,
 }
 
 /// Oracle metrics of the VM.

diff --git a/core/lib/multivm/src/versions/vm_boojum_integration/implementation/execution.rs b/core/lib/multivm/src/versions/vm_boojum_integration/implementation/execution.rs
@@ -80,6 +80,7 @@ impl<S: WriteStorage, H: HistoryMode> Vm<S, H> {
             spent_pubdata_counter_before,
             pubdata_published,
             logs.total_log_queries_count,
+            tx_tracer.circuits_tracer.estimated_circuits_used,
         );
         let result = tx_tracer.result_tracer.into_result();
 

diff --git a/core/lib/multivm/src/versions/vm_boojum_integration/implementation/statistics.rs b/core/lib/multivm/src/versions/vm_boojum_integration/implementation/statistics.rs
@@ -24,6 +24,7 @@ impl<S: WriteStorage, H: HistoryMode> Vm<S, H> {
         spent_pubdata_counter_before: u32,
         pubdata_published: u32,
         total_log_queries_count: usize,
+        estimated_circuits_used: f32,
     ) -> VmExecutionStatistics {
         let computational_gas_used = self.calculate_computational_gas_used(
             tracer,
@@ -40,6 +41,7 @@ impl<S: WriteStorage, H: HistoryMode> Vm<S, H> {
             computational_gas_used,
             total_log_queries: total_log_queries_count,
             pubdata_published,
+            estimated_circuits_used,
         }
     }