From 55aca55aa29959d92da11a3a06dcac18697d626e Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:40:03 +0300 Subject: [PATCH 1/8] remove witness generator config --- .../config/src/configs/witness_generator.rs | 45 ------------ core/lib/config/src/testonly.rs | 29 -------- core/lib/env_config/src/lib.rs | 1 - core/lib/env_config/src/witness_generator.rs | 50 -------------- core/lib/protobuf_config/src/lib.rs | 1 - .../src/proto/witness_generator.proto | 20 ------ .../protobuf_config/src/witness_generator.rs | 68 ------------------- core/lib/zksync_core/src/proto/mod.proto | 54 +++++++-------- etc/env/base/witness_generator.toml | 7 -- 9 files changed, 26 insertions(+), 249 deletions(-) delete mode 100644 core/lib/config/src/configs/witness_generator.rs delete mode 100644 core/lib/env_config/src/witness_generator.rs delete mode 100644 core/lib/protobuf_config/src/proto/witness_generator.proto delete mode 100644 core/lib/protobuf_config/src/witness_generator.rs delete mode 100644 etc/env/base/witness_generator.toml diff --git a/core/lib/config/src/configs/witness_generator.rs b/core/lib/config/src/configs/witness_generator.rs deleted file mode 100644 index 010742e4b04..00000000000 --- a/core/lib/config/src/configs/witness_generator.rs +++ /dev/null @@ -1,45 +0,0 @@ -use std::time::Duration; - -// Built-in uses -// External uses -use serde::Deserialize; - -#[derive(Debug, Deserialize, Clone, PartialEq)] -pub enum BasicWitnessGeneratorDataSource { - FromPostgres, - FromPostgresShadowBlob, - FromBlob, -} - -/// Configuration for the witness generation -#[derive(Debug, Deserialize, Clone, PartialEq)] -pub struct WitnessGeneratorConfig { - /// Max time for witness to be generated - pub generation_timeout_in_secs: u16, - /// Currently only a single (largest) key is supported. - pub initial_setup_key_path: String, - /// https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key - pub key_download_url: String, - /// Max attempts for generating witness - pub max_attempts: u32, - // Percentage of the blocks that gets proven in the range [0.0, 1.0] - // when 0.0 implies all blocks are skipped and 1.0 implies all blocks are proven. - pub blocks_proving_percentage: Option, - pub dump_arguments_for_blocks: Vec, - // Optional l1 batch number to process block until(inclusive). - // This parameter is used in case of performing circuit upgrades(VK/Setup keys), - // to not let witness-generator pick new job and finish all the existing jobs with old circuit. - pub last_l1_batch_to_process: Option, - /// Where will basic Witness Generator load its data from - pub data_source: BasicWitnessGeneratorDataSource, -} - -impl WitnessGeneratorConfig { - pub fn witness_generation_timeout(&self) -> Duration { - Duration::from_secs(self.generation_timeout_in_secs as u64) - } - - pub fn last_l1_batch_to_process(&self) -> u32 { - self.last_l1_batch_to_process.unwrap_or(u32::MAX) - } -} diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 6ac8ef24993..e4daa8993bd 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -653,35 +653,6 @@ impl Distribution for EncodeDist { } } -impl Distribution for EncodeDist { - fn sample( - &self, - rng: &mut R, - ) -> configs::witness_generator::BasicWitnessGeneratorDataSource { - type T = configs::witness_generator::BasicWitnessGeneratorDataSource; - match rng.gen_range(0..2) { - 0 => T::FromPostgres, - 1 => T::FromPostgresShadowBlob, - _ => T::FromBlob, - } - } -} - -impl Distribution for EncodeDist { - fn sample(&self, rng: &mut R) -> configs::WitnessGeneratorConfig { - configs::WitnessGeneratorConfig { - generation_timeout_in_secs: self.sample(rng), - initial_setup_key_path: self.sample(rng), - key_download_url: self.sample(rng), - max_attempts: self.sample(rng), - blocks_proving_percentage: self.sample(rng), - dump_arguments_for_blocks: self.sample_collect(rng), - last_l1_batch_to_process: self.sample(rng), - data_source: self.sample(rng), - } - } -} - impl Distribution for EncodeDist { fn sample(&self, rng: &mut R) -> configs::ObservabilityConfig { configs::ObservabilityConfig { diff --git a/core/lib/env_config/src/lib.rs b/core/lib/env_config/src/lib.rs index edc768b0be0..739118c50ac 100644 --- a/core/lib/env_config/src/lib.rs +++ b/core/lib/env_config/src/lib.rs @@ -22,7 +22,6 @@ mod observability; mod proof_data_handler; mod snapshots_creator; mod utils; -mod witness_generator; mod genesis; #[cfg(test)] diff --git a/core/lib/env_config/src/witness_generator.rs b/core/lib/env_config/src/witness_generator.rs deleted file mode 100644 index 28d7a9bbbd9..00000000000 --- a/core/lib/env_config/src/witness_generator.rs +++ /dev/null @@ -1,50 +0,0 @@ -use zksync_config::configs::WitnessGeneratorConfig; - -use crate::{envy_load, FromEnv}; - -impl FromEnv for WitnessGeneratorConfig { - fn from_env() -> anyhow::Result { - envy_load("witness", "WITNESS_") - } -} - -#[cfg(test)] -mod tests { - use zksync_config::configs::witness_generator::BasicWitnessGeneratorDataSource; - - use super::*; - use crate::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> WitnessGeneratorConfig { - WitnessGeneratorConfig { - generation_timeout_in_secs: 900_u16, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - max_attempts: 4, - blocks_proving_percentage: Some(30), - dump_arguments_for_blocks: vec![2, 3], - last_l1_batch_to_process: None, - data_source: BasicWitnessGeneratorDataSource::FromBlob, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - WITNESS_GENERATION_TIMEOUT_IN_SECS=900 - WITNESS_INITIAL_SETUP_KEY_PATH="key" - WITNESS_KEY_DOWNLOAD_URL="value" - WITNESS_MAX_ATTEMPTS=4 - WITNESS_DUMP_ARGUMENTS_FOR_BLOCKS="2,3" - WITNESS_BLOCKS_PROVING_PERCENTAGE="30" - WITNESS_DATA_SOURCE="FromBlob" - "#; - lock.set_env(config); - - let actual = WitnessGeneratorConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/protobuf_config/src/lib.rs b/core/lib/protobuf_config/src/lib.rs index 573cc3ca056..2b09366ea80 100644 --- a/core/lib/protobuf_config/src/lib.rs +++ b/core/lib/protobuf_config/src/lib.rs @@ -24,7 +24,6 @@ mod object_store; mod observability; mod proof_data_handler; mod snapshots_creator; -mod witness_generator; pub mod proto; #[cfg(test)] diff --git a/core/lib/protobuf_config/src/proto/witness_generator.proto b/core/lib/protobuf_config/src/proto/witness_generator.proto deleted file mode 100644 index af1b7340cef..00000000000 --- a/core/lib/protobuf_config/src/proto/witness_generator.proto +++ /dev/null @@ -1,20 +0,0 @@ -syntax = "proto3"; - -package zksync.config.witness_generator; - -enum BasicWitnessGeneratorDataSource { - FROM_POSTGRES = 0; - FROM_POSTGRES_SHADOW_BLOB = 1; - FROM_BLOB = 2; -} - -message WitnessGenerator { - optional uint32 generation_timeout_in_secs = 1; // required; s - optional string initial_setup_key_path = 2; // required; fs path - optional string key_download_url = 3; // required; url - optional uint32 max_attempts = 4; // required - optional uint32 blocks_proving_percentage = 5; // optional; 0-100 (percentage) - repeated uint32 dump_arguments_for_blocks = 6; - optional uint32 last_l1_batch_to_process = 7; // optional - optional BasicWitnessGeneratorDataSource data_source = 8; // required -} diff --git a/core/lib/protobuf_config/src/witness_generator.rs b/core/lib/protobuf_config/src/witness_generator.rs deleted file mode 100644 index b21f2280072..00000000000 --- a/core/lib/protobuf_config/src/witness_generator.rs +++ /dev/null @@ -1,68 +0,0 @@ -use anyhow::Context as _; -use zksync_config::configs; -use zksync_protobuf::{repr::ProtoRepr, required}; - -use crate::proto::witness_generator as proto; - -impl proto::BasicWitnessGeneratorDataSource { - fn new(x: &configs::witness_generator::BasicWitnessGeneratorDataSource) -> Self { - type From = configs::witness_generator::BasicWitnessGeneratorDataSource; - match x { - From::FromPostgres => Self::FromPostgres, - From::FromPostgresShadowBlob => Self::FromPostgresShadowBlob, - From::FromBlob => Self::FromBlob, - } - } - fn parse(&self) -> configs::witness_generator::BasicWitnessGeneratorDataSource { - type To = configs::witness_generator::BasicWitnessGeneratorDataSource; - match self { - Self::FromPostgres => To::FromPostgres, - Self::FromPostgresShadowBlob => To::FromPostgresShadowBlob, - Self::FromBlob => To::FromBlob, - } - } -} - -impl ProtoRepr for proto::WitnessGenerator { - type Type = configs::WitnessGeneratorConfig; - fn read(&self) -> anyhow::Result { - Ok(Self::Type { - generation_timeout_in_secs: required(&self.generation_timeout_in_secs) - .and_then(|x| Ok((*x).try_into()?)) - .context("generation_timeout_in_secs")?, - initial_setup_key_path: required(&self.initial_setup_key_path) - .context("initial_setup_key_path")? - .clone(), - key_download_url: required(&self.key_download_url) - .context("key_download_url")? - .clone(), - max_attempts: *required(&self.max_attempts).context("max_attempts")?, - blocks_proving_percentage: self - .blocks_proving_percentage - .map(|x| x.try_into()) - .transpose() - .context("blocks_proving_percentage")?, - dump_arguments_for_blocks: self.dump_arguments_for_blocks.clone(), - last_l1_batch_to_process: self.last_l1_batch_to_process, - data_source: required(&self.data_source) - .and_then(|x| Ok(proto::BasicWitnessGeneratorDataSource::try_from(*x)?)) - .context("data_source")? - .parse(), - }) - } - - fn build(this: &Self::Type) -> Self { - Self { - generation_timeout_in_secs: Some(this.generation_timeout_in_secs.into()), - initial_setup_key_path: Some(this.initial_setup_key_path.clone()), - key_download_url: Some(this.key_download_url.clone()), - max_attempts: Some(this.max_attempts), - blocks_proving_percentage: this.blocks_proving_percentage.map(|x| x.into()), - dump_arguments_for_blocks: this.dump_arguments_for_blocks.clone(), - last_l1_batch_to_process: this.last_l1_batch_to_process, - data_source: Some( - proto::BasicWitnessGeneratorDataSource::new(&this.data_source).into(), - ), - } - } -} diff --git a/core/lib/zksync_core/src/proto/mod.proto b/core/lib/zksync_core/src/proto/mod.proto index a5ebe82970d..0a0342e968a 100644 --- a/core/lib/zksync_core/src/proto/mod.proto +++ b/core/lib/zksync_core/src/proto/mod.proto @@ -23,38 +23,36 @@ import "zksync/config/observability.proto"; import "zksync/config/proof_data_handler.proto"; import "zksync/config/snapshots_creator.proto"; import "zksync/config/utils.proto"; -import "zksync/config/witness_generator.proto"; import "zksync/core/consensus.proto"; message TempConfigStore { - optional config.database.Postgres postgres = 1; - optional config.api.HealthCheck health_check = 2; - optional config.api.MerkleTreeApi merkle_tree_api = 3; - optional config.api.Web3JsonRpc web3_json_rpc = 4; - optional config.chain.CircuitBreaker circuit_breaker = 5; - optional config.chain.Mempool mempool = 6; - optional config.chain.EthNetwork network = 7; - optional config.chain.OperationsManager operations_manager = 8; - optional config.chain.StateKeeper state_keeper = 9; - optional config.house_keeper.HouseKeeper house_keeper = 10; - optional config.fri_proof_compressor.FriProofCompressor fri_proof_compressor = 11; - optional config.fri_prover.FriProver fri_prover = 12; - optional config.fri_prover_group.FriProverGroup fri_prover_group = 13; - optional config.fri_witness_generator.FriWitnessGenerator fri_witness_generator = 14; - optional config.utils.Prometheus prometheus = 15; - optional config.proof_data_handler.ProofDataHandler proof_data_handler = 16; - optional config.witness_generator.WitnessGenerator witness_generator = 17; - optional config.api.Api api = 18; - optional config.contracts.Contracts contracts = 19; - optional config.database.DB db = 20; - optional config.eth_client.ETHClient eth_client = 21; - optional config.eth_sender.ETHSender eth_sender = 22; - optional config.eth_watch.ETHWatch eth_watch = 23; - optional config.eth_sender.GasAdjuster gas_adjuster = 24; - optional config.object_store.ObjectStore object_store = 25; - optional consensus.Config consensus = 26; + optional config.database.Postgres postgres = 1; + optional config.api.HealthCheck health_check = 2; + optional config.api.MerkleTreeApi merkle_tree_api = 3; + optional config.api.Web3JsonRpc web3_json_rpc = 4; + optional config.chain.CircuitBreaker circuit_breaker = 5; + optional config.chain.Mempool mempool = 6; + optional config.chain.EthNetwork network = 7; + optional config.chain.OperationsManager operations_manager = 8; + optional config.chain.StateKeeper state_keeper = 9; + optional config.house_keeper.HouseKeeper house_keeper = 10; + optional config.fri_proof_compressor.FriProofCompressor fri_proof_compressor = 11; + optional config.fri_prover.FriProver fri_prover = 12; + optional config.fri_prover_group.FriProverGroup fri_prover_group = 13; + optional config.fri_witness_generator.FriWitnessGenerator fri_witness_generator = 14; + optional config.utils.Prometheus prometheus = 15; + optional config.proof_data_handler.ProofDataHandler proof_data_handler = 16; + optional config.api.Api api = 18; + optional config.contracts.Contracts contracts = 19; + optional config.database.DB db = 20; + optional config.eth_client.ETHClient eth_client = 21; + optional config.eth_sender.ETHSender eth_sender = 22; + optional config.eth_watch.ETHWatch eth_watch = 23; + optional config.eth_sender.GasAdjuster gas_adjuster = 24; + optional config.object_store.ObjectStore object_store = 25; + optional consensus.Config consensus = 26; } message Secrets { - optional consensus.Secrets consensus = 1; // optional + optional consensus.Secrets consensus = 1; // optional } diff --git a/etc/env/base/witness_generator.toml b/etc/env/base/witness_generator.toml deleted file mode 100644 index c5df54919ad..00000000000 --- a/etc/env/base/witness_generator.toml +++ /dev/null @@ -1,7 +0,0 @@ -[witness] -generation_timeout_in_secs=900 -initial_setup_key_path="./keys/setup/setup_2^22.key" -key_download_url="https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2^22.key" -max_attempts=1 -dump_arguments_for_blocks="2,3" -data_source="FromPostgres" \ No newline at end of file From 8ee49576392189addc7f384f1e37b0eb196b9dc6 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:40:16 +0300 Subject: [PATCH 2/8] remove witness generator config --- core/lib/config/src/configs/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index 93116151d7a..d4893014241 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -19,7 +19,6 @@ pub use self::{ proof_data_handler::ProofDataHandlerConfig, snapshots_creator::SnapshotsCreatorConfig, utils::PrometheusConfig, - witness_generator::WitnessGeneratorConfig, }; pub mod alerts; @@ -44,6 +43,5 @@ pub mod observability; pub mod proof_data_handler; pub mod snapshots_creator; pub mod utils; -pub mod witness_generator; const BYTES_IN_MEGABYTE: usize = 1_024 * 1_024; From 252a6ec6954de0ee38fe166b2fd0d6ef7aa13034 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:47:01 +0300 Subject: [PATCH 3/8] return deleted file back --- core/lib/protobuf_config/src/proto/mod.proto | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 core/lib/protobuf_config/src/proto/mod.proto diff --git a/core/lib/protobuf_config/src/proto/mod.proto b/core/lib/protobuf_config/src/proto/mod.proto new file mode 100644 index 00000000000..7876c28c641 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/mod.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package zksync.core; + +import "zksync/core/consensus.proto"; + +message Secrets { + optional consensus.Secrets consensus = 1; // optional +} From 028be504d53d623027069e33fe6d0609dfdfb485 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:31:59 +0300 Subject: [PATCH 4/8] some cleanups + remove other unused vars --- core/lib/config/src/configs/fri_prover.rs | 3 - core/lib/config/src/testonly.rs | 3 - ...be812a6d7e76df4e545d639a564992f12bbe1.json | 100 -------------- core/lib/env_config/src/fri_prover.rs | 6 - .../protobuf_config/src/proto/prover.proto | 128 +++++++++--------- core/lib/protobuf_config/src/prover.rs | 28 ---- etc/env/base/fri_prover.toml | 23 ++-- etc/env/file_based/general.yaml | 3 - 8 files changed, 73 insertions(+), 221 deletions(-) delete mode 100644 core/lib/dal/.sqlx/query-37069b0fbe07f12f6ac93d434b7be812a6d7e76df4e545d639a564992f12bbe1.json diff --git a/core/lib/config/src/configs/fri_prover.rs b/core/lib/config/src/configs/fri_prover.rs index 958b8c5fec2..5921b03ca9d 100644 --- a/core/lib/config/src/configs/fri_prover.rs +++ b/core/lib/config/src/configs/fri_prover.rs @@ -17,11 +17,8 @@ pub struct FriProverConfig { pub prometheus_port: u16, pub max_attempts: u32, pub generation_timeout_in_secs: u16, - pub base_layer_circuit_ids_to_be_verified: Vec, - pub recursive_layer_circuit_ids_to_be_verified: Vec, pub setup_load_mode: SetupLoadMode, pub specialized_group_id: u8, - pub witness_vector_generator_thread_count: Option, pub queue_capacity: usize, pub witness_vector_receiver_port: u16, pub zone_read_url: String, diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 5e37a9c39f0..8c030329fa7 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -421,11 +421,8 @@ impl Distribution for EncodeDist { prometheus_port: self.sample(rng), max_attempts: self.sample(rng), generation_timeout_in_secs: self.sample(rng), - base_layer_circuit_ids_to_be_verified: self.sample_collect(rng), - recursive_layer_circuit_ids_to_be_verified: self.sample_collect(rng), setup_load_mode: self.sample(rng), specialized_group_id: self.sample(rng), - witness_vector_generator_thread_count: self.sample(rng), queue_capacity: self.sample(rng), witness_vector_receiver_port: self.sample(rng), zone_read_url: self.sample(rng), diff --git a/core/lib/dal/.sqlx/query-37069b0fbe07f12f6ac93d434b7be812a6d7e76df4e545d639a564992f12bbe1.json b/core/lib/dal/.sqlx/query-37069b0fbe07f12f6ac93d434b7be812a6d7e76df4e545d639a564992f12bbe1.json deleted file mode 100644 index a830db9f4c9..00000000000 --- a/core/lib/dal/.sqlx/query-37069b0fbe07f12f6ac93d434b7be812a6d7e76df4e545d639a564992f12bbe1.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n miniblocks.number,\n COALESCE(\n miniblocks.l1_batch_number,\n (\n SELECT\n (MAX(number) + 1)\n FROM\n l1_batches\n ),\n (\n SELECT\n MAX(l1_batch_number) + 1\n FROM\n snapshot_recovery\n )\n ) AS \"l1_batch_number!\",\n (\n SELECT\n MAX(m2.number)\n FROM\n miniblocks m2\n WHERE\n miniblocks.l1_batch_number = m2.l1_batch_number\n ) AS \"last_batch_miniblock?\",\n miniblocks.timestamp,\n miniblocks.l1_gas_price,\n miniblocks.l2_fair_gas_price,\n miniblocks.fair_pubdata_price,\n miniblocks.bootloader_code_hash,\n miniblocks.default_aa_code_hash,\n miniblocks.virtual_blocks,\n miniblocks.hash,\n miniblocks.gas_limit,\n miniblocks.protocol_version AS \"protocol_version!\",\n miniblocks.fee_account_address AS \"fee_account_address!\"\n FROM\n miniblocks\n WHERE\n miniblocks.number = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "number", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "l1_batch_number!", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "last_batch_miniblock?", - "type_info": "Int8" - }, - { - "ordinal": 3, - "name": "timestamp", - "type_info": "Int8" - }, - { - "ordinal": 4, - "name": "l1_gas_price", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "l2_fair_gas_price", - "type_info": "Int8" - }, - { - "ordinal": 6, - "name": "fair_pubdata_price", - "type_info": "Int8" - }, - { - "ordinal": 7, - "name": "bootloader_code_hash", - "type_info": "Bytea" - }, - { - "ordinal": 8, - "name": "default_aa_code_hash", - "type_info": "Bytea" - }, - { - "ordinal": 9, - "name": "virtual_blocks", - "type_info": "Int8" - }, - { - "ordinal": 10, - "name": "hash", - "type_info": "Bytea" - }, - { - "ordinal": 11, - "name": "gas_limit", - "type_info": "Int8" - }, - { - "ordinal": 12, - "name": "protocol_version!", - "type_info": "Int4" - }, - { - "ordinal": 13, - "name": "fee_account_address!", - "type_info": "Bytea" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false, - null, - null, - false, - false, - false, - true, - true, - true, - false, - false, - true, - true, - false - ] - }, - "hash": "37069b0fbe07f12f6ac93d434b7be812a6d7e76df4e545d639a564992f12bbe1" -} diff --git a/core/lib/env_config/src/fri_prover.rs b/core/lib/env_config/src/fri_prover.rs index f888436dc71..a7121f2f15c 100644 --- a/core/lib/env_config/src/fri_prover.rs +++ b/core/lib/env_config/src/fri_prover.rs @@ -28,11 +28,8 @@ mod tests { prometheus_port: 3315, max_attempts: 10, generation_timeout_in_secs: 300, - base_layer_circuit_ids_to_be_verified: vec![1, 5], - recursive_layer_circuit_ids_to_be_verified: vec![1, 2, 3], setup_load_mode: SetupLoadMode::FromDisk, specialized_group_id: 10, - witness_vector_generator_thread_count: Some(5), queue_capacity: 10, witness_vector_receiver_port: 3316, zone_read_url: "http://metadata.google.internal/computeMetadata/v1/instance/zone" @@ -56,11 +53,8 @@ mod tests { FRI_PROVER_PROMETHEUS_PORT="3315" FRI_PROVER_MAX_ATTEMPTS="10" FRI_PROVER_GENERATION_TIMEOUT_IN_SECS="300" - FRI_PROVER_BASE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,5" - FRI_PROVER_RECURSIVE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,2,3" FRI_PROVER_SETUP_LOAD_MODE="FromDisk" FRI_PROVER_SPECIALIZED_GROUP_ID="10" - FRI_PROVER_WITNESS_VECTOR_GENERATOR_THREAD_COUNT="5" FRI_PROVER_QUEUE_CAPACITY="10" FRI_PROVER_WITNESS_VECTOR_RECEIVER_PORT="3316" FRI_PROVER_ZONE_READ_URL="http://metadata.google.internal/computeMetadata/v1/instance/zone" diff --git a/core/lib/protobuf_config/src/proto/prover.proto b/core/lib/protobuf_config/src/proto/prover.proto index da1f295a582..939f617e471 100644 --- a/core/lib/protobuf_config/src/proto/prover.proto +++ b/core/lib/protobuf_config/src/proto/prover.proto @@ -1,99 +1,97 @@ syntax = "proto3"; + import "zksync/config/object_store.proto"; package zksync.config.prover; message ProofCompressor { - optional uint32 compression_mode = 1; // required; u8 - optional uint32 prometheus_listener_port = 2; // required; u16 - optional string prometheus_pushgateway_url = 3; // required - optional uint64 prometheus_push_interval_ms = 4; // optional; ms - optional uint32 generation_timeout_in_secs = 5; // required; s - optional uint32 max_attempts = 6; // required - optional string universal_setup_path = 7; // required; fs path - optional string universal_setup_download_url = 8; // required - optional bool verify_wrapper_proof = 9; // required + optional uint32 compression_mode = 1; // required; u8 + optional uint32 prometheus_listener_port = 2; // required; u16 + optional string prometheus_pushgateway_url = 3; // required + optional uint64 prometheus_push_interval_ms = 4; // optional; ms + optional uint32 generation_timeout_in_secs = 5; // required; s + optional uint32 max_attempts = 6; // required + optional string universal_setup_path = 7; // required; fs path + optional string universal_setup_download_url = 8; // required + optional bool verify_wrapper_proof = 9; // required } enum SetupLoadMode { - FROM_DISK = 0; - FROM_MEMORY = 1; + FROM_DISK = 0; + FROM_MEMORY = 1; } message Prover { - optional string setup_data_path = 1; // required; fs path? - optional uint32 prometheus_port = 2; // required; u16 - optional uint32 max_attempts = 3; // required - optional uint32 generation_timeout_in_secs = 4; // required; s - repeated uint32 base_layer_circuit_ids_to_be_verified = 5; // required - repeated uint32 recursive_layer_circuit_ids_to_be_verified = 6; // required - optional SetupLoadMode setup_load_mode = 7; // required - optional uint32 specialized_group_id = 8; // required; u8 - optional uint64 witness_vector_generator_thread_count = 9; // optional - optional uint64 queue_capacity = 10; // required - optional uint32 witness_vector_receiver_port = 11; // required; u16 - optional string zone_read_url = 12; // required - optional bool shall_save_to_public_bucket = 13; // required - optional config.object_store.ObjectStore object_store = 20; + optional string setup_data_path = 1; // required; fs path? + optional uint32 prometheus_port = 2; // required; u16 + optional uint32 max_attempts = 3; // required + optional uint32 generation_timeout_in_secs = 4; // required; s + optional SetupLoadMode setup_load_mode = 7; // required + optional uint32 specialized_group_id = 8; // required; u8 + optional uint64 queue_capacity = 10; // required + optional uint32 witness_vector_receiver_port = 11; // required; u16 + optional string zone_read_url = 12; // required + optional bool shall_save_to_public_bucket = 13; // required + optional config.object_store.ObjectStore object_store = 20; } message CircuitIdRoundTuple { - optional uint32 circuit_id = 1; // required; u8 - optional uint32 aggregation_round = 2; // required; u8 + optional uint32 circuit_id = 1; // required; u8 + optional uint32 aggregation_round = 2; // required; u8 } message ProverGroup { - repeated CircuitIdRoundTuple group_0 = 1; - repeated CircuitIdRoundTuple group_1 = 2; - repeated CircuitIdRoundTuple group_2 = 3; - repeated CircuitIdRoundTuple group_3 = 4; - repeated CircuitIdRoundTuple group_4 = 5; - repeated CircuitIdRoundTuple group_5 = 6; - repeated CircuitIdRoundTuple group_6 = 7; - repeated CircuitIdRoundTuple group_7 = 8; - repeated CircuitIdRoundTuple group_8 = 9; - repeated CircuitIdRoundTuple group_9 = 10; - repeated CircuitIdRoundTuple group_10 = 11; - repeated CircuitIdRoundTuple group_11 = 12; - repeated CircuitIdRoundTuple group_12 = 13; + repeated CircuitIdRoundTuple group_0 = 1; + repeated CircuitIdRoundTuple group_1 = 2; + repeated CircuitIdRoundTuple group_2 = 3; + repeated CircuitIdRoundTuple group_3 = 4; + repeated CircuitIdRoundTuple group_4 = 5; + repeated CircuitIdRoundTuple group_5 = 6; + repeated CircuitIdRoundTuple group_6 = 7; + repeated CircuitIdRoundTuple group_7 = 8; + repeated CircuitIdRoundTuple group_8 = 9; + repeated CircuitIdRoundTuple group_9 = 10; + repeated CircuitIdRoundTuple group_10 = 11; + repeated CircuitIdRoundTuple group_11 = 12; + repeated CircuitIdRoundTuple group_12 = 13; } message ProverGateway { - optional string api_url = 1; // required - optional uint32 api_poll_duration_secs = 2; // required; s - optional uint32 prometheus_listener_port = 3; // required; u16 - optional string prometheus_pushgateway_url = 4; // required - optional uint64 prometheus_push_interval_ms = 5; // optional; ms + optional string api_url = 1; // required + optional uint32 api_poll_duration_secs = 2; // required; s + optional uint32 prometheus_listener_port = 3; // required; u16 + optional string prometheus_pushgateway_url = 4; // required + optional uint64 prometheus_push_interval_ms = 5; // optional; ms } message WitnessGenerator { - optional uint32 generation_timeout_in_secs = 1; // required; - optional uint32 max_attempts = 2; // required - optional uint32 blocks_proving_percentage = 3; // optional; 0-100 - repeated uint32 dump_arguments_for_blocks = 4; - optional uint32 last_l1_batch_to_process = 5; // optional - optional uint32 force_process_block = 6; // optional - optional bool shall_save_to_public_bucket = 7; // required - optional uint32 basic_generation_timeout_in_secs = 8; // optional; - optional uint32 leaf_generation_timeout_in_secs = 9; // optional; - optional uint32 node_generation_timeout_in_secs = 10; // optional; - optional uint32 scheduler_generation_timeout_in_secs = 11; // optional; + optional uint32 generation_timeout_in_secs = 1; // required; + optional uint32 max_attempts = 2; // required + optional uint32 blocks_proving_percentage = 3; // optional; 0-100 + repeated uint32 dump_arguments_for_blocks = 4; + optional uint32 last_l1_batch_to_process = 5; // optional + optional uint32 force_process_block = 6; // optional + optional bool shall_save_to_public_bucket = 7; // required + optional uint32 basic_generation_timeout_in_secs = 8; // optional; + optional uint32 leaf_generation_timeout_in_secs = 9; // optional; + optional uint32 node_generation_timeout_in_secs = 10; // optional; + optional uint32 scheduler_generation_timeout_in_secs = 11; // optional; } message WitnessVectorGenerator { - optional uint32 max_prover_reservation_duration_in_secs = 1; // required; s - optional uint32 prover_instance_wait_timeout_in_secs = 2; // required; s - optional uint32 prover_instance_poll_time_in_milli_secs = 3; // required; ms - optional uint32 prometheus_listener_port = 4; // required; u16 - optional string prometheus_pushgateway_url = 5; // required - optional uint64 prometheus_push_interval_ms = 6; // optional; ms - optional uint32 specialized_group_id = 7; // required; u8 + optional uint32 max_prover_reservation_duration_in_secs = 1; // required; s + optional uint32 prover_instance_wait_timeout_in_secs = 2; // required; s + optional uint32 prover_instance_poll_time_in_milli_secs = 3; // required; ms + optional uint32 prometheus_listener_port = 4; // required; u16 + optional string prometheus_pushgateway_url = 5; // required + optional uint64 prometheus_push_interval_ms = 6; // optional; ms + optional uint32 specialized_group_id = 7; // required; u8 } message ProofDataHandler { - optional uint32 http_port = 1; // required; u16 - optional uint32 proof_generation_timeout_in_secs = 2; // required; s + optional uint32 http_port = 1; // required; u16 + optional uint32 proof_generation_timeout_in_secs = 2; // required; s } diff --git a/core/lib/protobuf_config/src/prover.rs b/core/lib/protobuf_config/src/prover.rs index 3dd582fb206..985ed235620 100644 --- a/core/lib/protobuf_config/src/prover.rs +++ b/core/lib/protobuf_config/src/prover.rs @@ -304,16 +304,6 @@ impl ProtoRepr for proto::Prover { generation_timeout_in_secs: required(&self.generation_timeout_in_secs) .and_then(|x| Ok((*x).try_into()?)) .context("generation_timeout_in_secs")?, - base_layer_circuit_ids_to_be_verified: self - .base_layer_circuit_ids_to_be_verified - .iter() - .map(|a| *a as u8) - .collect(), - recursive_layer_circuit_ids_to_be_verified: self - .recursive_layer_circuit_ids_to_be_verified - .iter() - .map(|a| *a as u8) - .collect(), setup_load_mode: required(&self.setup_load_mode) .and_then(|x| Ok(proto::SetupLoadMode::try_from(*x)?)) .context("setup_load_mode")? @@ -321,11 +311,6 @@ impl ProtoRepr for proto::Prover { specialized_group_id: required(&self.specialized_group_id) .and_then(|x| Ok((*x).try_into()?)) .context("specialized_group_id")?, - witness_vector_generator_thread_count: self - .witness_vector_generator_thread_count - .map(|x| x.try_into()) - .transpose() - .context("witness_vector_generator_thread_count")?, queue_capacity: required(&self.queue_capacity) .and_then(|x| Ok((*x).try_into()?)) .context("queue_capacity")?, @@ -347,21 +332,8 @@ impl ProtoRepr for proto::Prover { prometheus_port: Some(this.prometheus_port.into()), max_attempts: Some(this.max_attempts), generation_timeout_in_secs: Some(this.generation_timeout_in_secs.into()), - base_layer_circuit_ids_to_be_verified: this - .base_layer_circuit_ids_to_be_verified - .iter() - .map(|a| *a as u32) - .collect(), - recursive_layer_circuit_ids_to_be_verified: this - .recursive_layer_circuit_ids_to_be_verified - .iter() - .map(|a| *a as u32) - .collect(), setup_load_mode: Some(proto::SetupLoadMode::new(&this.setup_load_mode).into()), specialized_group_id: Some(this.specialized_group_id.into()), - witness_vector_generator_thread_count: this - .witness_vector_generator_thread_count - .map(|x| x.try_into().unwrap()), queue_capacity: Some(this.queue_capacity.try_into().unwrap()), witness_vector_receiver_port: Some(this.witness_vector_receiver_port.into()), zone_read_url: Some(this.zone_read_url.clone()), diff --git a/etc/env/base/fri_prover.toml b/etc/env/base/fri_prover.toml index 94af27417ae..8fd3ad49b57 100644 --- a/etc/env/base/fri_prover.toml +++ b/etc/env/base/fri_prover.toml @@ -1,14 +1,11 @@ [fri_prover] -setup_data_path="/usr/src/setup-data" -prometheus_port=3315 -max_attempts=10 -generation_timeout_in_secs=600 -base_layer_circuit_ids_to_be_verified="1" -recursive_layer_circuit_ids_to_be_verified="1" -setup_load_mode="FromDisk" -specialized_group_id=100 -witness_vector_generator_thread_count=5 -queue_capacity=10 -witness_vector_receiver_port=3316 -zone_read_url="http://metadata.google.internal/computeMetadata/v1/instance/zone" -shall_save_to_public_bucket=true +setup_data_path = "/usr/src/setup-data" +prometheus_port = 3315 +max_attempts = 10 +generation_timeout_in_secs = 600 +setup_load_mode = "FromDisk" +specialized_group_id = 100 +queue_capacity = 10 +witness_vector_receiver_port = 3316 +zone_read_url = "http://metadata.google.internal/computeMetadata/v1/instance/zone" +shall_save_to_public_bucket = true diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index eb52db083a1..21337350535 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -163,11 +163,8 @@ prover: prometheus_port: 3315 max_attempts: 10 generation_timeout_in_secs: 600 - base_layer_circuit_ids_to_be_verified: [ 1 ] - recursive_layer_circuit_ids_to_be_verified: [ 1 ] setup_load_mode: FROM_DISK specialized_group_id: 100 - witness_vector_generator_thread_count: 5 queue_capacity: 10 witness_vector_receiver_port: 3316 zone_read_url: http://metadata.google.internal/computeMetadata/v1/instance/zone From 26e0f321bd1cc8416d0cf5b5e595d4b6b029a476 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:40:30 +0300 Subject: [PATCH 5/8] remove wg.toml --- etc/env/dev.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/env/dev.toml b/etc/env/dev.toml index e95ad7b9e9e..b50b6ecc5bf 100644 --- a/etc/env/dev.toml +++ b/etc/env/dev.toml @@ -14,7 +14,6 @@ base = [ 'base/nfs.toml', 'base/rust.toml', 'base/private.toml', - 'base/witness_generator.toml', 'base/house_keeper.toml', 'base/fri_prover.toml', 'base/fri_witness_generator.toml', From 1e0b23bdace81bf0985efa9e61dd9a5b2c759252 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Tue, 9 Apr 2024 16:47:40 +0300 Subject: [PATCH 6/8] fix build --- core/lib/protobuf_config/src/proto/mod.proto | 9 --------- core/lib/protobuf_config/src/proto/prover.proto | 2 +- core/lib/zksync_core/src/temp_config_store/mod.rs | 3 +-- 3 files changed, 2 insertions(+), 12 deletions(-) delete mode 100644 core/lib/protobuf_config/src/proto/mod.proto diff --git a/core/lib/protobuf_config/src/proto/mod.proto b/core/lib/protobuf_config/src/proto/mod.proto deleted file mode 100644 index 7876c28c641..00000000000 --- a/core/lib/protobuf_config/src/proto/mod.proto +++ /dev/null @@ -1,9 +0,0 @@ -syntax = "proto3"; - -package zksync.core; - -import "zksync/core/consensus.proto"; - -message Secrets { - optional consensus.Secrets consensus = 1; // optional -} diff --git a/core/lib/protobuf_config/src/proto/prover.proto b/core/lib/protobuf_config/src/proto/prover.proto index c14c003225c..b0ecbbcbe37 100644 --- a/core/lib/protobuf_config/src/proto/prover.proto +++ b/core/lib/protobuf_config/src/proto/prover.proto @@ -32,7 +32,7 @@ message Prover { optional uint32 witness_vector_receiver_port = 11; // required; u16 optional string zone_read_url = 12; // required optional uint32 availability_check_interval_in_secs = 21; // required; s - optionalbool shall_save_to_public_bucket = 13; // required + optional bool shall_save_to_public_bucket = 13; // required optional config.object_store.ObjectStore object_store = 20; } diff --git a/core/lib/zksync_core/src/temp_config_store/mod.rs b/core/lib/zksync_core/src/temp_config_store/mod.rs index 2c5769f1f86..f471a7b251c 100644 --- a/core/lib/zksync_core/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core/src/temp_config_store/mod.rs @@ -11,7 +11,7 @@ use zksync_config::{ wallets::{AddressWallet, EthSender, StateKeeper, Wallet, Wallets}, FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, GeneralConfig, - ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, WitnessGeneratorConfig, + ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, ETHConfig, ETHWatchConfig, GasAdjusterConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig, @@ -56,7 +56,6 @@ pub struct TempConfigStore { pub fri_witness_generator_config: Option, pub prometheus_config: Option, pub proof_data_handler_config: Option, - pub witness_generator_config: Option, pub api_config: Option, pub db_config: Option, pub eth_sender_config: Option, From cd174004aca037cc676d25f7d15d613bb06c2342 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Tue, 9 Apr 2024 17:01:31 +0300 Subject: [PATCH 7/8] fix build --- core/bin/zksync_server/src/main.rs | 3 +-- core/lib/zksync_core/src/proto/mod.proto | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 core/lib/zksync_core/src/proto/mod.proto diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs index b6d6a3e7af8..1d8daf1b857 100644 --- a/core/bin/zksync_server/src/main.rs +++ b/core/bin/zksync_server/src/main.rs @@ -13,7 +13,7 @@ use zksync_config::{ house_keeper::HouseKeeperConfig, ContractsConfig, FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, ObservabilityConfig, - PrometheusConfig, ProofDataHandlerConfig, WitnessGeneratorConfig, + PrometheusConfig, ProofDataHandlerConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, ETHConfig, ETHWatchConfig, GasAdjusterConfig, GenesisConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig, @@ -269,7 +269,6 @@ fn load_env_config() -> anyhow::Result { fri_witness_generator_config: FriWitnessGeneratorConfig::from_env().ok(), prometheus_config: PrometheusConfig::from_env().ok(), proof_data_handler_config: ProofDataHandlerConfig::from_env().ok(), - witness_generator_config: WitnessGeneratorConfig::from_env().ok(), api_config: ApiConfig::from_env().ok(), db_config: DBConfig::from_env().ok(), eth_sender_config: ETHConfig::from_env().ok(), diff --git a/core/lib/zksync_core/src/proto/mod.proto b/core/lib/zksync_core/src/proto/mod.proto new file mode 100644 index 00000000000..7876c28c641 --- /dev/null +++ b/core/lib/zksync_core/src/proto/mod.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package zksync.core; + +import "zksync/core/consensus.proto"; + +message Secrets { + optional consensus.Secrets consensus = 1; // optional +} From 8ef0e357dbdef65d28a2357a673ab5d66dc72f21 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Tue, 9 Apr 2024 17:21:13 +0300 Subject: [PATCH 8/8] add reserved fields to protobuf --- core/lib/protobuf_config/src/proto/prover.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/core/lib/protobuf_config/src/proto/prover.proto b/core/lib/protobuf_config/src/proto/prover.proto index b0ecbbcbe37..a365a2e2886 100644 --- a/core/lib/protobuf_config/src/proto/prover.proto +++ b/core/lib/protobuf_config/src/proto/prover.proto @@ -34,6 +34,7 @@ message Prover { optional uint32 availability_check_interval_in_secs = 21; // required; s optional bool shall_save_to_public_bucket = 13; // required optional config.object_store.ObjectStore object_store = 20; + reserved 5, 6, 9; reserved "base_layer_circuit_ids_to_be_verified", "recursive_layer_circuit_ids_to_be_verified", "witness_vector_generator_thread_count"; }