Skip to content

Commit

Permalink
I do am have stupid
Browse files Browse the repository at this point in the history
  • Loading branch information
popzxc committed Nov 6, 2023
1 parent 63321bc commit a8713c8
Show file tree
Hide file tree
Showing 19 changed files with 90 additions and 58 deletions.
2 changes: 1 addition & 1 deletion core/lib/env_config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ mod fri_prover_group;
mod fri_witness_generator;
mod fri_witness_vector_generator;
mod house_keeper;
mod object_store;
pub mod object_store;
mod proof_data_handler;
mod prover;
mod prover_group;
Expand Down
2 changes: 1 addition & 1 deletion prover/circuit_synthesizer/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ zksync_dal = { path = "../../core/lib/dal" }
zksync_types = { path = "../../core/lib/types" }
zksync_queued_job_processor = { path = "../../core/lib/queued_job_processor" }
zksync_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/env_config" }
zksync_object_store = { path = "../../core/lib/object_store" }
zksync_utils = { path = "../../core/lib/utils" }
zksync_prover_fri_utils = { path = "../prover_fri_utils" }
Expand Down
4 changes: 2 additions & 2 deletions prover/circuit_synthesizer/src/circuit_synthesizer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,10 @@ impl CircuitSynthesizer {
blob_store: store_factory.create_store().await,
allowed_circuit_types: allowed_circuit_types
.map(|x| x.into_iter().map(|x| x.1).collect()),
region: get_region()
region: get_region(&prover_groups)
.await
.map_err(CircuitSynthesizerError::GetRegionFailed)?,
zone: get_zone()
zone: get_zone(&prover_groups)
.await
.map_err(CircuitSynthesizerError::GetZoneFailed)?,
vk_commitments,
Expand Down
8 changes: 6 additions & 2 deletions prover/circuit_synthesizer/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ use prometheus_exporter::PrometheusExporterConfig;
use structopt::StructOpt;
use tokio::{sync::oneshot, sync::watch};

use zksync_config::configs::{AlertsConfig, CircuitSynthesizerConfig, ProverGroupConfig};
use zksync_config::configs::{
AlertsConfig, CircuitSynthesizerConfig, ObjectStoreConfig, ProverGroupConfig,
};
use zksync_dal::connection::DbVariant;
use zksync_dal::ConnectionPool;
use zksync_env_config::FromEnv;
Expand Down Expand Up @@ -51,10 +53,12 @@ async fn main() -> anyhow::Result<()> {
.context("failed to build a connection pool")?;
let vk_commitments = get_cached_commitments();

let object_store_config =
ObjectStoreConfig::from_env().context("ObjectStoreConfig::from_env()")?;
let circuit_synthesizer = CircuitSynthesizer::new(
config.clone(),
ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?,
&ObjectStoreFactory::from_env().context("ObjectStoreFactory::from_env()")?,
&ObjectStoreFactory::new(object_store_config),
vk_commitments,
pool,
)
Expand Down
2 changes: 1 addition & 1 deletion prover/proof_fri_compressor/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ edition = "2021"
zksync_types = { path = "../../core/lib/types" }
zksync_dal = { path = "../../core/lib/dal" }
zksync_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/env_config" }
zksync_object_store = { path = "../../core/lib/object_store" }
zksync_utils = { path = "../../core/lib/utils" }
prometheus_exporter = { path = "../../core/lib/prometheus_exporter" }
Expand Down
7 changes: 4 additions & 3 deletions prover/proof_fri_compressor/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use prometheus_exporter::PrometheusExporterConfig;
use zksync_config::configs::FriProofCompressorConfig;
use zksync_dal::connection::DbVariant;
use zksync_dal::ConnectionPool;
use zksync_env_config::FromEnv;
use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv};
use zksync_object_store::ObjectStoreFactory;
use zksync_queued_job_processor::JobProcessor;
use zksync_utils::wait_for_tasks::wait_for_tasks;
Expand Down Expand Up @@ -53,8 +53,9 @@ async fn main() -> anyhow::Result<()> {
.build()
.await
.context("failed to build a connection pool")?;
let blob_store = ObjectStoreFactory::prover_from_env()
.context("ObjectSToreFactor::prover_from_env()")?
let object_store_config =
ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?;
let blob_store = ObjectStoreFactory::new(object_store_config.0)
.create_store()
.await;
let proof_compressor = ProofCompressor::new(
Expand Down
2 changes: 1 addition & 1 deletion prover/prover/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ publish = false # We don't want to publish our binaries.
[dependencies]
zksync_dal = { path = "../../core/lib/dal" }
zksync_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/env_config" }
zksync_utils = { path = "../../core/lib/utils" }
zksync_prover_utils = { path = "../../core/lib/prover_utils" }
zksync_circuit_breaker = { path = "../../core/lib/circuit_breaker" }
Expand Down
29 changes: 20 additions & 9 deletions prover/prover/src/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ use queues::Buffer;

use prometheus_exporter::PrometheusExporterConfig;
use zksync_config::{
configs::{api::PrometheusConfig, prover_group::ProverGroupConfig, AlertsConfig},
configs::{
api::PrometheusConfig, prover_group::ProverGroupConfig, AlertsConfig, ObjectStoreConfig,
},
ApiConfig, ProverConfig, ProverConfigs,
};
use zksync_dal::{connection::DbVariant, ConnectionPool};
Expand All @@ -33,8 +35,12 @@ async fn graceful_shutdown() -> anyhow::Result<impl Future<Output = ()>> {
.context("ProverConfigs")?
.non_gpu
.assembly_receiver_port;
let region = get_region().await.context("get_region()")?;
let zone = get_zone().await.context("get_zone()")?;
let prover_group_config =
ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?;
let region = get_region(&prover_group_config)
.await
.context("get_region()")?;
let zone = get_zone(&prover_group_config).await.context("get_zone()")?;
let address = SocketAddress { host, port };
Ok(async move {
pool.access_storage()
Expand Down Expand Up @@ -120,8 +126,12 @@ pub async fn run() -> anyhow::Result<()> {
.prometheus
};

let region = get_region().await.context("get_regtion()")?;
let zone = get_zone().await.context("get_zone()")?;
let prover_group_config =
ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?;
let region = get_region(&prover_group_config)
.await
.context("get_region()")?;
let zone = get_zone(&prover_group_config).await.context("get_zone()")?;

let (stop_signal_sender, stop_signal_receiver) = oneshot::channel();
let mut stop_signal_sender = Some(stop_signal_sender);
Expand All @@ -144,9 +154,8 @@ pub async fn run() -> anyhow::Result<()> {
// Though we still need to create a channel because circuit breaker expects `stop_receiver`.
let (_stop_sender, stop_receiver) = tokio::sync::watch::channel(false);

let circuit_ids = ProverGroupConfig::from_env()
.context("ProverGroupConfig::from_env()")?
.get_circuit_ids_for_group_id(prover_config.specialized_prover_group_id);
let circuit_ids =
prover_group_config.get_circuit_ids_for_group_id(prover_config.specialized_prover_group_id);

tracing::info!(
"Starting proof generation for circuits: {circuit_ids:?} \
Expand Down Expand Up @@ -184,7 +193,9 @@ pub async fn run() -> anyhow::Result<()> {
)));

let params = ProverParams::new(&prover_config);
let store_factory = ObjectStoreFactory::from_env().context("ObjectStoreFactory::from_env()")?;
let object_store_config =
ObjectStoreConfig::from_env().context("ObjectStoreConfig::from_env()")?;
let store_factory = ObjectStoreFactory::new(object_store_config);

let circuit_provider_pool = ConnectionPool::singleton(DbVariant::Prover)
.build()
Expand Down
2 changes: 1 addition & 1 deletion prover/prover_fri/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ edition = "2021"
zksync_types = { path = "../../core/lib/types" }
zksync_dal = { path = "../../core/lib/dal" }
zksync_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/env_config" }
prometheus_exporter = { path = "../../core/lib/prometheus_exporter" }
vlog = { path = "../../core/lib/vlog" }
zksync_object_store = { path = "../../core/lib/object_store" }
Expand Down
36 changes: 22 additions & 14 deletions prover/prover_fri/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@ use tokio::task::JoinHandle;

use prometheus_exporter::PrometheusExporterConfig;
use zksync_config::configs::fri_prover_group::FriProverGroupConfig;
use zksync_config::configs::FriProverConfig;
use zksync_config::ObjectStoreConfig;
use zksync_config::configs::{FriProverConfig, ProverGroupConfig};
use zksync_dal::connection::DbVariant;
use zksync_dal::ConnectionPool;
use zksync_env_config::FromEnv;
use zksync_env_config::{
object_store::{ProverObjectStoreConfig, PublicObjectStoreConfig},
FromEnv,
};
use zksync_object_store::{ObjectStore, ObjectStoreFactory};
use zksync_prover_fri_utils::get_all_circuit_id_round_tuples_for;

Expand All @@ -34,7 +36,9 @@ async fn graceful_shutdown(port: u16) -> anyhow::Result<impl Future<Output = ()>
.await
.context("failed to build a connection pool")?;
let host = local_ip().context("Failed obtaining local IP address")?;
let zone = get_zone().await.context("get_zone()")?;
let prover_group_config =
ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?;
let zone = get_zone(&prover_group_config).await.context("get_zone()")?;
let address = SocketAddress { host, port };
Ok(async move {
pool.access_storage()
Expand Down Expand Up @@ -84,22 +88,23 @@ async fn main() -> anyhow::Result<()> {
.context("Error setting Ctrl+C handler")?;

let (stop_sender, stop_receiver) = tokio::sync::watch::channel(false);
let blob_store =
ObjectStoreFactory::prover_from_env().context("ObjectStoreFactory::prover_from_env()")?;
let object_store_config =
ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?;
let object_store_factory = ObjectStoreFactory::new(object_store_config.0);
let public_object_store_config =
PublicObjectStoreConfig::from_env().context("PublicObjectStoreConfig::from_env()")?;
let public_blob_store = match prover_config.shall_save_to_public_bucket {
false => None,
true => Some(
ObjectStoreFactory::new(
ObjectStoreConfig::public_from_env()
.context("ObjectStoreConfig::public_from_env()")?,
)
.create_store()
.await,
ObjectStoreFactory::new(public_object_store_config.0)
.create_store()
.await,
),
};
let specialized_group_id = prover_config.specialized_group_id;

let circuit_ids_for_round_to_be_proven = FriProverGroupConfig::from_env()
.context("FriProverGroupConfig::from_env()")?
.get_circuit_ids_for_group_id(specialized_group_id)
.unwrap_or_default();
let circuit_ids_for_round_to_be_proven =
Expand All @@ -118,7 +123,7 @@ async fn main() -> anyhow::Result<()> {
let prover_tasks = get_prover_tasks(
prover_config,
stop_receiver.clone(),
blob_store,
object_store_factory,
public_blob_store,
pool,
circuit_ids_for_round_to_be_proven,
Expand Down Expand Up @@ -203,7 +208,10 @@ async fn get_prover_tasks(
let witness_vector_queue = FixedSizeQueue::new(prover_config.queue_capacity);
let shared_witness_vector_queue = Arc::new(Mutex::new(witness_vector_queue));
let consumer = shared_witness_vector_queue.clone();
let zone = get_zone().await.context("get_zone()")?;

let prover_group_config =
ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?;
let zone = get_zone(&prover_group_config).await.context("get_zone()")?;
let local_ip = local_ip().context("Failed obtaining local IP address")?;
let address = SocketAddress {
host: local_ip,
Expand Down
1 change: 1 addition & 0 deletions prover/prover_fri/src/prover_job_processor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ pub fn load_setup_data_cache(config: &FriProverConfig) -> anyhow::Result<SetupLo
&config.specialized_group_id
);
let prover_setup_metadata_list = FriProverGroupConfig::from_env()
.context("FriProverGroupConfig::from_env()")?
.get_circuit_ids_for_group_id(config.specialized_group_id)
.expect(
"At least one circuit should be configured for group when running in FromMemory mode",
Expand Down
2 changes: 1 addition & 1 deletion prover/prover_fri_gateway/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ edition = "2021"
zksync_types = { path = "../../core/lib/types" }
zksync_dal = { path = "../../core/lib/dal" }
zksync_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/env_config" }
zksync_object_store = { path = "../../core/lib/object_store" }
zksync_utils = { path = "../../core/lib/utils" }
prometheus_exporter = { path = "../../core/lib/prometheus_exporter" }
Expand Down
7 changes: 4 additions & 3 deletions prover/prover_fri_gateway/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use prometheus_exporter::PrometheusExporterConfig;
use zksync_config::configs::FriProverGatewayConfig;
use zksync_dal::connection::DbVariant;
use zksync_dal::ConnectionPool;
use zksync_env_config::FromEnv;
use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv};
use zksync_object_store::ObjectStoreFactory;
use zksync_types::prover_server_api::{ProofGenerationDataRequest, SubmitProofRequest};
use zksync_utils::wait_for_tasks::wait_for_tasks;
Expand Down Expand Up @@ -40,8 +40,9 @@ async fn main() -> anyhow::Result<()> {
.build()
.await
.context("failed to build a connection pool")?;
let store_factory =
ObjectStoreFactory::prover_from_env().context("ObjectStoreFactory::prover_from_env()")?;
let object_store_config =
ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?;
let store_factory = ObjectStoreFactory::new(object_store_config.0);

let proof_submitter = PeriodicApiStruct {
blob_store: store_factory.create_store().await,
Expand Down
2 changes: 1 addition & 1 deletion prover/setup_key_generator_and_server/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ path = "src/main.rs"
zksync_types = { path = "../../core/lib/types" }
vlog = { path = "../../core/lib/vlog" }
zksync_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/env_config" }

circuit_testing = { git = "https://github.com/matter-labs/era-circuit_testing.git", branch = "main" }
api = { git = "https://github.com/matter-labs/era-heavy-ops-service.git", branch = "v1.3.3", features = [
Expand Down
2 changes: 1 addition & 1 deletion prover/vk_setup_data_generator_server_fri/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ circuit_definitions = { git = "https://github.com/matter-labs/era-zkevm_test_har
] }
shivini = { git = "https://github.com/matter-labs/era-shivini.git", branch = "main", optional = true }
zksync_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/env_config" }

anyhow = "1.0"
tracing = "0.1"
Expand Down
2 changes: 1 addition & 1 deletion prover/witness_generator/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ publish = false # We don't want to publish our binaries.
[dependencies]
zksync_dal = { path = "../../core/lib/dal" }
zksync_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/env_config" }
zksync_system_constants = { path = "../../core/lib/constants" }
prometheus_exporter = { path = "../../core/lib/prometheus_exporter" }
vlog = { path = "../../core/lib/vlog" }
Expand Down
22 changes: 12 additions & 10 deletions prover/witness_generator/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ use std::time::Instant;
use structopt::StructOpt;
use tokio::sync::watch;
use zksync_config::configs::{FriWitnessGeneratorConfig, PrometheusConfig};
use zksync_config::ObjectStoreConfig;
use zksync_dal::{connection::DbVariant, ConnectionPool};
use zksync_env_config::FromEnv;
use zksync_env_config::{
object_store::{ProverObjectStoreConfig, PublicObjectStoreConfig},
FromEnv,
};
use zksync_object_store::ObjectStoreFactory;
use zksync_prover_utils::get_stop_signal_receiver;
use zksync_queued_job_processor::JobProcessor;
Expand Down Expand Up @@ -72,8 +74,9 @@ async fn main() -> anyhow::Result<()> {
let started_at = Instant::now();
let use_push_gateway = opt.batch_size.is_some();

let store_factory =
ObjectStoreFactory::prover_from_env().context("ObjectStoreFactor::prover_from_env()")?;
let object_store_config =
ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?;
let store_factory = ObjectStoreFactory::new(object_store_config.0);
let config =
FriWitnessGeneratorConfig::from_env().context("FriWitnessGeneratorConfig::from_env()")?;
let prometheus_config = PrometheusConfig::from_env().context("PrometheusConfig::from_env()")?;
Expand Down Expand Up @@ -123,17 +126,16 @@ async fn main() -> anyhow::Result<()> {
};
let prometheus_task = prometheus_config.run(stop_receiver.clone());

let public_object_store_config =
PublicObjectStoreConfig::from_env().context("PublicObjectStoreConfig::from_env()")?;
let witness_generator_task = match opt.round {
AggregationRound::BasicCircuits => {
let public_blob_store = match config.shall_save_to_public_bucket {
false => None,
true => Some(
ObjectStoreFactory::new(
ObjectStoreConfig::public_from_env()
.context("ObjectStoreConfig::public_from_env()")?,
)
.create_store()
.await,
ObjectStoreFactory::new(public_object_store_config.0)
.create_store()
.await,
),
};
let generator = BasicWitnessGenerator::new(
Expand Down
2 changes: 1 addition & 1 deletion prover/witness_vector_generator/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ edition = "2021"
zksync_types = { path = "../../core/lib/types" }
zksync_dal = { path = "../../core/lib/dal" }
zksync_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/config" }
zksync_env_config = { path = "../../core/lib/env_config" }
zksync_object_store = { path = "../../core/lib/object_store" }
zksync_prover_fri_utils = { path = "../prover_fri_utils" }
zksync_utils = { path = "../../core/lib/utils" }
Expand Down
Loading

0 comments on commit a8713c8

Please sign in to comment.