feat(Prover CLI): status batch command (#1638)

## What ❔ This PR adds a new command to the prover CLI: 'status batch'. This command retrieves information about the prover jobs for a list of L1 batch numbers. To run the command: ```shell zk f cargo run -- status batch -n [1..] ``` This command accepts any number of L1 batch numbers. Example output: ``` $ zk f run --release -- status batch -n 1 == Batch 1 Status == > In Progress ⌛️ == Proving Stages == -- Aggregaton Round 0 -- Basic Witness Generator: Done ✅ > Prover Jobs: In progress ⌛️ -- Aggregaton Round 1 -- Leaf Witness Generator: In progress ⌛️ > Prover Jobs: Waiting for proofs ⏱️ -- Aggregaton Round 2 -- Node Witness Generator: In progress ⌛️ > Prover Jobs: Waiting for proofs ⏱️ -- Aggregaton Round 3 -- Recursion Tip: In progress ⌛️ > Prover Jobs: Waiting for proofs ⏱️ -- Aggregaton Round 4 -- Scheduler: In progress ⌛️ > Prover Jobs: Waiting for proofs ⏱️ -- Compressor -- > Compressor job not found 🚫 ``` Please note that this command is temporary. Once the new `zk` tool is ready, this command will be something like 'pli status batch -n 1 2'. For now, the 'zk' tool is needed to set up the environment. ## Checklist   - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. - [x] Linkcheck has been run via `zk linkcheck`. --------- Co-authored-by: Joaquin Carletti <joaquin@192.168.1.7> Co-authored-by: Ivan Litteri <67517699+ilitteri@users.noreply.github.com> Co-authored-by: Ivan Litteri <ivanlitteri@Ivans-MacBook-Pro.local> Co-authored-by: ilitteri <ilitteri@fi.uba.ar>
matter-labs · May 3, 2024 · 3fd6d65 · 3fd6d65
1 parent c8ee740
commit 3fd6d65
Show file tree

Hide file tree

Showing 20 changed files with 1,602 additions and 41 deletions.
diff --git a/core/lib/basic_types/src/prover_dal.rs b/core/lib/basic_types/src/prover_dal.rs
@@ -1,9 +1,14 @@
 //! Types exposed by the prover DAL for general-purpose use.
 use std::{net::IpAddr, ops::Add, str::FromStr};
 
-use chrono::{DateTime, Duration, Utc};
+use chrono::{DateTime, Duration, NaiveDateTime, NaiveTime, Utc};
+use strum::{Display, EnumString};
 
-use crate::{basic_fri_types::AggregationRound, L1BatchNumber};
+use crate::{
+    basic_fri_types::{AggregationRound, Eip4844Blobs},
+    protocol_version::ProtocolVersionId,
+    L1BatchNumber,
+};
 
 // This currently lives in `zksync_prover_types` -- we don't want a dependency between prover types (`zkevm_test_harness`) and DAL.
 // This will be gone as part of 1.5.0, when EIP4844 becomes normal jobs, rather than special cased ones.
@@ -93,13 +98,13 @@ pub struct JobPosition {
     pub sequence_number: usize,
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, PartialEq)]
 pub struct ProverJobStatusFailed {
     pub started_at: DateTime<Utc>,
     pub error: String,
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct ProverJobStatusSuccessful {
     pub started_at: DateTime<Utc>,
     pub time_taken: Duration,
@@ -114,12 +119,12 @@ impl Default for ProverJobStatusSuccessful {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, PartialEq)]
 pub struct ProverJobStatusInProgress {
     pub started_at: DateTime<Utc>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct WitnessJobStatusSuccessful {
     pub started_at: DateTime<Utc>,
     pub time_taken: Duration,
@@ -134,13 +139,13 @@ impl Default for WitnessJobStatusSuccessful {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 pub struct WitnessJobStatusFailed {
     pub started_at: DateTime<Utc>,
     pub error: String,
 }
 
-#[derive(Debug, strum::Display, strum::EnumString, strum::AsRefStr)]
+#[derive(Debug, strum::Display, strum::EnumString, strum::AsRefStr, PartialEq)]
 pub enum ProverJobStatus {
     #[strum(serialize = "queued")]
     Queued,
@@ -156,7 +161,7 @@ pub enum ProverJobStatus {
     Ignored,
 }
 
-#[derive(Debug, strum::Display, strum::EnumString, strum::AsRefStr)]
+#[derive(Debug, Clone, strum::Display, strum::EnumString, strum::AsRefStr)]
 pub enum WitnessJobStatus {
     #[strum(serialize = "failed")]
     Failed(WitnessJobStatusFailed),
@@ -229,3 +234,121 @@ impl FromStr for GpuProverInstanceStatus {
         }
     }
 }
+
+pub struct ProverJobFriInfo {
+    pub id: u32,
+    pub l1_batch_number: L1BatchNumber,
+    pub circuit_id: u32,
+    pub circuit_blob_url: String,
+    pub aggregation_round: AggregationRound,
+    pub sequence_number: u32,
+    pub status: ProverJobStatus,
+    pub error: Option<String>,
+    pub attempts: u8,
+    pub processing_started_at: Option<NaiveDateTime>,
+    pub created_at: NaiveDateTime,
+    pub updated_at: NaiveDateTime,
+    pub time_taken: Option<NaiveTime>,
+    pub is_blob_cleaned: Option<bool>,
+    pub depth: u32,
+    pub is_node_final_proof: bool,
+    pub proof_blob_url: Option<String>,
+    pub protocol_version: Option<ProtocolVersionId>,
+    pub picked_by: Option<String>,
+}
+
+pub struct BasicWitnessGeneratorJobInfo {
+    pub l1_batch_number: L1BatchNumber,
+    pub merkle_tree_paths_blob_url: Option<String>,
+    pub attempts: u32,
+    pub status: WitnessJobStatus,
+    pub error: Option<String>,
+    pub created_at: NaiveDateTime,
+    pub updated_at: NaiveDateTime,
+    pub processing_started_at: Option<NaiveDateTime>,
+    pub time_taken: Option<NaiveTime>,
+    pub is_blob_cleaned: Option<bool>,
+    pub protocol_version: Option<i32>,
+    pub picked_by: Option<String>,
+    pub eip_4844_blobs: Option<Eip4844Blobs>,
+}
+
+pub struct LeafWitnessGeneratorJobInfo {
+    pub id: u32,
+    pub l1_batch_number: L1BatchNumber,
+    pub circuit_id: u32,
+    pub closed_form_inputs_blob_url: Option<String>,
+    pub attempts: u32,
+    pub status: WitnessJobStatus,
+    pub error: Option<String>,
+    pub created_at: NaiveDateTime,
+    pub updated_at: NaiveDateTime,
+    pub processing_started_at: Option<NaiveDateTime>,
+    pub time_taken: Option<NaiveTime>,
+    pub is_blob_cleaned: Option<bool>,
+    pub number_of_basic_circuits: Option<i32>,
+    pub protocol_version: Option<i32>,
+    pub picked_by: Option<String>,
+}
+
+pub struct NodeWitnessGeneratorJobInfo {
+    pub id: u32,
+    pub l1_batch_number: L1BatchNumber,
+    pub circuit_id: u32,
+    pub depth: u32,
+    pub status: WitnessJobStatus,
+    pub attempts: u32,
+    pub aggregations_url: Option<String>,
+    pub processing_started_at: Option<NaiveDateTime>,
+    pub time_taken: Option<NaiveTime>,
+    pub error: Option<String>,
+    pub created_at: NaiveDateTime,
+    pub updated_at: NaiveDateTime,
+    pub number_of_dependent_jobs: Option<i32>,
+    pub protocol_version: Option<i32>,
+    pub picked_by: Option<String>,
+}
+
+pub struct SchedulerWitnessGeneratorJobInfo {
+    pub l1_batch_number: L1BatchNumber,
+    pub scheduler_partial_input_blob_url: String,
+    pub status: WitnessJobStatus,
+    pub processing_started_at: Option<NaiveDateTime>,
+    pub time_taken: Option<NaiveTime>,
+    pub error: Option<String>,
+    pub created_at: NaiveDateTime,
+    pub updated_at: NaiveDateTime,
+    pub attempts: u32,
+    pub protocol_version: Option<i32>,
+    pub picked_by: Option<String>,
+}
+
+#[derive(Debug, EnumString, Display)]
+pub enum ProofCompressionJobStatus {
+    #[strum(serialize = "queued")]
+    Queued,
+    #[strum(serialize = "in_progress")]
+    InProgress,
+    #[strum(serialize = "successful")]
+    Successful,
+    #[strum(serialize = "failed")]
+    Failed,
+    #[strum(serialize = "sent_to_server")]
+    SentToServer,
+    #[strum(serialize = "skipped")]
+    Skipped,
+}
+
+pub struct ProofCompressionJobInfo {
+    pub l1_batch_number: L1BatchNumber,
+    pub attempts: u32,
+    pub status: ProofCompressionJobStatus,
+    pub fri_proof_blob_url: Option<String>,
+    pub l1_proof_blob_url: Option<String>,
+    pub error: Option<String>,
+    pub created_at: NaiveDateTime,
+    pub updated_at: NaiveDateTime,
+    pub processing_started_at: Option<NaiveDateTime>,
+    pub time_taken: Option<NaiveTime>,
+    pub picked_by: Option<String>,
+}
diff --git a/prover/Cargo.lock b/prover/Cargo.lock
diff --git a/prover/prover_cli/Cargo.toml b/prover/prover_cli/Cargo.toml
@@ -13,12 +13,12 @@ categories.workspace = true
 tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
 env_logger = "0.10"
 log = "0.4"
+colored = "2.1.0"
 
 clap = { workspace = true, features = ["derive"] }
 tracing.workspace = true
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
 bincode.workspace = true
-colored.workspace = true
 hex.workspace = true
 anyhow.workspace = true
 zksync_config.workspace = true
@@ -29,4 +29,4 @@ zksync_types.workspace = true
 zksync_prover_fri_types.workspace = true
 zksync_prover_interface.workspace = true
 prover_dal.workspace = true
-
+strum.workspace = true
diff --git a/prover/prover_cli/README.md b/prover/prover_cli/README.md
@@ -1,14 +1,71 @@
-# CLI to better understand and debug provers
+# Usage
 
-For now, it has only one command 'file-info'
+> Note: For now, its necessary to use the 'zk f' tool to set up the environment. The main command will later be changed
+> to `pli`.
 
+```bash
+Usage: zk f cargo run --release -- <COMMAND>
+
+Commands:
+  file-info
+  status
+  help       Print this message or the help of the given subcommand(s)
+
+Options:
+  -h, --help     Print help
+  -V, --version  Print version
+```
+
+## Status
+
+### Status batch
+
+Displays the proof status for a given batch or a set of batches.
+
+Example:
+
+```bash
+$ zk f run --release -- status batch -n 1
+
+== Batch 1 Status ==
+> In Progress ⌛️
+
+== Proving Stages ==
+-- Aggregation Round 0 --
+Basic Witness Generator: Done ✅
+> Prover Jobs: In progress ⌛️
+
+-- Aggregation Round 1 --
+Leaf Witness Generator: In progress ⌛️
+> Prover Jobs: Waiting for proofs ⏱️
+
+-- Aggregation Round 2 --
+Node Witness Generator: In progress ⌛️
+> Prover Jobs: Waiting for proofs ⏱️
+
+-- Aggregation Round 3 --
+Recursion Tip: In progress ⌛️
+> Prover Jobs: Waiting for proofs ⏱️
+
+-- Aggregation Round 4 --
+Scheduler: In progress ⌛️
+> Prover Jobs: Waiting for proofs ⏱️
+
+-- Compressor --
+> Compressor job not found 🚫
 ```
+
+## File-Info
+
+Displays the information about a given file:
+
+```bash
 cargo run -- file-info --file-path /zksync-era/prover/artifacts/proofs_fri/l1_batch_proof_1.bin
 ```
 
 Example outputs:
 
-```
+```bash
 L1 proof
 AUX info:
   L1 msg linear hash: [163, 243, 172, 16, 189, 59, 100, 227, 249, 46, 226, 220, 82, 135, 213, 208, 221, 228, 49, 46, 121, 136, 78, 163, 15, 155, 199, 82, 64, 24, 172, 198]
@@ -18,7 +75,7 @@ AUX info:
 Inputs: [Fr(0x00000000775db828700e0ebbe0384f8a017598a271dfb6c96ebb2baf22a7a572)]
 ```
 
-```
+```bash
  == Circuit ==
 Type: basic. Id: 1 (Scheduler)
 Geometry: CSGeometry { num_columns_under_copy_permutation: 130, num_witness_columns: 0, num_constant_columns: 4, max_allowed_constraint_degree: 8 }

diff --git a/prover/prover_cli/src/cli.rs b/prover/prover_cli/src/cli.rs
@@ -1,6 +1,6 @@
 use clap::{command, Parser, Subcommand};
 
-use crate::commands::get_file_info;
+use crate::commands::{self, get_file_info};
 
 pub const VERSION_STRING: &str = env!("CARGO_PKG_VERSION");
 
@@ -14,12 +14,15 @@ struct ProverCLI {
 #[derive(Subcommand)]
 enum ProverCommand {
     FileInfo(get_file_info::Args),
+    #[command(subcommand)]
+    Status(commands::StatusCommand),
 }
 
 pub async fn start() -> anyhow::Result<()> {
     let ProverCLI { command } = ProverCLI::parse();
     match command {
         ProverCommand::FileInfo(args) => get_file_info::run(args).await?,
+        ProverCommand::Status(cmd) => cmd.run().await?,
     };
 
     Ok(())

diff --git a/prover/prover_cli/src/commands/mod.rs b/prover/prover_cli/src/commands/mod.rs
@@ -1 +1,4 @@
 pub(crate) mod get_file_info;
+pub(crate) mod status;
+
+pub(crate) use status::StatusCommand;