Initial code for dump-block feature (#4684)

* Initial code for dump-block feature * Cargo fmt pass * Display current folder * Display current folder 2 * Add denunciations to grpc block header * Cargo fmt pass * Cargo clippy pass * Try to update setup-protoc github actions * Try to update setup-protoc github actions 2 * Update serde * Update massa-sc-runtime && massa-proto-rs commit id
massalabs · May 13, 2024 · a1a2190 · a1a2190
1 parent a4d9cda
commit a1a2190
Show file tree

Hide file tree

Showing 23 changed files with 333 additions and 20 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -197,9 +197,11 @@ jobs:
         with:
           shared-key: "massa"
           save-if: ${{ github.ref_name == 'main' }}
-      - uses: arduino/setup-protoc@v1
+      - uses: arduino/setup-protoc@v3
         with:
-          version: '3.x'
+          # version: '3.x'
+          version: "23.x"
+          include-pre-releases: false
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rs/cargo@v1
         with:

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -106,8 +106,8 @@ massa_versioning = { path = "./massa-versioning" }
 massa_wallet = { path = "./massa-wallet" }
 
 # Massa projects dependencies
-massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", "rev" = "426fd325a55dfcc4033920bed2de075a7e7ad4b7" }
-massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "e95066a6d3a963ff0125616f404a84e5abb63d63" }
+massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", "rev" = "38950875a7aa406fedc4f0b8336864e5ff290f2c" }
+massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "80352eb9f2a6b90a441cd64433d8874c33fb384f" }
 peernet = { git = "https://github.com/massalabs/PeerNet", "rev" = "04b05ddd320fbe76cc858115af7b5fc28bdb8310" }
 
 # Common dependencies

diff --git a/massa-api/Cargo.toml b/massa-api/Cargo.toml
@@ -6,6 +6,8 @@ edition = "2021"
 [features]
 test-exports = ["dep:massa_channel", "dep:massa_grpc", "massa_grpc/test-exports"]
 execution-trace = ["massa_execution_exports/execution-trace"]
+dump-block = ["massa_execution_exports/dump-block"]
+
 
 [dependencies]
 massa_api_exports = { workspace = true }

diff --git a/massa-api/src/tests/public.rs b/massa-api/src/tests/public.rs
@@ -694,6 +694,10 @@ async fn execute_read_only_bytecode() {
                     block_info: None,
                     state_changes: massa_final_state::StateChanges::default(),
                     events: massa_execution_exports::EventStore::default(),
+                    #[cfg(feature = "execution-trace")]
+                    slot_trace: None,
+                    #[cfg(feature = "dump-block")]
+                    storage: None,
                 },
                 gas_cost: 100,
                 call_result: "toto".as_bytes().to_vec(),
@@ -774,6 +778,10 @@ async fn execute_read_only_call() {
                     block_info: None,
                     state_changes: massa_final_state::StateChanges::default(),
                     events: massa_execution_exports::EventStore::default(),
+                    #[cfg(feature = "execution-trace")]
+                    slot_trace: None,
+                    #[cfg(feature = "dump-block")]
+                    storage: None,
                 },
                 gas_cost: 100,
                 call_result: "toto".as_bytes().to_vec(),

diff --git a/massa-execution-exports/Cargo.toml b/massa-execution-exports/Cargo.toml
@@ -8,6 +8,7 @@ edition = "2021"
 gas_calibration = ["tempfile"]
 test-exports = ["massa_models/test-exports", "tempfile", "mockall"]
 execution-trace = ["massa-sc-runtime/execution-trace"]
+dump-block = []
 
 [dependencies]
 displaydoc = {workspace = true}

diff --git a/massa-execution-exports/src/settings.rs b/massa-execution-exports/src/settings.rs
@@ -100,4 +100,6 @@ pub struct ExecutionConfig {
     pub broadcast_slot_execution_traces_channel_capacity: usize,
     /// Max execution traces slot to keep in trace history cache
     pub max_execution_traces_slot_limit: usize,
+    /// Where to dump blocks
+    pub block_dump_folder_path: PathBuf,
 }
diff --git a/massa-execution-exports/src/test_exports/config.rs b/massa-execution-exports/src/test_exports/config.rs
@@ -19,6 +19,14 @@ impl Default for ExecutionConfig {
                 .expect("Overflow when creating constant ledger_entry_datastore_base_size"),
         };
 
+        // Create a tmp dir then only storing the path will drop the original tmp dir object
+        // thus deleting the folders
+        // So we need to create it manually (not really safe but ok for unit testing)
+        let hd_cache_path = TempDir::new().unwrap().path().to_path_buf();
+        std::fs::create_dir_all(hd_cache_path.clone()).unwrap();
+        let block_dump_folder_path = TempDir::new().unwrap().path().to_path_buf();
+        std::fs::create_dir_all(block_dump_folder_path.clone()).unwrap();
+
         Self {
             readonly_queue_length: 100,
             max_final_events: 1000,
@@ -57,7 +65,7 @@ impl Default for ExecutionConfig {
             .unwrap(),
             base_operation_gas_cost: BASE_OPERATION_GAS_COST,
             last_start_period: 0,
-            hd_cache_path: TempDir::new().unwrap().path().to_path_buf(),
+            hd_cache_path,
             lru_cache_size: 1000,
             hd_cache_size: 10_000,
             snip_amount: 10,
@@ -72,6 +80,7 @@ impl Default for ExecutionConfig {
             broadcast_traces_enabled: true,
             broadcast_slot_execution_traces_channel_capacity: 5000,
             max_execution_traces_slot_limit: 320,
+            block_dump_folder_path,
         }
     }
 }
diff --git a/massa-execution-exports/src/types.rs b/massa-execution-exports/src/types.rs
@@ -361,6 +361,9 @@ pub struct ExecutionOutput {
     /// slot trace
     #[cfg(feature = "execution-trace")]
     pub slot_trace: Option<(SlotAbiCallStack, Vec<Transfer>)>,
+    /// storage
+    #[cfg(feature = "dump-block")]
+    pub storage: Option<Storage>,
 }
 
 #[cfg(feature = "execution-trace")]

diff --git a/massa-execution-worker/Cargo.toml b/massa-execution-worker/Cargo.toml
@@ -43,6 +43,10 @@ execution-trace = [
     "schnellru",
     "massa_execution_exports/execution-trace",
 ]
+dump-block = [
+    "prost",
+    "massa_execution_exports/dump-block",
+]
 
 [dependencies]
 anyhow = { workspace = true }
@@ -83,6 +87,7 @@ tempfile = { workspace = true, optional = true }
 massa_wallet = { workspace = true }
 massa-proto-rs = { workspace = true }
 schnellru = { workspace = true, optional = true }
+prost = { version = "=0.12", optional = true }
 
 [dev-dependencies]
 massa_storage = { workspace = true }

diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs
@@ -913,6 +913,8 @@ impl ExecutionContext {
             events: std::mem::take(&mut self.events),
             #[cfg(feature = "execution-trace")]
             slot_trace: None,
+            #[cfg(feature = "dump-block")]
+            storage: None,
         }
     }
 

diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs
@@ -53,10 +53,22 @@ use tracing::{debug, info, trace, warn};
 use crate::trace_history::TraceHistory;
 #[cfg(feature = "execution-trace")]
 use massa_execution_exports::{AbiTrace, SlotAbiCallStack, Transfer};
+#[cfg(feature = "dump-block")]
+use massa_models::block::FilledBlock;
 #[cfg(feature = "execution-trace")]
 use massa_models::config::{BASE_OPERATION_GAS_COST, MAX_GAS_PER_BLOCK, MAX_OPERATIONS_PER_BLOCK};
+#[cfg(feature = "dump-block")]
+use massa_models::operation::Operation;
 #[cfg(feature = "execution-trace")]
 use massa_models::prehash::PreHashMap;
+#[cfg(feature = "dump-block")]
+use massa_models::secure_share::SecureShare;
+#[cfg(feature = "dump-block")]
+use massa_proto_rs::massa::model::v1 as grpc_model;
+#[cfg(feature = "dump-block")]
+use prost::Message;
+#[cfg(feature = "dump-block")]
+use std::io::Write;
 
 /// Used to acquire a lock on the execution context
 macro_rules! context_guard {
@@ -325,6 +337,53 @@ impl ExecutionState {
                 }
             }
         }
+
+        #[cfg(feature = "dump-block")]
+        {
+            let block_folder = &self.config.block_dump_folder_path;
+            let block_file_path = block_folder.join(format!(
+                "block_slot_{}_{}.bin",
+                exec_out.slot.thread, exec_out.slot.period
+            ));
+
+            let mut fs = std::fs::File::create(block_file_path.clone())
+                .unwrap_or_else(|_| panic!("Cannot create file: {:?}", block_file_path));
+
+            let mut block_ser = vec![];
+            if let Some(block_info) = exec_out.block_info {
+                let block_id = block_info.block_id;
+                let storage = exec_out.storage.unwrap();
+                let guard = storage.read_blocks();
+                let secured_block = guard
+                    .get(&block_id)
+                    .unwrap_or_else(|| panic!("Unable to get block for block id: {}", block_id));
+
+                let operations: Vec<(OperationId, Option<SecureShare<Operation, OperationId>>)> =
+                    secured_block
+                        .content
+                        .operations
+                        .iter()
+                        .map(|operation_id| {
+                            match storage.read_operations().get(operation_id).cloned() {
+                                Some(verifiable_operation) => {
+                                    (*operation_id, Some(verifiable_operation))
+                                }
+                                None => (*operation_id, None),
+                            }
+                        })
+                        .collect();
+
+                let filled_block = FilledBlock {
+                    header: secured_block.content.header.clone(),
+                    operations,
+                };
+
+                let grpc_filled_block = grpc_model::FilledBlock::from(filled_block);
+                grpc_filled_block.encode(&mut block_ser).unwrap();
+            }
+            fs.write_all(&block_ser[..])
+                .expect("Unable to write block to disk");
+        }
     }
 
     /// Applies an execution output to the active (non-final) state
@@ -1419,16 +1478,21 @@ impl ExecutionState {
         self.trace_history
             .write()
             .save_transfers_for_slot(*slot, transfers.clone());
-        // Finish slot
-        #[cfg(not(feature = "execution-trace"))]
-        let exec_out = context_guard!(self).settle_slot(block_info);
 
+        // Finish slot
+        #[allow(unused_mut)]
+        let mut exec_out = context_guard!(self).settle_slot(block_info);
         #[cfg(feature = "execution-trace")]
-        let exec_out = {
-            let mut out = context_guard!(self).settle_slot(block_info);
-            out.slot_trace = Some((slot_trace, transfers));
-            out
+        {
+            exec_out.slot_trace = Some((slot_trace, transfers));
         };
+        #[cfg(feature = "dump-block")]
+        {
+            exec_out.storage = match exec_target {
+                Some((_block_id, block_metadata)) => block_metadata.storage.clone(),
+                _ => None,
+            }
+        }
 
         // Broadcast a slot execution output to active channel subscribers.
         if self.config.broadcast_enabled {