Merge pull request #556 from hirosystems/develop

chore(release): publish v1.4.2

components/chainhook-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "chainhook"
-version = "1.4.1"
+version = "1.5.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -15,7 +15,7 @@ redis = "0.21.5"
 serde-redis = "0.12.0"
 hex = "0.4.3"
 rand = "0.8.5"
-chainhook-sdk = { version = "0.12.1", default-features = false, features = [
+chainhook-sdk = { version = "0.12.6", default-features = false, features = [
     "zeromq",
 ], path = "../chainhook-sdk" }
 hiro-system-kit = "0.3.4" 

components/chainhook-cli/src/cli/mod.rs

@@ -8,18 +8,21 @@ use crate::scan::stacks::{
 use crate::service::http_api::document_predicate_api_server;
 use crate::service::Service;
 use crate::storage::{
-    get_last_block_height_inserted, get_stacks_block_at_block_height, is_stacks_block_present,
-    open_readonly_stacks_db_conn,
+    delete_confirmed_entry_from_stacks_blocks, delete_unconfirmed_entry_from_stacks_blocks,
+    get_last_block_height_inserted, get_last_unconfirmed_block_height_inserted,
+    get_stacks_block_at_block_height, insert_unconfirmed_entry_in_stacks_blocks,
+    is_stacks_block_present, open_readonly_stacks_db_conn, open_readwrite_stacks_db_conn,
+    set_last_confirmed_insert_key,
 };
 
 use chainhook_sdk::chainhooks::types::{
     BitcoinChainhookFullSpecification, BitcoinChainhookNetworkSpecification, BitcoinPredicateType,
-    ChainhookFullSpecification, FileHook, HookAction, OrdinalOperations,
+    ChainhookFullSpecification, FileHook, HookAction, InscriptionFeedData, OrdinalOperations,
     StacksChainhookFullSpecification, StacksChainhookNetworkSpecification, StacksPredicate,
     StacksPrintEventBasedPredicate,
 };
 use chainhook_sdk::types::{BitcoinNetwork, BlockIdentifier, StacksNetwork};
-use chainhook_sdk::utils::Context;
+use chainhook_sdk::utils::{BlockHeights, Context};
 use clap::{Parser, Subcommand};
 use hiro_system_kit;
 use std::collections::BTreeMap;
@@ -217,6 +220,48 @@ enum StacksDbCommand {
     /// Retrieve a block from the Stacks db
     #[clap(name = "get", bin_name = "get")]
     GetBlock(GetBlockDbCommand),
+    /// Deletes a block from the confirmed block db and moves it to the unconfirmed block db.
+    #[clap(name = "unconfirm", bin_name = "unconfirm")]
+    UnconfirmBlock(UnconfirmBlockDbCommand),
+    /// Get latest blocks from the unconfirmed and confirmed block db.
+    #[clap(name = "get-latest", bin_name = "get-latest")]
+    GetLatest(GetLatestBlocksDbCommand),
+    /// Update blocks from database
+    #[clap(name = "drop", bin_name = "drop")]
+    Drop(DropBlockCommand),
+}
+
+#[derive(Parser, PartialEq, Clone, Debug)]
+struct GetLatestBlocksDbCommand {
+    /// Load config file path
+    #[clap(long = "config-path")]
+    pub config_path: Option<String>,
+    /// The number of blocks from the chain tip to fetch.
+    #[clap(long = "count")]
+    pub count: u64,
+}
+
+#[derive(Parser, PartialEq, Clone, Debug)]
+struct DropBlockCommand {
+    /// Load config file path
+    #[clap(long = "config-path")]
+    pub config_path: Option<String>,
+    /// Interval of blocks (--interval 767430:800000)
+    #[clap(long = "interval", conflicts_with = "blocks")]
+    pub blocks_interval: Option<String>,
+    /// List of blocks (--blocks 767430,767431,767433,800000)
+    #[clap(long = "blocks", conflicts_with = "interval")]
+    pub blocks: Option<String>,
+}
+
+#[derive(Parser, PartialEq, Clone, Debug)]
+struct UnconfirmBlockDbCommand {
+    /// Load config file path
+    #[clap(long = "config-path")]
+    pub config_path: Option<String>,
+    /// The block height to unconfirm
+    #[clap(long = "block-height")]
+    pub block_height: u64,
 }
 
 #[derive(Parser, PartialEq, Clone, Debug)]
@@ -391,7 +436,9 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
                                 end_block: Some(767430),
                                 blocks: None,
                                 predicate: BitcoinPredicateType::OrdinalsProtocol(
-                                    OrdinalOperations::InscriptionFeed,
+                                    OrdinalOperations::InscriptionFeed(InscriptionFeedData {
+                                        meta_protocols: None,
+                                    }),
                                 ),
                                 expire_after_occurrence: None,
                                 action: HookAction::FileAppend(FileHook {
@@ -555,6 +602,182 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
             }
         },
         Command::Stacks(subcmd) => match subcmd {
+            StacksCommand::Db(StacksDbCommand::UnconfirmBlock(cmd)) => {
+                let config = Config::default(false, false, false, &cmd.config_path)?;
+                let stacks_db_rw =
+                    open_readwrite_stacks_db_conn(&config.expected_cache_path(), &ctx)
+                        .expect("unable to read stacks_db");
+
+                match get_stacks_block_at_block_height(cmd.block_height, true, 3, &stacks_db_rw) {
+                    Ok(Some(block)) => {
+                        let mut delete_confirmed = false;
+                        let mut insert_unconfirmed = false;
+                        match get_stacks_block_at_block_height(
+                            cmd.block_height,
+                            false,
+                            3,
+                            &stacks_db_rw,
+                        ) {
+                            Ok(Some(_)) => {
+                                warn!(ctx.expect_logger(), "Block {} was found in both the confirmed and unconfirmed database. Deleting from confirmed database.", cmd.block_height);
+                                delete_confirmed = true;
+                            }
+                            Ok(None) => {
+                                info!(ctx.expect_logger(), "Block {} found in confirmed database. Deleting from confirmed database and inserting into unconfirmed.", cmd.block_height);
+                                delete_confirmed = true;
+                                insert_unconfirmed = true;
+                            }
+                            Err(e) => {
+                                error!(
+                                    ctx.expect_logger(),
+                                    "Error making request to database: {e}",
+                                );
+                            }
+                        }
+                        if delete_confirmed {
+                            if let Some(last_inserted) =
+                                get_last_block_height_inserted(&stacks_db_rw, &ctx)
+                            {
+                                if last_inserted == block.block_identifier.index {
+                                    set_last_confirmed_insert_key(
+                                        &block.parent_block_identifier,
+                                        &stacks_db_rw,
+                                        &ctx,
+                                    )?;
+                                }
+                            }
+                            delete_confirmed_entry_from_stacks_blocks(
+                                &block.block_identifier,
+                                &stacks_db_rw,
+                                &ctx,
+                            )?;
+                        }
+                        if insert_unconfirmed {
+                            insert_unconfirmed_entry_in_stacks_blocks(&block, &stacks_db_rw, &ctx)?;
+                        }
+                    }
+                    Ok(None) => {
+                        warn!(ctx.expect_logger(), "Block {} not present in the confirmed database. No database changes were made by this command.", cmd.block_height);
+                    }
+                    Err(e) => {
+                        error!(ctx.expect_logger(), "Error making request to database: {e}",);
+                    }
+                }
+            }
+            StacksCommand::Db(StacksDbCommand::GetLatest(cmd)) => {
+                let config = Config::default(false, false, false, &cmd.config_path)?;
+                let stacks_db = open_readonly_stacks_db_conn(&config.expected_cache_path(), &ctx)
+                    .expect("unable to read stacks_db");
+
+                match get_last_block_height_inserted(&stacks_db, &ctx) {
+                    Some(confirmed_tip) => {
+                        let min_block = confirmed_tip - cmd.count;
+                        info!(
+                            ctx.expect_logger(),
+                            "Getting confirmed blocks {} through {}", min_block, confirmed_tip
+                        );
+                        let mut confirmed_blocks = vec![];
+                        let mut cursor = confirmed_tip;
+                        while cursor > min_block {
+                            match get_stacks_block_at_block_height(cursor, true, 3, &stacks_db) {
+                                Ok(Some(block)) => {
+                                    confirmed_blocks.push(block.block_identifier.index);
+                                    cursor -= 1;
+                                }
+                                Ok(None) => {
+                                    warn!(
+                                        ctx.expect_logger(),
+                                        "Block {} not present in confirmed database", cursor
+                                    );
+                                    cursor -= 1;
+                                }
+                                Err(e) => {
+                                    error!(ctx.expect_logger(), "{e}",);
+                                    break;
+                                }
+                            }
+                        }
+                        info!(
+                            ctx.expect_logger(),
+                            "Found confirmed blocks: {:?}", confirmed_blocks
+                        );
+                    }
+                    None => {
+                        warn!(ctx.expect_logger(), "No confirmed blocks found in db");
+                    }
+                };
+
+                match get_last_unconfirmed_block_height_inserted(&stacks_db, &ctx) {
+                    Some(unconfirmed_tip) => {
+                        let min_block = unconfirmed_tip - cmd.count;
+                        info!(
+                            ctx.expect_logger(),
+                            "Getting unconfirmed blocks {} through {}", min_block, unconfirmed_tip
+                        );
+                        let mut confirmed_blocks = vec![];
+                        let mut cursor = unconfirmed_tip;
+                        while cursor > min_block {
+                            match get_stacks_block_at_block_height(cursor, false, 3, &stacks_db) {
+                                Ok(Some(block)) => {
+                                    confirmed_blocks.push(block.block_identifier.index);
+                                    cursor -= 1;
+                                }
+                                Ok(None) => {
+                                    warn!(
+                                        ctx.expect_logger(),
+                                        "Block {} not present in unconfirmed database", cursor
+                                    );
+                                    cursor -= 1;
+                                }
+                                Err(e) => {
+                                    error!(ctx.expect_logger(), "{e}",);
+                                    break;
+                                }
+                            }
+                        }
+                        info!(
+                            ctx.expect_logger(),
+                            "Found unconfirmed blocks: {:?}", confirmed_blocks
+                        );
+                    }
+                    None => {
+                        warn!(ctx.expect_logger(), "No confirmed blocks found in db");
+                    }
+                };
+            }
+            StacksCommand::Db(StacksDbCommand::Drop(cmd)) => {
+                let config = Config::default(false, false, false, &cmd.config_path)?;
+                let stacks_db_rw =
+                    open_readwrite_stacks_db_conn(&config.expected_cache_path(), &ctx)
+                        .expect("unable to read stacks_db");
+
+                let block_heights = parse_blocks_heights_spec(&cmd.blocks_interval, &cmd.blocks)
+                    .get_sorted_entries()
+                    .unwrap();
+                let total_blocks = block_heights.len();
+                println!("{} blocks will be deleted. Confirm? [Y/n]", total_blocks);
+                let mut buffer = String::new();
+                std::io::stdin().read_line(&mut buffer).unwrap();
+                if buffer.starts_with('n') {
+                    return Err("Deletion aborted".to_string());
+                }
+
+                for index in block_heights.into_iter() {
+                    let block_identifier = BlockIdentifier {
+                        index,
+                        hash: "".into(),
+                    };
+                    let _ = delete_unconfirmed_entry_from_stacks_blocks(
+                        &block_identifier,
+                        &stacks_db_rw,
+                        &ctx,
+                    );
+                }
+                info!(
+                    ctx.expect_logger(),
+                    "Cleaning stacks_db: {} blocks dropped", total_blocks
+                );
+            }
             StacksCommand::Db(StacksDbCommand::GetBlock(cmd)) => {
                 let config = Config::default(false, false, false, &cmd.config_path)?;
                 let stacks_db = open_readonly_stacks_db_conn(&config.expected_cache_path(), &ctx)
@@ -653,3 +876,34 @@ pub fn load_predicate_from_path(
         .map_err(|e| format!("unable to parse json file {}\n{:?}", predicate_path, e))?;
     Ok(predicate)
 }
+
+fn parse_blocks_heights_spec(
+    blocks_interval: &Option<String>,
+    blocks: &Option<String>,
+) -> BlockHeights {
+    let blocks = match (blocks_interval, blocks) {
+        (Some(interval), None) => {
+            let blocks = interval.split(':').collect::<Vec<_>>();
+            let start_block: u64 = blocks
+                .first()
+                .expect("unable to get start_block")
+                .parse::<u64>()
+                .expect("unable to parse start_block");
+            let end_block: u64 = blocks
+                .get(1)
+                .expect("unable to get end_block")
+                .parse::<u64>()
+                .expect("unable to parse end_block");
+            BlockHeights::BlockRange(start_block, end_block)
+        }
+        (None, Some(blocks)) => {
+            let blocks = blocks
+                .split(',')
+                .map(|b| b.parse::<u64>().expect("unable to parse block"))
+                .collect::<Vec<_>>();
+            BlockHeights::Blocks(blocks)
+        }
+        _ => unreachable!(),
+    };
+    blocks
+}

components/chainhook-cli/src/service/tests/mod.rs

@@ -127,7 +127,11 @@ async fn it_handles_bitcoin_predicates_with_network(network: &str) {
 #[test_case(json!({"scope": "stacks_protocol","operation": "leader_registered"}) ; "with scope stacks_protocol operation leader_registered")]
 #[test_case(json!({"scope": "stacks_protocol","operation": "stx_transferred"}) ; "with scope stacks_protocol operation stx_transferred")]
 #[test_case(json!({"scope": "stacks_protocol","operation": "stx_locked"}) ; "with scope stacks_protocol operation stx_locked")]
-#[test_case(json!({"scope": "ordinals_protocol","operation": "inscription_feed"}) ; "with scope ordinals_protocol operation inscription_feed")]
+#[test_case(json!({"scope": "ordinals_protocol","operation": "inscription_feed"}) ; "with scope ordinals_protocol operation inscription_feed with no meta_protocols")]
+// todo: a test that uses both meta_protocols (eg ["all", "brc-20"]) is broken because our test requires the result is in the same order
+// this should be fixed on the test end, since the order shouldn't matter.
+#[test_case(json!({"scope": "ordinals_protocol","operation": "inscription_feed", "meta_protocols": ["all"]}) ; "with scope ordinals_protocol operation inscription_feed with all meta_protocols")]
+#[test_case(json!({"scope": "ordinals_protocol","operation": "inscription_feed", "meta_protocols": ["brc-20"]}) ; "with scope ordinals_protocol operation inscription_feed with brc-20 meta_protocols")]
 #[tokio::test]
 async fn it_handles_bitcoin_if_this_predicates(if_this: JsonValue) {
     let predicate = build_bitcoin_payload(None, Some(if_this), None, None, None);