RPC chain example

.gitignore

@@ -4,3 +4,5 @@ Cargo.lock
 out.perf-folded
 perf.data
 rust-perf.svg
+
+.bdk_example_db

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "bdk_core",
     "bdk_cli_lib",
     "bdk_esplora_example",
+    "bdk_rpc_example",
     "bdk_keychain",
     "bdk_file_store"
 ]

bdk_cli_lib/src/lib.rs

@@ -19,12 +19,13 @@ use bdk_keychain::{
 };
 pub use clap;
 use clap::{Parser, Subcommand};
+pub use log;
 use std::{cmp::Reverse, collections::HashMap, fmt::Debug, path::PathBuf, time::Duration};
 
 #[derive(Parser)]
 #[clap(author, version, about, long_about = None)]
 #[clap(propagate_version = true)]
-pub struct Args<C: clap::Subcommand> {
+pub struct Args<A: clap::Args, C: clap::Subcommand> {
     #[clap(env = "DESCRIPTOR")]
     pub descriptor: String,
     #[clap(env = "CHANGE_DESCRIPTOR")]
@@ -36,6 +37,12 @@ pub struct Args<C: clap::Subcommand> {
     #[clap(env = "BDK_DB_DIR", long, default_value = ".bdk_example_db")]
     pub db_dir: PathBuf,
 
+    #[clap(env = "BDK_CP_LIMIT", long, default_value = "20")]
+    pub cp_limit: usize,
+
+    #[clap(flatten)]
+    pub chain_args: A,
+
     #[clap(subcommand)]
     pub command: Commands<C>,
 }
@@ -500,8 +507,8 @@ where
     Ok(())
 }
 
-pub fn init<C: clap::Subcommand, I>() -> anyhow::Result<(
-    Args<C>,
+pub fn init<A: clap::Args, C: clap::Subcommand, I>() -> anyhow::Result<(
+    Args<A, C>,
     KeyMap,
     KeychainTracker<Keychain, I>,
     KeychainStore<Keychain, I>,
@@ -510,12 +517,13 @@ where
     I: sparse_chain::ChainIndex,
     KeychainChangeSet<Keychain, I>: serde::Serialize + serde::de::DeserializeOwned,
 {
-    let args = Args::<C>::parse();
+    let args = Args::<A, C>::parse();
     let secp = Secp256k1::default();
     let (descriptor, mut keymap) =
         Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &args.descriptor)?;
 
     let mut keychain_tracker = KeychainTracker::default();
+    keychain_tracker.set_checkpoint_limit(Some(args.cp_limit));
     keychain_tracker
         .txout_index
         .add_keychain(Keychain::External, descriptor);

bdk_core/src/spk_txout_index.rs

@@ -65,7 +65,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     ///
     /// [`ForEachTxout`]: crate::ForEachTxout
     pub fn scan(&mut self, txouts: &impl ForEachTxout) {
-        txouts.for_each_txout(&mut |(op, txout)| self.scan_txout(op, txout))
+        txouts.for_each_txout(&mut |(op, txout)| self.scan_txout(op, txout));
     }
 
     /// Scan a single `TxOut` for a matching script pubkey
@@ -174,16 +174,26 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// let change_index = 1;
     /// let unused_change_spks = txout_index.unused((change_index, u32::MIN)..(change_index, u32::MAX));
     /// ```
-    pub fn unused<R>(&self, range: R) -> impl DoubleEndedIterator<Item = (&I, &Script)>
-    where
-        R: RangeBounds<I>,
-    {
+    pub fn unused<R: RangeBounds<I>>(
+        &self,
+        range: R,
+    ) -> impl DoubleEndedIterator<Item = (&I, &Script)> {
         self.unused
             .range(range)
             .map(|index| (index, self.spk_at_index(index).expect("must exist")))
     }
 
-    /// Returns whether the script pubkey at `index` has been used or not.
+    pub fn remove_unused(&mut self, index: &I) -> bool {
+        let is_removed = self.unused.remove(index);
+        if is_removed {
+            let spk = self.script_pubkeys.remove(index).expect("should exist");
+            let _removed_index = self.spk_indexes.remove(&spk);
+            debug_assert!(_removed_index == Some(index.clone()));
+        }
+        is_removed
+    }
+
+    /// Returns whether the script pubkey at index `index` has been used or not.
     ///
     /// Here "unused" means that after the script pubkey was stored in the index, the index has
     /// never scanned a transaction output with it.

bdk_esplora_example/src/main.rs

@@ -7,9 +7,15 @@ use std::io::{self, Write};
 const DEFAULT_PARALLEL_REQUESTS: u8 = 5;
 use bdk_cli::{
     anyhow::{self, Context},
-    clap::{self, Subcommand},
+    clap::{self, Args, Subcommand},
 };
 
+#[derive(Args, Debug, Clone)]
+struct EsploraUrlArgs {
+    #[clap(env = "ESPLORA_URL", long)]
+    url: Option<String>,
+}
+
 #[derive(Subcommand, Debug, Clone)]
 enum EsploraCommands {
     /// Scans the addresses in the wallet using esplora API.
@@ -33,12 +39,17 @@ enum EsploraCommands {
 }
 
 fn main() -> anyhow::Result<()> {
-    let (args, keymap, mut keychain_tracker, mut db) = bdk_cli::init::<EsploraCommands, _>()?;
-    let esplora_url = match args.network {
-        Network::Bitcoin => "https://mempool.space/api",
-        Network::Testnet => "https://mempool.space/testnet/api",
-        Network::Regtest => "http://localhost:3000",
-        Network::Signet => "https://mempool.space/signet/api",
+    let (args, keymap, mut keychain_tracker, mut db) =
+        bdk_cli::init::<EsploraUrlArgs, EsploraCommands, _>()?;
+
+    let esplora_url = match &args.chain_args.url {
+        Some(url) => url.as_str(),
+        None => match args.network {
+            Network::Bitcoin => "https://mempool.space/api",
+            Network::Testnet => "https://mempool.space/testnet/api",
+            Network::Regtest => "http://localhost:3000",
+            Network::Signet => "https://mempool.space/signet/api",
+        },
     };
 
     let client = Client::new(esplora_url, DEFAULT_PARALLEL_REQUESTS)?;

bdk_keychain/src/keychain_txout_index.rs

@@ -1,3 +1,4 @@
+use alloc::vec::Vec;
 use bdk_core::{
     bitcoin::{secp256k1::Secp256k1, OutPoint, Script, TxOut},
     collections::*,
@@ -79,14 +80,14 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     ///
     /// [`ForEachTxout`]: bdk_core::ForEachTxout
     pub fn scan(&mut self, txouts: &impl ForEachTxout) {
-        self.inner.scan(txouts);
+        self.inner.scan(txouts)
     }
 
     /// Scan a single `TxOut` for a matching script pubkey.
     ///
     /// If it matches the index will store and index it.
     pub fn scan_txout(&mut self, op: OutPoint, txout: &TxOut) {
-        self.inner.scan_txout(op, &txout);
+        self.inner.scan_txout(op, &txout)
     }
 
     pub fn inner(&self) -> &SpkTxOutIndex<(K, u32)> {
@@ -144,8 +145,8 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
         self.inner
             .script_pubkeys()
             .range(&(keychain.clone(), u32::MIN)..=&(keychain.clone(), u32::MAX))
+            .next_back()
             .map(|((_, index), _)| *index)
-            .last()
     }
 
     /// Gets the current derivation index for each keychain in the index.
@@ -165,7 +166,8 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     pub fn store_all_up_to(&mut self, keychains: &BTreeMap<K, u32>) -> bool {
         keychains
             .into_iter()
-            .any(|(keychain, index)| self.store_up_to(keychain, *index))
+            .map(|(keychain, index)| self.store_up_to(keychain, *index))
+            .fold(false, |acc, r| acc || r)
     }
 
     /// Derives script pubkeys from the descriptor **up to and including** `up_to` and stores them
@@ -281,6 +283,42 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
             })
             .collect()
     }
+
+    pub fn derive_until_unused_gap(&mut self, gap: u32) -> bool {
+        if gap < 1 {
+            return false;
+        }
+
+        let up_to_per_keychain = self
+            .keychains()
+            .iter()
+            .map(|(keychain, _)| {
+                let up_to = self
+                    .last_active_index(keychain)
+                    .map(|i| i + gap)
+                    .unwrap_or(gap.saturating_sub(1));
+                (keychain.clone(), up_to)
+            })
+            .collect::<BTreeMap<_, _>>();
+
+        self.store_all_up_to(&up_to_per_keychain)
+    }
+
+    pub fn prune_unused(&mut self, mut keychain_bounds: BTreeMap<K, u32>) {
+        for (keychain, _) in &self.keychains {
+            keychain_bounds.entry(keychain.clone()).or_insert(0);
+        }
+        for (keychain, lower_bound) in keychain_bounds {
+            let indexes_to_prune = self
+                .inner
+                .unused((keychain.clone(), lower_bound)..=(keychain.clone(), u32::MAX))
+                .map(|((_, i), _)| *i)
+                .collect::<Vec<_>>();
+            for index in indexes_to_prune {
+                self.inner.remove_unused(&(keychain.clone(), index));
+            }
+        }
+    }
 }
 
 fn descriptor_into_script_iter(

bdk_rpc_example/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "bdk_rpc_example"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+# BDK Core
+bdk_core = { path = "../bdk_core", features = ["serde"] }
+bdk_cli = { path = "../bdk_cli_lib"}
+bdk_keychain = { path = "../bdk_keychain" }
+bdk_file_store = { path = "../bdk_file_store"}
+
+# Rpc
+bitcoincore-rpc = "0.16.0"
+
+# Helpers
+ctrlc = "3.2.4"