WIP: Inital work on RPC chain example

.gitignore

@@ -4,3 +4,5 @@ Cargo.lock
 out.perf-folded
 perf.data
 rust-perf.svg
+
+.bdk_example_db

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "bdk_core",
     "bdk_cli_lib",
     "bdk_esplora_example",
+    "bdk_rpc_example",
     "bdk_keychain",
     "bdk_file_store"
 ]

bdk_cli_lib/src/lib.rs

@@ -19,12 +19,13 @@ use bdk_keychain::{
 };
 pub use clap;
 use clap::{Parser, Subcommand};
+pub use log;
 use std::{cmp::Reverse, collections::HashMap, fmt::Debug, path::PathBuf, time::Duration};
 
 #[derive(Parser)]
 #[clap(author, version, about, long_about = None)]
 #[clap(propagate_version = true)]
-pub struct Args<C: clap::Subcommand> {
+pub struct Args<A: clap::Args, C: clap::Subcommand> {
     #[clap(env = "DESCRIPTOR")]
     pub descriptor: String,
     #[clap(env = "CHANGE_DESCRIPTOR")]
@@ -36,6 +37,9 @@ pub struct Args<C: clap::Subcommand> {
     #[clap(env = "BDK_DB_DIR", long, default_value = ".bdk_example_db")]
     pub db_dir: PathBuf,
 
+    #[clap(flatten)]
+    pub chain_args: A,
+
     #[clap(subcommand)]
     pub command: Commands<C>,
 }
@@ -500,8 +504,8 @@ where
     Ok(())
 }
 
-pub fn init<C: clap::Subcommand, I>() -> anyhow::Result<(
-    Args<C>,
+pub fn init<A: clap::Args, C: clap::Subcommand, I>() -> anyhow::Result<(
+    Args<A, C>,
     KeyMap,
     KeychainTracker<Keychain, I>,
     KeychainStore<Keychain, I>,
@@ -510,7 +514,7 @@ where
     I: sparse_chain::ChainIndex,
     KeychainChangeSet<Keychain, I>: serde::Serialize + serde::de::DeserializeOwned,
 {
-    let args = Args::<C>::parse();
+    let args = Args::<A, C>::parse();
     let secp = Secp256k1::default();
     let (descriptor, mut keymap) =
         Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &args.descriptor)?;

bdk_core/src/spk_txout_index.rs

@@ -1,3 +1,5 @@
+use core::ops::RangeBounds;
+
 use crate::{
     collections::{BTreeMap, BTreeSet, HashMap, HashSet},
     ForEachTxout,
@@ -63,7 +65,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     ///
     /// [`ForEachTxout`]: crate::ForEachTxout
     pub fn scan(&mut self, txouts: &impl ForEachTxout) {
-        txouts.for_each_txout(&mut |(op, txout)| self.scan_txout(op, txout))
+        txouts.for_each_txout(&mut |(op, txout)| self.scan_txout(op, txout));
     }
 
     /// Scan a single `TxOut` for a matching script pubkey
@@ -128,6 +130,24 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
         self.unused.insert(index);
     }
 
+    pub fn remove_unused(&mut self, index: &I) -> bool {
+        let is_removed = self.unused.remove(index);
+        if is_removed {
+            let spk = self.script_pubkeys.remove(index).expect("should exist");
+            self.spk_indexes.remove(&spk);
+        }
+        is_removed
+    }
+
+    pub fn split_off_unused(&mut self, index: &I) -> BTreeSet<I> {
+        let set = self.unused.split_off(index);
+        for index in &set {
+            let spk = self.script_pubkeys.remove(index).expect("should exist");
+            self.spk_indexes.remove(&spk);
+        }
+        set
+    }
+
     /// Iterate over the script pubkeys that have been derived but do not have a transaction spending to them.
     pub fn iter_unused(
         &self,
@@ -137,6 +157,15 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
             .map(|index| (index, self.spk_at_index(index).expect("must exist")))
     }
 
+    pub fn range_unused<R>(&self, range: R) -> impl DoubleEndedIterator<Item = (&I, &Script)>
+    where
+        R: RangeBounds<I>,
+    {
+        self.unused
+            .range(range)
+            .map(|index| (index, self.spk_at_index(index).expect("must exist")))
+    }
+
     /// Returns whether the script pubkey at index `index` has been used or not.
     ///
     /// i.e. has a transaction which spends to it.

bdk_esplora_example/src/main.rs

@@ -7,9 +7,15 @@ use std::io::{self, Write};
 const DEFAULT_PARALLEL_REQUESTS: u8 = 5;
 use bdk_cli::{
     anyhow::{self, Context},
-    clap::{self, Subcommand},
+    clap::{self, Args, Subcommand},
 };
 
+#[derive(Args, Debug, Clone)]
+struct EsploraUrlArgs {
+    #[clap(env = "ESPLORA_URL", long)]
+    url: Option<String>,
+}
+
 #[derive(Subcommand, Debug, Clone)]
 enum EsploraCommands {
     /// Scans the addresses in the wallet using esplora API.
@@ -33,12 +39,17 @@ enum EsploraCommands {
 }
 
 fn main() -> anyhow::Result<()> {
-    let (args, keymap, mut keychain_tracker, mut db) = bdk_cli::init::<EsploraCommands, _>()?;
-    let esplora_url = match args.network {
-        Network::Bitcoin => "https://mempool.space/api",
-        Network::Testnet => "https://mempool.space/testnet/api",
-        Network::Regtest => "http://localhost:3000",
-        Network::Signet => "https://mempool.space/signet/api",
+    let (args, keymap, mut keychain_tracker, mut db) =
+        bdk_cli::init::<EsploraUrlArgs, EsploraCommands, _>()?;
+
+    let esplora_url = match &args.chain_args.url {
+        Some(url) => url.as_str(),
+        None => match args.network {
+            Network::Bitcoin => "https://mempool.space/api",
+            Network::Testnet => "https://mempool.space/testnet/api",
+            Network::Regtest => "http://localhost:3000",
+            Network::Signet => "https://mempool.space/signet/api",
+        },
     };
 
     let client = Client::new(esplora_url, DEFAULT_PARALLEL_REQUESTS)?;

bdk_keychain/src/keychain_txout_index.rs

@@ -1,3 +1,4 @@
+use alloc::vec::Vec;
 use bdk_core::{
     bitcoin::{secp256k1::Secp256k1, OutPoint, Script, TxOut},
     collections::*,
@@ -79,14 +80,14 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     ///
     /// [`ForEachTxout`]: bdk_core::ForEachTxout
     pub fn scan(&mut self, txouts: &impl ForEachTxout) {
-        self.inner.scan(txouts);
+        self.inner.scan(txouts)
     }
 
     /// Scan a single `TxOut` for a matching script pubkey.
     ///
     /// If it matches the index will store and index it.
     pub fn scan_txout(&mut self, op: OutPoint, txout: &TxOut) {
-        self.inner.scan_txout(op, &txout);
+        self.inner.scan_txout(op, &txout)
     }
 
     pub fn inner(&self) -> &SpkTxOutIndex<(K, u32)> {
@@ -144,8 +145,8 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
         self.inner
             .script_pubkeys()
             .range(&(keychain.clone(), u32::MIN)..=&(keychain.clone(), u32::MAX))
+            .next_back()
             .map(|((_, index), _)| *index)
-            .last()
     }
 
     /// Gets the current derivation index for each keychain in the index.
@@ -156,6 +157,25 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
             .collect()
     }
 
+    pub fn last_active(&self, keychain: &K) -> Option<(u32, &Script)> {
+        self.inner
+            .script_pubkeys()
+            .range((keychain.clone(), u32::MIN)..=(keychain.clone(), u32::MAX))
+            .filter(|(index, _)| self.is_used(index))
+            .next_back()
+            .map(|(&(_, index), script)| (index, script))
+    }
+
+    pub fn last_active_indicies(&self) -> BTreeMap<K, u32> {
+        self.keychains
+            .iter()
+            .filter_map(|(keychain, _)| {
+                self.last_active(keychain)
+                    .map(|(index, _)| (keychain.clone(), index))
+            })
+            .collect()
+    }
+
     /// Convenience method to call [`derive_spks_up_to`] on several keychains.
     ///
     /// Returns whether any new script pubkeys were derived (or if they had already all been
@@ -257,6 +277,42 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
                 .unwrap()
         }
     }
+
+    pub fn derive_until_unused_gap(&mut self, gap: u32) -> bool {
+        if gap == 0 {
+            return false;
+        }
+
+        let up_to_per_keychain = self
+            .keychains()
+            .iter()
+            .map(|(keychain, _)| {
+                let up_to = self
+                    .last_active(keychain)
+                    .map(|(last_active_index, _)| last_active_index + gap)
+                    .unwrap_or(gap.saturating_sub(1));
+                (keychain.clone(), up_to)
+            })
+            .collect::<BTreeMap<_, _>>();
+
+        self.store_all_up_to(&up_to_per_keychain)
+    }
+
+    pub fn prune_unused(&mut self, mut keychain_bounds: BTreeMap<K, u32>) {
+        for (keychain, _) in &self.keychains {
+            keychain_bounds.entry(keychain.clone()).or_insert(0);
+        }
+        for (keychain, lower_bound) in keychain_bounds {
+            let indexes_to_prune = self
+                .inner
+                .range_unused((keychain.clone(), lower_bound)..=(keychain.clone(), u32::MAX))
+                .map(|((_, i), _)| *i)
+                .collect::<Vec<_>>();
+            for index in indexes_to_prune {
+                self.inner.remove_unused(&(keychain.clone(), index));
+            }
+        }
+    }
 }
 
 fn descriptor_into_script_iter(

bdk_rpc_example/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "bdk_rpc_example"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+# BDK Core
+bdk_core = { path = "../bdk_core", features = ["serde"] }
+bdk_cli = { path = "../bdk_cli_lib"}
+bdk_keychain = { path = "../bdk_keychain" }
+bdk_file_store = { path = "../bdk_file_store"}
+
+# Rpc
+bitcoincore-rpc = "0.16.0"
+
+# Helpers
+ctrlc = "3.2.4"