start introducing persistence

APIs look ok but not using it yet.

Cargo.toml

@@ -19,7 +19,7 @@ miniscript = { version = "9.0.0", features = ["serde"]  }
 bitcoin = { version = "0.29" , features = ["serde", "base64", "rand"]}
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
-bdk_chain = { git = "https://github.com/LLFourn/bdk_core_staging.git",  rev = "666ef9acdeb09af9d220a267f4784929886b09a6", features = ["miniscript", "serde"] }
+bdk_chain = { git = "https://github.com/LLFourn/bdk_core_staging.git",  rev = "842cfc8b50e1eca5b1d83b598eb3d2e4ae2147ca", features = ["miniscript", "serde"] }
 rand = "^0.8"
 
 # Optional dependencies
@@ -37,6 +37,7 @@ js-sys = "0.3"
 [features]
 default = ["std"]
 std = []
+file-store = [ "std", "bdk_chain/file_store"]
 compiler = ["miniscript/compiler"]
 all-keys = ["keys-bip39"]
 keys-bip39 = ["bip39"]

src/wallet/export.rs

@@ -117,8 +117,8 @@ impl FullyNodedExport {
     ///
     /// If the database is empty or `include_blockheight` is false, the `blockheight` field
     /// returned will be `0`.
-    pub fn export_wallet(
-        wallet: &Wallet,
+    pub fn export_wallet<D>(
+        wallet: &Wallet<D>,
         label: &str,
         include_blockheight: bool,
     ) -> Result<Self, &'static str> {
@@ -231,8 +231,8 @@ mod test {
         descriptor: &str,
         change_descriptor: Option<&str>,
         network: Network,
-    ) -> Wallet {
-        let mut wallet = Wallet::new(descriptor, change_descriptor, network).unwrap();
+    ) -> Wallet<()> {
+        let mut wallet = Wallet::new(descriptor, change_descriptor, (), network).unwrap();
         let transaction = Transaction {
             input: vec![],
             output: vec![],

src/wallet/mod.rs

@@ -19,7 +19,7 @@ use alloc::{
     sync::Arc,
     vec::Vec,
 };
-use bdk_chain::{keychain::KeychainTracker, sparse_chain, BlockId, ConfirmationTime};
+use bdk_chain::{keychain::KeychainTracker, sparse_chain, BlockId, ConfirmationTime, chain_graph};
 use bitcoin::secp256k1::Secp256k1;
 use core::fmt;
 use core::ops::Deref;
@@ -45,6 +45,7 @@ pub(crate) mod utils;
 #[cfg(feature = "hardware-signer")]
 #[cfg_attr(docsrs, doc(cfg(feature = "hardware-signer")))]
 pub mod hardwaresigner;
+pub mod persist;
 
 pub use utils::IsDust;
 
@@ -79,13 +80,12 @@ const COINBASE_MATURITY: u32 = 100;
 /// [`Database`]: crate::database::Database
 /// [`signer`]: crate::signer
 #[derive(Debug)]
-pub struct Wallet {
+pub struct Wallet<D = ()> {
     signers: Arc<SignersContainer>,
     change_signers: Arc<SignersContainer>,
     keychain_tracker: KeychainTracker<KeychainKind, ConfirmationTime>,
-
+    persist: persist::Persist<D>,
     network: Network,
-
     secp: SecpCtx,
 }
 
@@ -138,26 +138,67 @@ impl fmt::Display for AddressInfo {
 }
 
 impl Wallet {
+    /// Creates a wallet that does not persist data.
+    pub fn new_no_persist<E: IntoWalletDescriptor>(
+        descriptor: E,
+        change_descriptor: Option<E>,
+        network: Network,
+    ) -> Result<Self, crate::descriptor::DescriptorError> {
+        Self::new(descriptor, change_descriptor, (), network).map_err(|e| match e {
+            NewError::Descriptor(e) => e,
+            NewError::Persist(_) => unreachable!("no persistence so it can't fail")
+        })
+    }
+}
+
+#[derive(Debug)]
+pub enum NewError<P> {
+    Descriptor(crate::descriptor::DescriptorError),
+    Persist(P),
+}
+
+impl<P> core::fmt::Display for NewError<P>
+where
+    P: core::fmt::Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            NewError::Descriptor(e) => e.fmt(f),
+            NewError::Persist(e) => write!(f, "failed to load wallet from database: {}", e),
+        }
+    }
+}
+
+#[cfg(feautre = "std")]
+impl<P: core::fmt::Display> std::error::Error for NewError<P> {}
+
+impl<D> Wallet<D> {
     /// Create a wallet.
     ///
     /// The only way this can fail is if the descriptors passed in do not match the checksums in `database`.
     pub fn new<E: IntoWalletDescriptor>(
         descriptor: E,
         change_descriptor: Option<E>,
+        mut db: D,
         network: Network,
-    ) -> Result<Self, Error> {
+    ) -> Result<Self, NewError<D::LoadError>>
+    where
+        D: persist::Backend,
+    {
         let secp = Secp256k1::new();
 
         let mut keychain_tracker = KeychainTracker::default();
-        let (descriptor, keymap) = into_wallet_descriptor_checked(descriptor, &secp, network)?;
+        let (descriptor, keymap) = into_wallet_descriptor_checked(descriptor, &secp, network)
+            .map_err(NewError::Descriptor)?;
         keychain_tracker
             .txout_index
             .add_keychain(KeychainKind::External, descriptor.clone());
         let signers = Arc::new(SignersContainer::build(keymap, &descriptor, &secp));
         let change_signers = match change_descriptor {
             Some(desc) => {
                 let (change_descriptor, change_keymap) =
-                    into_wallet_descriptor_checked(desc, &secp, network)?;
+                    into_wallet_descriptor_checked(desc, &secp, network)
+                        .map_err(NewError::Descriptor)?;
 
                 let change_signers = Arc::new(SignersContainer::build(
                     change_keymap,
@@ -174,10 +215,17 @@ impl Wallet {
             None => Arc::new(SignersContainer::new()),
         };
 
+        db.load_into_keychain_tracker(&mut keychain_tracker)
+            .map_err(NewError::Persist)?;
+
+
+        let persist = persist::Persist::new(db);
+
         Ok(Wallet {
             signers,
             change_signers,
             network,
+            persist,
             secp,
             keychain_tracker,
         })
@@ -370,7 +418,11 @@ impl Wallet {
         let inputs = tx
             .input
             .iter()
-            .map(|txin| graph.txout(txin.previous_output).map(|txout| txout.value))
+            .map(|txin| {
+                graph
+                    .get_txout(txin.previous_output)
+                    .map(|txout| txout.value)
+            })
             .sum::<Option<u64>>();
         let outputs = tx.output.iter().map(|txout| txout.value).sum();
         let fee = inputs.map(|inputs| inputs.saturating_sub(outputs));
@@ -389,25 +441,31 @@ impl Wallet {
     pub fn insert_checkpoint(
         &mut self,
         block_id: BlockId,
-    ) -> Result<bool, sparse_chain::InsertCheckpointErr> {
-        self.keychain_tracker.insert_checkpoint(block_id)
+    ) -> Result<bool, sparse_chain::InsertCheckpointError> {
+        let changeset = self.keychain_tracker.insert_checkpoint(block_id)?;
+        let changed = changeset.is_empty();
+        self.persist.stage(changeset);
+        Ok(changed)
     }
 
     /// Add a transaction to the wallet. Will only work if height <= latest checkpoint
     pub fn insert_tx(
         &mut self,
         tx: Transaction,
         position: ConfirmationTime,
-    ) -> Result<bool, sparse_chain::InsertTxErr> {
-        self.keychain_tracker.insert_tx(tx, Some(position))
+    ) -> Result<bool, chain_graph::InsertTxError<ConfirmationTime>> {
+        let changeset = self.keychain_tracker.insert_tx(tx, position)?;
+        let changed = changeset.is_empty();
+        self.persist.stage(changeset);
+        Ok(changed)
     }
 
     #[deprecated(note = "use Wallet::transactions instead")]
     /// Deprecated. use `Wallet::transactions` instead.
     pub fn list_transactions(&self, include_raw: bool) -> Vec<TransactionDetails> {
         self.keychain_tracker
             .chain()
-            .iter_txids()
+            .txids()
             .map(|(_, txid)| self.get_tx(*txid, include_raw).expect("must exist"))
             .collect()
     }
@@ -418,14 +476,9 @@ impl Wallet {
         &self,
     ) -> impl DoubleEndedIterator<Item = (ConfirmationTime, &Transaction)> + '_ {
         self.keychain_tracker
-            .chain()
-            .iter_txids()
-            .map(move |(pos, txid)| {
-                (
-                    *pos,
-                    self.keychain_tracker.graph().tx(*txid).expect("must exist"),
-                )
-            })
+            .chain_graph()
+            .transactions_in_chain()
+            .map(|(pos, tx)| (*pos, tx))
     }
 
     /// Return the balance, separated into available, trusted-pending, untrusted-pending and immature
@@ -449,7 +502,7 @@ impl Wallet {
             let is_coinbase = self
                 .keychain_tracker
                 .graph()
-                .tx(utxo.outpoint.txid)
+                .get_tx(utxo.outpoint.txid)
                 .expect("must exist")
                 .is_coin_base();
 
@@ -501,7 +554,7 @@ impl Wallet {
     /// # use bdk::{Wallet, KeychainKind};
     /// # use bdk::bitcoin::Network;
     /// # use bdk::database::MemoryDatabase;
-    /// let wallet = Wallet::new("wpkh(tprv8ZgxMBicQKsPe73PBRSmNbTfbcsZnwWhz5eVmhHpi31HW29Z7mc9B4cWGRQzopNUzZUT391DeDJxL2PefNunWyLgqCKRMDkU1s2s8bAfoSk/84'/0'/0'/0/*)", None, Network::Testnet, MemoryDatabase::new())?;
+    /// let wallet = Wallet::new_no_persist("wpkh(tprv8ZgxMBicQKsPe73PBRSmNbTfbcsZnwWhz5eVmhHpi31HW29Z7mc9B4cWGRQzopNUzZUT391DeDJxL2PefNunWyLgqCKRMDkU1s2s8bAfoSk/84'/0'/0'/0/*)", None, Network::Testnet, MemoryDatabase::new())?;
     /// for secret_key in wallet.get_signers(KeychainKind::External).signers().iter().filter_map(|s| s.descriptor_secret_key()) {
     ///     // secret_key: tprv8ZgxMBicQKsPe73PBRSmNbTfbcsZnwWhz5eVmhHpi31HW29Z7mc9B4cWGRQzopNUzZUT391DeDJxL2PefNunWyLgqCKRMDkU1s2s8bAfoSk/84'/0'/0'/0/*
     ///     println!("secret_key: {}", secret_key);
@@ -542,7 +595,7 @@ impl Wallet {
     /// ```
     ///
     /// [`TxBuilder`]: crate::TxBuilder
-    pub fn build_tx(&mut self) -> TxBuilder<'_, DefaultCoinSelectionAlgorithm, CreateTx> {
+    pub fn build_tx(&mut self) -> TxBuilder<'_, D, DefaultCoinSelectionAlgorithm, CreateTx> {
         TxBuilder {
             wallet: alloc::rc::Rc::new(core::cell::RefCell::new(self)),
             params: TxParams::default(),
@@ -950,7 +1003,7 @@ impl Wallet {
     pub fn build_fee_bump(
         &mut self,
         txid: Txid,
-    ) -> Result<TxBuilder<'_, DefaultCoinSelectionAlgorithm, BumpFee>, Error> {
+    ) -> Result<TxBuilder<'_, D, DefaultCoinSelectionAlgorithm, BumpFee>, Error> {
         let graph = self.keychain_tracker.graph();
         let txout_index = &self.keychain_tracker.txout_index;
         let tx_and_height = self.keychain_tracker.chain_graph().get_tx_in_chain(txid);
@@ -971,6 +1024,11 @@ impl Wallet {
         }
 
         let fee = graph.calculate_fee(&tx).ok_or(Error::FeeRateUnavailable)?;
+        if fee < 0 {
+            // It's available but it's wrong so let's say it's unavailable
+            return Err(Error::FeeRateUnavailable)?;
+        }
+        let fee = fee as u64;
         let feerate = FeeRate::from_wu(fee, tx.weight());
 
         // remove the inputs from the tx and process them
@@ -1525,7 +1583,7 @@ impl Wallet {
             .map_err(MiniscriptPsbtError::Conversion)?;
 
         let prev_output = utxo.outpoint;
-        if let Some(prev_tx) = self.keychain_tracker.graph().tx(prev_output.txid) {
+        if let Some(prev_tx) = self.keychain_tracker.graph().get_tx(prev_output.txid) {
             if desc.is_witness() || desc.is_taproot() {
                 psbt_input.witness_utxo = Some(prev_tx.output[prev_output.vout as usize].clone());
             }
@@ -1644,7 +1702,7 @@ pub(crate) mod test {
 
     /// Return a fake wallet that appears to be funded for testing.
     pub fn get_funded_wallet(descriptor: &str) -> (Wallet, bitcoin::Txid) {
-        let mut wallet = Wallet::new(descriptor, None, Network::Regtest).unwrap();
+        let mut wallet = Wallet::new_no_persist(descriptor, None, Network::Regtest).unwrap();
         let address = wallet.get_address(AddressIndex::New).address;
 
         let tx = Transaction {
@@ -2578,7 +2636,7 @@ pub(crate) mod test {
     #[test]
     fn test_create_tx_policy_path_no_csv() {
         let descriptors = get_test_wpkh();
-        let mut wallet = Wallet::new(descriptors, None, Network::Regtest).unwrap();
+        let mut wallet = Wallet::new_no_persist(descriptors, None, Network::Regtest).unwrap();
 
         let tx = Transaction {
             version: 0,
@@ -3914,7 +3972,7 @@ pub(crate) mod test {
 
     #[test]
     fn test_unused_address() {
-        let mut wallet = Wallet::new("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
+        let mut wallet = Wallet::new_no_persist("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
                                              None, Network::Testnet).unwrap();
 
         assert_eq!(
@@ -3930,7 +3988,7 @@ pub(crate) mod test {
     #[test]
     fn test_next_unused_address() {
         let descriptor = "wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)";
-        let mut wallet = Wallet::new(descriptor, None, Network::Testnet).unwrap();
+        let mut wallet = Wallet::new_no_persist(descriptor, None, Network::Testnet).unwrap();
         assert_eq!(wallet.derivation_index(KeychainKind::External), None);
 
         assert_eq!(
@@ -3956,7 +4014,7 @@ pub(crate) mod test {
 
     #[test]
     fn test_peek_address_at_index() {
-        let mut wallet = Wallet::new("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
+        let mut wallet = Wallet::new_no_persist("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
                                              None, Network::Testnet).unwrap();
 
         assert_eq!(
@@ -3988,7 +4046,7 @@ pub(crate) mod test {
 
     #[test]
     fn test_peek_address_at_index_not_derivable() {
-        let mut wallet = Wallet::new("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/1)",
+        let mut wallet = Wallet::new_no_persist("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/1)",
                                              None, Network::Testnet).unwrap();
 
         assert_eq!(
@@ -4009,7 +4067,7 @@ pub(crate) mod test {
 
     #[test]
     fn test_returns_index_and_address() {
-        let mut wallet = Wallet::new("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
+        let mut wallet = Wallet::new_no_persist("wpkh(tpubEBr4i6yk5nf5DAaJpsi9N2pPYBeJ7fZ5Z9rmN4977iYLCGco1VyjB9tvvuvYtfZzjD5A8igzgw3HeWeeKFmanHYqksqZXYXGsw5zjnj7KM9/*)",
                                              None, Network::Testnet).unwrap();
 
         // new index 0
@@ -4068,7 +4126,7 @@ pub(crate) mod test {
     fn test_get_address() {
         use crate::descriptor::template::Bip84;
         let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
-        let mut wallet = Wallet::new(
+        let mut wallet = Wallet::new_no_persist(
             Bip84(key, KeychainKind::External),
             Some(Bip84(key, KeychainKind::Internal)),
             Network::Regtest,
@@ -4094,7 +4152,8 @@ pub(crate) mod test {
         );
 
         let mut wallet =
-            Wallet::new(Bip84(key, KeychainKind::External), None, Network::Regtest).unwrap();
+            Wallet::new_no_persist(Bip84(key, KeychainKind::External), None, Network::Regtest)
+                .unwrap();
 
         assert_eq!(
             wallet.get_internal_address(AddressIndex::New),
@@ -4114,7 +4173,8 @@ pub(crate) mod test {
 
         let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let mut wallet =
-            Wallet::new(Bip84(key, KeychainKind::External), None, Network::Regtest).unwrap();
+            Wallet::new_no_persist(Bip84(key, KeychainKind::External), None, Network::Regtest)
+                .unwrap();
 
         let mut used_set = HashSet::new();
 
@@ -4582,7 +4642,7 @@ pub(crate) mod test {
 
         // re-create the wallet with an empty db
         let wallet_empty =
-            Wallet::new(get_test_tr_single_sig_xprv(), None, Network::Regtest).unwrap();
+            Wallet::new_no_persist(get_test_tr_single_sig_xprv(), None, Network::Regtest).unwrap();
 
         // signing with an empty db means that we will only look at the psbt to infer the
         // derivation index
@@ -4682,7 +4742,7 @@ pub(crate) mod test {
     #[test]
     fn test_spend_coinbase() {
         let descriptor = get_test_wpkh();
-        let mut wallet = Wallet::new(descriptor, None, Network::Regtest).unwrap();
+        let mut wallet = Wallet::new_no_persist(descriptor, None, Network::Regtest).unwrap();
 
         let confirmation_height = 5;
         wallet