feat(wallet_ffi): wallet_get_utxos() (#4209)

Description
---
Added an FFI function to get a list utxos, a simple output storage querying interface and a circumvented output database access from the FFI context.

Motivation and Context
---
To let mobile wallets obtain a list of UTXOs.

How Has This Been Tested?
---
individual unit tests and `cargo test`


* added FFI `wallet_get_utxos()`, `destroy_tari_outputs` + unit test
added a simple output querying function

Signed-off-by: Andrey Gubarev <agubarev@protonmail.com>

* added FFI `wallet_get_utxos()`, `destroy_tari_outputs` + unit test.
added a simple output querying function.
added an output database shortcut for easier access from the wallet context, circumventing hops through async fall-through constructs; updated all usages - `cargo test` passing.

Signed-off-by: Andrey Gubarev <agubarev@protonmail.com>

applications/tari_console_wallet/src/init/mod.rs

@@ -41,6 +41,7 @@ use tari_p2p::{initialization::CommsInitializationError, peer_seeds::SeedPeer, T
 use tari_shutdown::ShutdownSignal;
 use tari_wallet::{
     error::{WalletError, WalletStorageError},
+    output_manager_service::storage::database::OutputManagerDatabase,
     storage::{
         database::{WalletBackend, WalletDatabase},
         sqlite_utilities::initialize_sqlite_database_backends,
@@ -260,6 +261,7 @@ pub async fn init_wallet(
     };
     let (wallet_backend, transaction_backend, output_manager_backend, contacts_backend, key_manager_backend) = backends;
     let wallet_db = WalletDatabase::new(wallet_backend);
+    let output_db = OutputManagerDatabase::new(output_manager_backend.clone());
 
     debug!(
         target: LOG_TARGET,
@@ -306,6 +308,7 @@ pub async fn init_wallet(
         node_identity,
         factories,
         wallet_db,
+        output_db,
         transaction_backend,
         output_manager_backend,
         contacts_backend,

base_layer/wallet/src/output_manager_service/handle.rs

@@ -53,7 +53,10 @@ use tower::Service;
 use crate::output_manager_service::{
     error::OutputManagerError,
     service::{Balance, OutputStatusesByTxId},
-    storage::models::{KnownOneSidedPaymentScript, SpendingPriority},
+    storage::{
+        database::OutputBackendQuery,
+        models::{KnownOneSidedPaymentScript, SpendingPriority},
+    },
 };
 
 /// API Request enum
@@ -101,6 +104,7 @@ pub enum OutputManagerRequest {
     CancelTransaction(TxId),
     GetSpentOutputs,
     GetUnspentOutputs,
+    GetOutputsBy(OutputBackendQuery),
     GetInvalidOutputs,
     ValidateUtxos,
     RevalidateTxos,
@@ -164,6 +168,7 @@ impl fmt::Display for OutputManagerRequest {
             CancelTransaction(v) => write!(f, "CancelTransaction ({})", v),
             GetSpentOutputs => write!(f, "GetSpentOutputs"),
             GetUnspentOutputs => write!(f, "GetUnspentOutputs"),
+            GetOutputsBy(q) => write!(f, "GetOutputs({:#?})", q),
             GetInvalidOutputs => write!(f, "GetInvalidOutputs"),
             ValidateUtxos => write!(f, "ValidateUtxos"),
             RevalidateTxos => write!(f, "RevalidateTxos"),
@@ -234,6 +239,7 @@ pub enum OutputManagerResponse {
     TransactionCancelled,
     SpentOutputs(Vec<UnblindedOutput>),
     UnspentOutputs(Vec<UnblindedOutput>),
+    Outputs(Vec<UnblindedOutput>),
     InvalidOutputs(Vec<UnblindedOutput>),
     BaseNodePublicKeySet,
     TxoValidationStarted(u64),
@@ -604,6 +610,13 @@ impl OutputManagerHandle {
         }
     }
 
+    pub async fn get_outputs_by(&mut self, q: OutputBackendQuery) -> Result<Vec<UnblindedOutput>, OutputManagerError> {
+        match self.handle.call(OutputManagerRequest::GetOutputsBy(q)).await?? {
+            OutputManagerResponse::Outputs(s) => Ok(s),
+            _ => Err(OutputManagerError::UnexpectedApiResponse),
+        }
+    }
+
     pub async fn get_invalid_outputs(&mut self) -> Result<Vec<UnblindedOutput>, OutputManagerError> {
         match self.handle.call(OutputManagerRequest::GetInvalidOutputs).await?? {
             OutputManagerResponse::InvalidOutputs(s) => Ok(s),

base_layer/wallet/src/output_manager_service/service.rs

@@ -83,7 +83,7 @@ use crate::{
         recovery::StandardUtxoRecoverer,
         resources::{OutputManagerKeyManagerBranch, OutputManagerResources},
         storage::{
-            database::{OutputManagerBackend, OutputManagerDatabase},
+            database::{OutputBackendQuery, OutputManagerBackend, OutputManagerDatabase},
             models::{DbUnblindedOutput, KnownOneSidedPaymentScript, SpendingPriority},
             OutputStatus,
         },
@@ -362,6 +362,10 @@ where
                 let outputs = self.fetch_unspent_outputs()?.into_iter().map(|v| v.into()).collect();
                 Ok(OutputManagerResponse::UnspentOutputs(outputs))
             },
+            OutputManagerRequest::GetOutputsBy(q) => {
+                let outputs = self.fetch_outputs_by(q)?.into_iter().map(|v| v.into()).collect();
+                Ok(OutputManagerResponse::Outputs(outputs))
+            },
             OutputManagerRequest::ValidateUtxos => {
                 self.validate_outputs().map(OutputManagerResponse::TxoValidationStarted)
             },
@@ -1569,6 +1573,10 @@ where
         Ok(self.resources.db.fetch_all_unspent_outputs()?)
     }
 
+    pub fn fetch_outputs_by(&self, q: OutputBackendQuery) -> Result<Vec<DbUnblindedOutput>, OutputManagerError> {
+        Ok(self.resources.db.fetch_outputs_by(q)?)
+    }
+
     pub fn fetch_invalid_outputs(&self) -> Result<Vec<DbUnblindedOutput>, OutputManagerError> {
         Ok(self.resources.db.get_invalid_outputs()?)
     }

base_layer/wallet/src/output_manager_service/storage/database/backend.rs

@@ -12,7 +12,7 @@ use crate::output_manager_service::{
     error::OutputManagerStorageError,
     service::{Balance, UTXOSelectionStrategy},
     storage::{
-        database::{DbKey, DbValue, WriteOperation},
+        database::{DbKey, DbValue, OutputBackendQuery, WriteOperation},
         models::DbUnblindedOutput,
     },
 };
@@ -114,4 +114,5 @@ pub trait OutputManagerBackend: Send + Sync + Clone {
         current_tip_height: Option<u64>,
     ) -> Result<Vec<DbUnblindedOutput>, OutputManagerStorageError>;
     fn fetch_outputs_by_tx_id(&self, tx_id: TxId) -> Result<Vec<DbUnblindedOutput>, OutputManagerStorageError>;
+    fn fetch_outputs_by(&self, q: OutputBackendQuery) -> Result<Vec<DbUnblindedOutput>, OutputManagerStorageError>;
 }

base_layer/wallet/src/output_manager_service/storage/database/mod.rs

@@ -22,7 +22,7 @@
 
 mod backend;
 use std::{
-    fmt::{Display, Error, Formatter},
+    fmt::{Debug, Display, Error, Formatter},
     sync::Arc,
 };
 
@@ -50,6 +50,35 @@ use crate::output_manager_service::{
 
 const LOG_TARGET: &str = "wallet::output_manager_service::database";
 
+#[derive(Debug, Copy, Clone)]
+pub enum SortDirection {
+    Asc,
+    Desc,
+}
+
+#[derive(Debug, Clone)]
+pub struct OutputBackendQuery {
+    pub tip_height: i64,
+    pub status: Vec<OutputStatus>,
+    pub pagination: Option<(i64, i64)>,
+    pub value_min: Option<(i64, bool)>,
+    pub value_max: Option<(i64, bool)>,
+    pub sorting: Vec<(&'static str, SortDirection)>,
+}
+
+impl Default for OutputBackendQuery {
+    fn default() -> Self {
+        Self {
+            tip_height: i64::MAX,
+            status: vec![OutputStatus::Spent],
+            pagination: None,
+            value_min: None,
+            value_max: None,
+            sorting: vec![],
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub enum DbKey {
     SpentOutput(BlindingFactor),
@@ -419,6 +448,10 @@ where T: OutputManagerBackend + 'static
         let outputs = self.db.fetch_outputs_by_tx_id(tx_id)?;
         Ok(outputs)
     }
+
+    pub fn fetch_outputs_by(&self, q: OutputBackendQuery) -> Result<Vec<DbUnblindedOutput>, OutputManagerStorageError> {
+        self.db.fetch_outputs_by(q)
+    }
 }
 
 fn unexpected_result<T>(req: DbKey, res: DbValue) -> Result<T, OutputManagerStorageError> {

base_layer/wallet/src/output_manager_service/storage/sqlite_db/mod.rs

@@ -45,7 +45,7 @@ use crate::{
         error::OutputManagerStorageError,
         service::{Balance, UTXOSelectionStrategy},
         storage::{
-            database::{DbKey, DbKeyValuePair, DbValue, OutputManagerBackend, WriteOperation},
+            database::{DbKey, DbKeyValuePair, DbValue, OutputBackendQuery, OutputManagerBackend, WriteOperation},
             models::{DbUnblindedOutput, KnownOneSidedPaymentScript},
             OutputStatus,
         },
@@ -1227,6 +1227,29 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase {
             .map(|o| DbUnblindedOutput::try_from(o.clone()))
             .collect::<Result<Vec<_>, _>>()
     }
+
+    fn fetch_outputs_by(&self, q: OutputBackendQuery) -> Result<Vec<DbUnblindedOutput>, OutputManagerStorageError> {
+        let conn = self.database_connection.get_pooled_connection()?;
+        Ok(OutputSql::fetch_outputs_by(q, &conn)?
+            .into_iter()
+            .filter_map(|mut x| {
+                if let Err(e) = self.decrypt_if_necessary(&mut x) {
+                    error!(target: LOG_TARGET, "failed to `decrypt_if_necessary`: {:#?}", e);
+                    return None;
+                }
+
+                DbUnblindedOutput::try_from(x)
+                    .map_err(|e| {
+                        error!(
+                            target: LOG_TARGET,
+                            "failed to convert `OutputSql` to `DbUnblindedOutput`: {:#?}", e
+                        );
+                        e
+                    })
+                    .ok()
+            })
+            .collect())
+    }
 }
 
 /// These are the fields that can be updated for an Output

base_layer/wallet/src/output_manager_service/storage/sqlite_db/output_sql.rs

@@ -47,6 +47,7 @@ use crate::{
         error::OutputManagerStorageError,
         service::{Balance, UTXOSelectionStrategy},
         storage::{
+            database::{OutputBackendQuery, SortDirection},
             models::DbUnblindedOutput,
             sqlite_db::{UpdateOutput, UpdateOutputSql},
             OutputStatus,
@@ -114,6 +115,66 @@ impl OutputSql {
         Ok(outputs::table.filter(outputs::status.eq(status as i32)).load(conn)?)
     }
 
+    /// Retrieves UTXOs by a set of given rules
+    // TODO: maybe use a shorthand macros
+    #[allow(clippy::cast_sign_loss)]
+    pub fn fetch_outputs_by(
+        q: OutputBackendQuery,
+        conn: &SqliteConnection,
+    ) -> Result<Vec<OutputSql>, OutputManagerStorageError> {
+        let mut query = outputs::table
+            .into_boxed()
+            .filter(outputs::script_lock_height.le(q.tip_height))
+            .filter(outputs::maturity.le(q.tip_height))
+            .filter(outputs::features_unique_id.is_null())
+            .filter(outputs::features_parent_public_key.is_null());
+
+        if let Some((offset, limit)) = q.pagination {
+            query = query.offset(offset).limit(limit);
+        }
+
+        // filtering by OutputStatus
+        query = match q.status.len() {
+            0 => query,
+            1 => query.filter(outputs::status.eq(q.status[0] as i32)),
+            _ => query.filter(outputs::status.eq_any::<Vec<i32>>(q.status.into_iter().map(|s| s as i32).collect())),
+        };
+
+        // if set, filtering by minimum value
+        if let Some((min, is_inclusive)) = q.value_min {
+            query = if is_inclusive {
+                query.filter(outputs::value.ge(min))
+            } else {
+                query.filter(outputs::value.gt(min))
+            };
+        }
+
+        // if set, filtering by max value
+        if let Some((max, is_inclusive)) = q.value_max {
+            query = if is_inclusive {
+                query.filter(outputs::value.le(max))
+            } else {
+                query.filter(outputs::value.lt(max))
+            };
+        }
+
+        use SortDirection::{Asc, Desc};
+        Ok(q.sorting
+            .into_iter()
+            .fold(query, |query, s| match s {
+                ("value", d) => match d {
+                    Asc => query.then_order_by(outputs::value.asc()),
+                    Desc => query.then_order_by(outputs::value.desc()),
+                },
+                ("mined_height", d) => match d {
+                    Asc => query.then_order_by(outputs::mined_height.asc()),
+                    Desc => query.then_order_by(outputs::mined_height.desc()),
+                },
+                _ => query,
+            })
+            .load(conn)?)
+    }
+
     /// Retrieves UTXOs than can be spent, sorted by priority, then value from smallest to largest.
     #[allow(clippy::cast_sign_loss)]
     pub fn fetch_unspent_outputs_for_spending(

base_layer/wallet/src/wallet.rs

@@ -86,7 +86,10 @@ use crate::{
     output_manager_service::{
         error::OutputManagerError,
         handle::OutputManagerHandle,
-        storage::{database::OutputManagerBackend, models::KnownOneSidedPaymentScript},
+        storage::{
+            database::{OutputManagerBackend, OutputManagerDatabase},
+            models::KnownOneSidedPaymentScript,
+        },
         OutputManagerServiceInitializer,
     },
     storage::database::{WalletBackend, WalletDatabase},
@@ -121,6 +124,7 @@ pub struct Wallet<T, U, V, W, X> {
     pub token_manager: TokenManagerHandle,
     pub updater_service: Option<SoftwareUpdaterHandle>,
     pub db: WalletDatabase<T>,
+    pub output_db: OutputManagerDatabase<V>,
     pub factories: CryptoFactories,
     _u: PhantomData<U>,
     _v: PhantomData<V>,
@@ -142,6 +146,7 @@ where
         node_identity: Arc<NodeIdentity>,
         factories: CryptoFactories,
         wallet_database: WalletDatabase<T>,
+        output_manager_database: OutputManagerDatabase<V>,
         transaction_backend: U,
         output_manager_backend: V,
         contacts_backend: W,
@@ -290,6 +295,7 @@ where
             updater_service: updater_handle,
             wallet_connectivity,
             db: wallet_database,
+            output_db: output_manager_database,
             factories,
             asset_manager: asset_manager_handle,
             token_manager: token_manager_handle,

base_layer/wallet/tests/wallet.rs

@@ -190,19 +190,22 @@ async fn create_wallet(
         ..Default::default()
     };
 
-    let metadata = ChainMetadata::new(std::i64::MAX as u64, Vec::new(), 0, 0, 0);
+    let metadata = ChainMetadata::new(i64::MAX as u64, Vec::new(), 0, 0, 0);
 
     let _db_value = wallet_backend.write(WriteOperation::Insert(DbKeyValuePair::BaseNodeChainMetadata(metadata)));
 
     let wallet_db = WalletDatabase::new(wallet_backend);
     let master_seed = read_or_create_master_seed(recovery_seed, &wallet_db).await?;
 
+    let output_db = OutputManagerDatabase::new(output_manager_backend.clone());
+
     Wallet::start(
         config,
         PeerSeedsConfig::default(),
         Arc::new(node_identity.clone()),
         factories,
         wallet_db,
+        output_db,
         transaction_backend,
         output_manager_backend,
         contacts_backend,
@@ -700,14 +703,17 @@ async fn test_import_utxo() {
         ..Default::default()
     };
 
+    let output_manager_backend = OutputManagerSqliteDatabase::new(connection.clone(), None);
+
     let mut alice_wallet = Wallet::start(
         config,
         PeerSeedsConfig::default(),
         alice_identity.clone(),
         factories.clone(),
         WalletDatabase::new(WalletSqliteDatabase::new(connection.clone(), None).unwrap()),
+        OutputManagerDatabase::new(output_manager_backend.clone()),
         TransactionServiceSqliteDatabase::new(connection.clone(), None),
-        OutputManagerSqliteDatabase::new(connection.clone(), None),
+        output_manager_backend,
         ContactsServiceSqliteDatabase::new(connection.clone()),
         KeyManagerSqliteDatabase::new(connection.clone(), None).unwrap(),
         shutdown.to_signal(),

base_layer/wallet_ffi/Cargo.toml

@@ -30,6 +30,7 @@ rand = "0.8"
 thiserror = "1.0.26"
 tokio = "1.11"
 env_logger = "0.7.0"
+num-traits = "0.2.15"
 
 # <workaround>
 # Temporary workaround until crates utilizing openssl have been updated from security-framework 2.4.0